// WRP TXT / Simple HTML Mode Routines
package main
// TODO:
// - add image processing times counter to the footer
// - img cache w/garbage collector / test back/button behavior in old browsers
// - add referer header
// - svg support
// - incorrect cert support in both markdown and image download
// - unify cdp and txt image handlers
// - use goroutiness to process images
// - get inner html from chromedp instead of html2markdown
//
// - BUG: DomainFromURL always prefixes with http instead of https
// reproduces on vsi vms docs
// - BUG: markdown table errors
// reproduces on hacker news
// - BUG: captcha errors using html to markdown, perhaps use cdp inner html + downloaded images
// reproduces on https://www.cnn.com/cnn-underscored/electronics
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
h2m "github.com/JohannesKaufmann/html-to-markdown"
"github.com/JohannesKaufmann/html-to-markdown/plugin"
"github.com/lithammer/shortuuid/v4"
"github.com/nfnt/resize"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
"golang.org/x/image/webp"
)
var imgStor imageStore
const imgZpfx = "/imgz/"
func init() {
imgStor.img = make(map[string]imageContainer)
}
type imageContainer struct {
data []byte
url string
added time.Time
}
type imageStore struct {
img map[string]imageContainer
sync.Mutex
}
func (i *imageStore) add(id, url string, img []byte) {
i.Lock()
defer i.Unlock()
i.img[id] = imageContainer{data: img, url: url, added: time.Now()}
}
func (i *imageStore) get(id string) ([]byte, error) {
i.Lock()
defer i.Unlock()
img, ok := i.img[id]
if !ok {
return nil, errors.New("not found")
}
return img.data, nil
}
func (i *imageStore) del(id string) {
i.Lock()
defer i.Unlock()
delete(i.img, id)
}
func fetchImage(id, url, imgType string, maxSize, imgOpt int) (int, error) {
log.Printf("Downloading IMGZ URL=%q for ID=%q", url, id)
var in []byte
var err error
switch url[:4] {
case "http":
r, err := http.Get(url) // TODO: possibly set a header "referer" here
if err != nil {
return 0, fmt.Errorf("Error downloading %q: %v", url, err)
}
if r.StatusCode != http.StatusOK {
return 0, fmt.Errorf("Error %q HTTP Status Code: %v", url, r.StatusCode)
}
defer r.Body.Close()
in, err = io.ReadAll(r.Body)
if err != nil {
return 0, fmt.Errorf("Error reading %q: %v", url, err)
}
case "data":
idx := strings.Index(url, ",")
if idx < 1 {
return 0, fmt.Errorf("image is embeded but unable to find coma: %q", url)
}
in, err = base64.StdEncoding.DecodeString(url[idx+1:])
if err != nil {
return 0, fmt.Errorf("error decoding image from url embed: %q: %v", url, err)
}
}
out, err := smallImg(in, imgType, maxSize, imgOpt)
if err != nil {
return 0, fmt.Errorf("Error scaling down image: %v", err)
}
imgStor.add(id, url, out)
return len(out), nil
}
func smallImg(src []byte, imgType string, maxSize, imgOpt int) ([]byte, error) {
t := http.DetectContentType(src)
var err error
var img image.Image
switch t {
case "image/png":
img, err = png.Decode(bytes.NewReader(src))
case "image/gif":
img, err = gif.Decode(bytes.NewReader(src))
case "image/jpeg":
img, err = jpeg.Decode(bytes.NewReader(src))
case "image/webp":
img, err = webp.Decode(bytes.NewReader(src))
default: // TODO: also add svg
err = errors.New("unknown content type: " + t)
}
if err != nil {
return nil, fmt.Errorf("image decode problem: %v", err)
}
img = resize.Thumbnail(uint(maxSize), uint(maxSize), img, resize.NearestNeighbor)
var outBuf bytes.Buffer
switch imgType {
case "png":
err = png.Encode(&outBuf, img)
case "gif":
err = gif.Encode(&outBuf, gifPalette(img, int64(imgOpt)), &gif.Options{})
case "jpg":
err = jpeg.Encode(&outBuf, img, &jpeg.Options{Quality: imgOpt})
}
if err != nil {
return nil, fmt.Errorf("gif encode problem: %v", err)
}
return outBuf.Bytes(), nil
}
type astTransformer struct {
imgType string
maxSize int
imgOpt int
totSize int
}
func (t *astTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
if link, ok := n.(*ast.Link); ok && entering {
link.Destination = append([]byte("/?m=html&t="+t.imgType+"&s="+strconv.Itoa(t.maxSize)+"&url="), link.Destination...)
}
if img, ok := n.(*ast.Image); ok && entering {
seq := shortuuid.New() + "." + t.imgType
size, err := fetchImage(seq, string(img.Destination), t.imgType, t.maxSize, t.imgOpt) // TODO: use goroutines with waitgroup
if err != nil {
log.Print(err)
n.Parent().RemoveChildren(n)
return ast.WalkContinue, nil
}
img.Destination = []byte(imgZpfx + seq)
t.totSize += size
}
return ast.WalkContinue, nil
})
}
func (rq *wrpReq) captureMarkdown() {
log.Printf("Processing Markdown conversion request for %v", rq.url)
// TODO: bug - DomainFromURL always prefixes with http:// instead of https
// this causes issues on some websites, fix or write a smarter DomainFromURL
c := h2m.NewConverter(h2m.DomainFromURL(rq.url), true, nil)
c.Use(plugin.GitHubFlavored())
md, err := c.ConvertURL(rq.url) // We could also get inner html from chromedp
if err != nil {
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
}
log.Printf("Got %v bytes md from %v", len(md), rq.url)
var imgOpt int
switch rq.imgType {
case "jpg":
imgOpt = int(rq.jQual)
case "gif":
imgOpt = int(rq.nColors)
}
t := &astTransformer{imgType: rq.imgType, maxSize: int(rq.maxSize), imgOpt: imgOpt}
gm := goldmark.New(
goldmark.WithExtensions(extension.GFM),
goldmark.WithParserOptions(parser.WithASTTransformers(util.Prioritized(t, 100))),
)
var ht bytes.Buffer
err = gm.Convert([]byte(md), &ht)
if err != nil {
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
}
log.Printf("Rendered %v bytes html for %v", len(ht.String()), rq.url)
rq.printUI(uiParams{
text: string(asciify([]byte(ht.String()))),
bgColor: "#FFFFFF",
imgSize: fmt.Sprintf("%.0f KB", float32(t.totSize)/1024.0),
})
}
func imgServerTxt(w http.ResponseWriter, r *http.Request) {
log.Printf("%s IMGZ Request for %s", r.RemoteAddr, r.URL.Path)
id := strings.Replace(r.URL.Path, imgZpfx, "", 1)
img, err := imgStor.get(id)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Printf("%s IMGZ error for %s: %v", r.RemoteAddr, r.URL.Path, err)
return
}
imgStor.del(id)
w.Header().Set("Content-Type", http.DetectContentType(img))
w.Header().Set("Content-Length", strconv.Itoa(len(img)))
w.Write(img)
w.(http.Flusher).Flush()
}