mirror of
https://github.com/tenox7/wrp.git
synced 2024-09-01 08:29:47 +00:00
add dependencies and goldmark cleanup
This commit is contained in:
parent
d00b1d5d7f
commit
7673e15b9e
5
go.mod
5
go.mod
@ -7,8 +7,10 @@ require (
|
|||||||
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb
|
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb
|
||||||
github.com/chromedp/cdproto v0.0.0-20240519224452-66462be74baa
|
github.com/chromedp/cdproto v0.0.0-20240519224452-66462be74baa
|
||||||
github.com/chromedp/chromedp v0.9.5
|
github.com/chromedp/chromedp v0.9.5
|
||||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2
|
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||||
github.com/soniakeys/quant v1.0.0
|
github.com/soniakeys/quant v1.0.0
|
||||||
|
github.com/yuin/goldmark v1.7.2
|
||||||
|
golang.org/x/image v0.18.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@ -20,7 +22,6 @@ require (
|
|||||||
github.com/gobwas/ws v1.4.0 // indirect
|
github.com/gobwas/ws v1.4.0 // indirect
|
||||||
github.com/josharian/intern v1.0.0 // indirect
|
github.com/josharian/intern v1.0.0 // indirect
|
||||||
github.com/mailru/easyjson v0.7.7 // indirect
|
github.com/mailru/easyjson v0.7.7 // indirect
|
||||||
github.com/yuin/goldmark v1.7.2 // indirect
|
|
||||||
golang.org/x/net v0.25.0 // indirect
|
golang.org/x/net v0.25.0 // indirect
|
||||||
golang.org/x/sys v0.20.0 // indirect
|
golang.org/x/sys v0.20.0 // indirect
|
||||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||||
|
3
go.sum
3
go.sum
@ -22,8 +22,6 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
|
|||||||
github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
|
github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
|
||||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2 h1:yEt5djSYb4iNtmV9iJGVday+i4e9u6Mrn5iP64HH5QM=
|
|
||||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
|
|
||||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||||
@ -53,7 +51,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
|
|
||||||
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||||
github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc=
|
github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc=
|
||||||
github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||||
|
53
wrp.go
53
wrp.go
@ -35,8 +35,6 @@ import (
|
|||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
h2m "github.com/JohannesKaufmann/html-to-markdown"
|
|
||||||
"github.com/JohannesKaufmann/html-to-markdown/plugin"
|
|
||||||
"github.com/MaxHalford/halfgone"
|
"github.com/MaxHalford/halfgone"
|
||||||
"github.com/chromedp/cdproto/css"
|
"github.com/chromedp/cdproto/css"
|
||||||
"github.com/chromedp/cdproto/emulation"
|
"github.com/chromedp/cdproto/emulation"
|
||||||
@ -44,12 +42,6 @@ import (
|
|||||||
"github.com/chromedp/cdproto/page"
|
"github.com/chromedp/cdproto/page"
|
||||||
"github.com/chromedp/chromedp"
|
"github.com/chromedp/chromedp"
|
||||||
"github.com/soniakeys/quant/median"
|
"github.com/soniakeys/quant/median"
|
||||||
"github.com/yuin/goldmark"
|
|
||||||
"github.com/yuin/goldmark/ast"
|
|
||||||
"github.com/yuin/goldmark/extension"
|
|
||||||
"github.com/yuin/goldmark/parser"
|
|
||||||
"github.com/yuin/goldmark/text"
|
|
||||||
"github.com/yuin/goldmark/util"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const version = "4.7.0"
|
const version = "4.7.0"
|
||||||
@ -430,51 +422,6 @@ func asciify(s []byte) []byte {
|
|||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
|
||||||
type astTransformer struct{}
|
|
||||||
|
|
||||||
func (t *astTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
|
|
||||||
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
|
||||||
if link, ok := n.(*ast.Link); ok && entering {
|
|
||||||
link.Destination = append([]byte("?t=txt&url="), link.Destination...)
|
|
||||||
}
|
|
||||||
if _, ok := n.(*ast.Image); ok && entering {
|
|
||||||
// TODO: perhaps instead of deleting images convert them to links
|
|
||||||
// smaller images or ascii? https://github.com/TheZoraiz/ascii-image-converter
|
|
||||||
n.Parent().RemoveChildren(n)
|
|
||||||
}
|
|
||||||
return ast.WalkContinue, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rq *wrpReq) captureMarkdown() {
|
|
||||||
log.Printf("Processing Markdown conversion request for %v", rq.url)
|
|
||||||
// TODO: bug - DomainFromURL always prefixes with http:// instead of https
|
|
||||||
// this causes issues on some websites, fix or write a smarter DomainFromURL
|
|
||||||
c := h2m.NewConverter(h2m.DomainFromURL(rq.url), true, nil)
|
|
||||||
c.Use(plugin.GitHubFlavored())
|
|
||||||
md, err := c.ConvertURL(rq.url) // We could also get inner html from chromedp
|
|
||||||
if err != nil {
|
|
||||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Printf("Got %v bytes md from %v", len(md), rq.url)
|
|
||||||
gm := goldmark.New(
|
|
||||||
goldmark.WithExtensions(extension.GFM),
|
|
||||||
goldmark.WithParserOptions(parser.WithASTTransformers(util.Prioritized(&astTransformer{}, 100))),
|
|
||||||
)
|
|
||||||
var ht bytes.Buffer
|
|
||||||
err = gm.Convert([]byte(md), &ht)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Printf("Rendered %v bytes html for %v", len(ht.String()), rq.url)
|
|
||||||
rq.printHTML(printParams{
|
|
||||||
text: string(asciify([]byte(ht.String()))),
|
|
||||||
bgColor: "#FFFFFF",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process HTTP requests to WRP '/' url
|
// Process HTTP requests to WRP '/' url
|
||||||
func pageServer(w http.ResponseWriter, r *http.Request) {
|
func pageServer(w http.ResponseWriter, r *http.Request) {
|
||||||
log.Printf("%s Page Request for %s [%+v]\n", r.RemoteAddr, r.URL.Path, r.URL.RawQuery)
|
log.Printf("%s Page Request for %s [%+v]\n", r.RemoteAddr, r.URL.Path, r.URL.RawQuery)
|
||||||
|
Loading…
Reference in New Issue
Block a user