From d8865df7a9f73980febb734ed191d84ad09d9936 Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Thu, 20 Jun 2019 16:20:51 +0000 Subject: [PATCH] Implements embedded TOR support --- commands/commands.go | 13 ++++++++++++- main.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/commands/commands.go b/commands/commands.go index c0f812d..b1eb2cf 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -25,6 +25,11 @@ import ( "golang.org/x/net/html/charset" ) +var ( + Transport *http.Transport = nil + UserAgent string +) + func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) { doc.Find("a").Each(func(i int, s *goquery.Selection) { @@ -133,6 +138,9 @@ func handleResponse(db *sql.DB, resp *http.Response, lastUrl *url.URL) { func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) { client := &http.Client{Jar: jar} + if Transport != nil { + client.Transport = Transport + } var lastUrl *url.URL @@ -155,7 +163,7 @@ func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) { log.Fatalln(err) } - req.Header.Set("User-Agent", "Wi 0.0") + req.Header.Set("User-Agent", UserAgent) resp, err := client.Do(req) if err != nil { @@ -214,6 +222,9 @@ func Form(db *sql.DB, jar *cookiejar.Jar, formID int64, formArgs []string) { } client := &http.Client{Jar: jar} + if Transport != nil { + client.Transport = Transport + } var lastUrl *url.URL diff --git a/main.go b/main.go index 2dbb8c2..1e922c2 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,8 @@ package main import ( + "context" + "net/http" "os" "strings" @@ -17,8 +19,20 @@ import ( cookiejar "github.com/juju/persistent-cookiejar" kingpin "gopkg.in/alecthomas/kingpin.v2" + + "github.com/cretz/bine/process" + "github.com/cretz/bine/tor" + "github.com/ipsn/go-libtor" ) +var creator = libtor.Creator + +type LibTorWrapper struct{} + +func (LibTorWrapper) New(ctx context.Context, args ...string) (process.Process, error) { + return creator.New(ctx, args...) +} + type searchList []string func (l *searchList) Set(value string) (err error) { @@ -41,6 +55,9 @@ func SearchList(settings kingpin.Settings) (target *[]string) { } var ( + useTor = kingpin.Flag("tor", "Use embedded tor").Default("false").Bool() + ua = kingpin.Flag("ua", "User-Agent").Default("Wi 0.1").String() + get = kingpin.Command("get", "Get url") getUrl = get.Arg("url", "Url").Required().String() @@ -93,6 +110,30 @@ func main() { defer jar.Save() + kingpin.Parse() + + var t *tor.Tor + if *useTor { + t, err = tor.Start(nil, &tor.StartConf{ + ProcessCreator: LibTorWrapper{}, + DataDir: wiDir + "/tor", + }) + if err != nil { + panic(err) + } + + defer t.Close() + + dialer, err := t.Dialer(nil, nil) + if err != nil { + panic(err) + } + + commands.Transport = &http.Transport{DialContext: dialer.DialContext} + } + + commands.UserAgent = *ua + switch kingpin.Parse() { case "get": commands.Get(db, jar, *getUrl)