From 9ff8f882ff15f84e62940ddd63e0655ca3ca8cfa Mon Sep 17 00:00:00 2001 From: sergeyfrolov Date: Fri, 15 Jun 2018 16:36:08 -0400 Subject: [PATCH] Add upstream proxy support, fix tests, cosmetics (#27) --- README.md | 10 ++- common_test.go | 26 ++++++- forwardproxy.go | 6 +- forwardproxy_test.go | 80 +++++++++++++++++++--- setup.go | 113 +++++++++++++++++++++++++++++-- setup_test.go | 11 ++- test/upstreamingproxy/index.html | 1 + test/upstreamingproxy/pic.png | Bin 0 -> 2155 bytes 8 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 test/upstreamingproxy/index.html create mode 100644 test/upstreamingproxy/pic.png diff --git a/README.md b/README.md index d15b132..7bf0036 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,15 @@ Open a block for more control; here's an example of all properties in use (note ``` forwardproxy { basicauth user1 0NtCL2JPJBgPPMmlPcJ - basicauth user2 秘密 + basicauth user2 密码 ports 80 443 hide_ip + hide_via probe_resistance secretlink.localhost serve_pac /secret-proxy.pac response_timeout 30 dial_timeout 30 + upstream https://user:password@extra-upstream-hop.com } ``` @@ -59,6 +61,12 @@ _Default: no timeout (other timeouts will eventually close the connection)._ Sets timeout (in seconds) for establishing TCP connection to target website. Affects all requests. _Default: 20 seconds._ +- **upstream [https://username:password@upstreamproxy.site:443]** +Sets upstream proxy to route all forwardproxy requests through it. +This setting does not affect non-forwardproxy requests nor requests with wrong credentials. +Supported schemes to remote host: https. +Supported schemes to localhost: socks5, http, https(certificate check is ignored). +_Default: no upstream proxy._ ## Client Configuration diff --git a/common_test.go b/common_test.go index 8134f1c..c3efb71 100644 --- a/common_test.go +++ b/common_test.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "encoding/hex" "fmt" - "github.com/mholt/caddy" "io" "io/ioutil" "net/http" @@ -13,10 +12,13 @@ import ( "strings" "testing" "time" + + "github.com/mholt/caddy" ) var credentialsEmpty = "" -var credentialsCorrect = "Basic dGVzdDpwYXNz" // test:pass +var credentialsCorrect = "Basic dGVzdDpwYXNz" // test:pass +var credentialsUpstreamCorrect = "basic dXBzdHJlYW10ZXN0OnVwc3RyZWFtcGFzcw==" // upstreamtest:upstreampass var credentialsWrong = []string{ "", "\"\"", @@ -53,7 +55,12 @@ var ( caddyForwardProxyAuth caddyTestServer // requires auth caddyForwardProxyProbeResist caddyTestServer // requires auth, and has probing resistance on caddyDummyProbeResist caddyTestServer // same as caddyForwardProxyProbeResist, but w/o forwardproxy - caddyTestTarget caddyTestServer + + // authenticated server upstreams to authenticated https proxy with different credentials + caddyAuthedUpstreamEnter caddyTestServer + + caddyTestTarget caddyTestServer + caddyHTTPTestTarget caddyTestServer ) func (c *caddyTestServer) marshal() []byte { @@ -137,6 +144,17 @@ func TestMain(m *testing.M) { proxyEnabled: false} caddyTestTarget.StartTestServer() + caddyHTTPTestTarget = caddyTestServer{addr: "localhost:6480", root: "./test/index", + directives: []string{"tls off"}, + proxyEnabled: false} + caddyHTTPTestTarget.StartTestServer() + + caddyAuthedUpstreamEnter = caddyTestServer{addr: "127.0.0.1:6585", root: "./test/upstreamingproxy", + directives: []string{"tls self_signed"}, + proxyEnabled: true, proxyDirectives: []string{"upstream https://test:pass@127.0.0.1:4891", + "basicauth upstreamtest upstreampass"}} + caddyAuthedUpstreamEnter.StartTestServer() + retCode := m.Run() caddyForwardProxy.Stop() @@ -144,6 +162,8 @@ func TestMain(m *testing.M) { caddyForwardProxyProbeResist.Stop() caddyDummyProbeResist.Stop() caddyTestTarget.Stop() + caddyHTTPTestTarget.Stop() + caddyAuthedUpstreamEnter.Stop() os.Exit(retCode) } diff --git a/forwardproxy.go b/forwardproxy.go index 52ca353..e938256 100644 --- a/forwardproxy.go +++ b/forwardproxy.go @@ -45,6 +45,10 @@ type ForwardProxy struct { dialTimeout time.Duration // for initial tcp connection hostname string // do not intercept requests to the hostname (except for hidden link) port string // port on which chain with forwardproxy is listening on + + // overridden dial allows to redirect requests to upstream proxy + dial func(network, address string) (net.Conn, error) + upstream string // address of upstream proxy } var bufferPool sync.Pool @@ -263,7 +267,7 @@ func (fp *ForwardProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, return http.StatusForbidden, errors.New("CONNECT port not allowed for " + r.URL.String()) } - targetConn, err := net.DialTimeout("tcp", r.URL.Hostname()+":"+r.URL.Port(), fp.dialTimeout) + targetConn, err := fp.dial("tcp", r.URL.Hostname()+":"+r.URL.Port()) if err != nil { return http.StatusBadGateway, errors.New(fmt.Sprintf("Dial %s failed: %v", r.URL.String(), err)) } diff --git a/forwardproxy_test.go b/forwardproxy_test.go index 30564ac..39fac6f 100644 --- a/forwardproxy_test.go +++ b/forwardproxy_test.go @@ -19,10 +19,6 @@ import ( "crypto/tls" "errors" "fmt" - _ "github.com/mholt/caddy/caddyhttp/header" - _ "github.com/mholt/caddy/caddyhttp/httpserver" - _ "github.com/mholt/caddy/caddyhttp/redirect" - _ "github.com/mholt/caddy/caddyhttp/root" "io" "net" "net/http" @@ -30,6 +26,11 @@ import ( "strings" "testing" "time" + + _ "github.com/mholt/caddy/caddyhttp/header" + _ "github.com/mholt/caddy/caddyhttp/httpserver" + _ "github.com/mholt/caddy/caddyhttp/redirect" + _ "github.com/mholt/caddy/caddyhttp/root" ) func dial(proxyAddr string, useTls bool) (net.Conn, error) { @@ -196,10 +197,10 @@ func TestGETNoAuth(t *testing.T) { useTls := true for _, httpTargetVer := range testHttpVersions { for _, resource := range testResources { - response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxy.addr, httpTargetVer, credentialsEmpty, useTls) + response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxy.addr, httpTargetVer, credentialsEmpty, useTls) if err != nil { t.Fatal(err) - } else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil { + } else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil { t.Fatal(err) } } @@ -210,10 +211,10 @@ func TestGETAuthCorrect(t *testing.T) { useTls := true for _, httpTargetVer := range testHttpVersions { for _, resource := range testResources { - response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, credentialsCorrect, useTls) + response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, credentialsCorrect, useTls) if err != nil { t.Fatal(err) - } else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil { + } else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil { t.Fatal(err) } } @@ -225,7 +226,7 @@ func TestGETAuthWrong(t *testing.T) { for _, wrongCreds := range credentialsWrong { for _, httpTargetVer := range testHttpVersions { for _, resource := range testResources { - response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, wrongCreds, useTls) + response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, wrongCreds, useTls) if err != nil { t.Fatal(err) } @@ -345,3 +346,64 @@ func TestPAC(t *testing.T) { t.Fatal(err) } } + +func TestCONNECTViaUpstream(t *testing.T) { + useTls := true + for _, httpProxyVer := range testHttpVersions { + for _, httpTargetVer := range testHttpVersions { + for _, resource := range testResources { + response, err := connectAndGetViaProxy(caddyTestTarget.addr, resource, caddyAuthedUpstreamEnter.addr, + httpTargetVer, credentialsUpstreamCorrect, httpProxyVer, useTls) + if err != nil { + t.Fatal(err) + } else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil { + t.Fatal(err) + } + } + } + } +} + +func TestGETViaUpstream(t *testing.T) { + useTls := true + for _, httpTargetVer := range testHttpVersions { + for _, resource := range testResources { + response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyAuthedUpstreamEnter.addr, httpTargetVer, + credentialsUpstreamCorrect, useTls) + if err != nil { + t.Fatal(err) + } else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil { + t.Fatal(err) + } + } + } +} + +func TestUpstreamPassthrough(t *testing.T) { + // Usptreaming proxy still hosts things as expected + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + ResponseHeaderTimeout: 2 * time.Second, + } + client := &http.Client{Transport: tr, Timeout: 2 * time.Second} + resp, err := client.Get("https://" + caddyAuthedUpstreamEnter.addr) + if err != nil { + t.Fatal(err) + } else if err = responseExpected(resp, caddyAuthedUpstreamEnter.contents[""]); err != nil { + t.Fatal(err) + } + + resp, err = client.Get("https://" + caddyAuthedUpstreamEnter.addr + "/pic.png") + if err != nil { + t.Fatal(err) + } else if err = responseExpected(resp, caddyAuthedUpstreamEnter.contents["/pic.png"]); err != nil { + t.Fatal(err) + } + + resp, err = client.Get("https://" + caddyAuthedUpstreamEnter.addr + "/idontexist") + if err != nil { + t.Fatal(err) + } else if resp.StatusCode != http.StatusNotFound { + t.Fatalf("Expected: 404 StatusNotFound, got %d. Response: %#v\n", resp.StatusCode, resp) + } +} diff --git a/setup.go b/setup.go index deaa2ef..3a67a56 100644 --- a/setup.go +++ b/setup.go @@ -17,15 +17,22 @@ package forwardproxy import ( "encoding/base64" "errors" - "github.com/mholt/caddy" - "github.com/mholt/caddy/caddyhttp/httpserver" "log" "net" "net/http" + "net/url" "strconv" "strings" "sync" "time" + + "bufio" + "crypto/tls" + "fmt" + + "github.com/mholt/caddy" + "github.com/mholt/caddy/caddyhttp/httpserver" + "golang.org/x/net/proxy" ) func setup(c *caddy.Controller) error { @@ -151,6 +158,11 @@ func setup(c *caddy.Controller) error { return errors.New("Parse error: dial_timeout cannot be negative.") } fp.dialTimeout = time.Second * time.Duration(timeout) + case "upstream": + if len(args) != 1 { + return c.ArgErr() + } + fp.upstream = args[0] default: return c.ArgErr() } @@ -165,11 +177,42 @@ func setup(c *caddy.Controller) error { } } - fp.httpTransport.DialContext = (&net.Dialer{ + dialer := &net.Dialer{ Timeout: fp.dialTimeout, KeepAlive: 30 * time.Second, DualStack: true, - }).DialContext + } + + if fp.upstream != "" { + upstreamURL, err := url.Parse(fp.upstream) + if err != nil { + return errors.New("failed to parse upstream address: " + err.Error()) + } + + if !isLocalhost(upstreamURL) && upstreamURL.Scheme != "https" { + return errors.New("insecure schemes are only allowed to localhost upstreams") + } + + // TODO: remove homebrewed Dialer when https://go-review.googlesource.com/c/net/+/111135 gets merged + proxy.RegisterDialerType("https", func(u *url.URL, _ proxy.Dialer) (proxy.Dialer, error) { + // CONNECT request is proxied as-is, so we don't care about target url, but it could be + // useful in future to implement policies of choosing between multiple upstream servers. + // Given dialer is not used, since it's the same dialer provided by us. + return NewHTTPDialer(dialer, true, upstreamURL), nil + }) + proxy.RegisterDialerType("http", func(u *url.URL, _ proxy.Dialer) (proxy.Dialer, error) { + return NewHTTPDialer(dialer, false, upstreamURL), nil + }) + newDialer, err := proxy.FromURL(upstreamURL, dialer) + if err != nil { + return errors.New("failed to create proxy to upstream: " + err.Error()) + } + fp.dial = newDialer.Dial + fp.httpTransport.Dial = newDialer.Dial + } else { + fp.dial = dialer.Dial + fp.httpTransport.DialContext = dialer.DialContext + } httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler { fp.Next = next @@ -187,3 +230,65 @@ func init() { Action: setup, }) } + +type HTTPDialer struct { + dialer *net.Dialer + upstreamUrl string + tlsConf *tls.Config + extraHeaders string // empty or whole lines together with \r\n\r\n +} + +func NewHTTPDialer(dialer *net.Dialer, useHTTPS bool, upstream *url.URL) *HTTPDialer { + d := &HTTPDialer{ + dialer: dialer, + upstreamUrl: upstream.Host, + tlsConf: nil, + } + if useHTTPS { + d.tlsConf = &tls.Config{ServerName: upstream.Hostname()} + if isLocalhost(upstream) { + log.Println("Localhost upstream detected, disabling verification of TLS ceritifcate") + d.tlsConf.InsecureSkipVerify = true + } + } + if upstream.User != nil { + d.extraHeaders = fmt.Sprintf("Proxy-Authorization: basic %s\r\n", + base64.StdEncoding.EncodeToString([]byte(upstream.User.String()))) + } + + return d +} + +func (d *HTTPDialer) Dial(network, addr string) (net.Conn, error) { + var err error + var c net.Conn + if d.tlsConf == nil { + c, err = d.dialer.Dial(network, d.upstreamUrl) + } else { + c, err = tls.DialWithDialer(d.dialer, network, d.upstreamUrl, d.tlsConf) + } + if err != nil { + return nil, err + } + // TODO: multiplexed http/2 to upstream, also will eventually be added to x/net/proxy + _, err = fmt.Fprintf(c, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n%s\r\n", addr, addr, d.extraHeaders) + if err != nil { + return nil, err + } + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Upstream responded with " + resp.Status) + } + return c, nil +} + +func isLocalhost(u *url.URL) bool { + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" || + u.Hostname() == "::1" { + return true + } + return false +} diff --git a/setup_test.go b/setup_test.go index 5686e44..d987869 100644 --- a/setup_test.go +++ b/setup_test.go @@ -15,8 +15,9 @@ package forwardproxy import ( - "github.com/mholt/caddy" "testing" + + "github.com/mholt/caddy" ) func TestSetup(t *testing.T) { @@ -111,4 +112,12 @@ func TestSetup(t *testing.T) { testParsing([]string{"dial_timeout 1 2"}, false) testParsing([]string{"dial_timeout seven"}, false) testParsing([]string{"dial_timeout 2"}, true) + + testParsing([]string{"upstream proxy.site"}, false) + testParsing([]string{"upstream https://proxy.site https://proxy.site"}, false) + testParsing([]string{"upstream http://localhost:1230"}, true) + testParsing([]string{"upstream socks5://127.0.0.1:999"}, true) + testParsing([]string{"upstream http://proxy.site"}, false) + testParsing([]string{"upstream https://proxy.site https://proxy.site"}, false) + testParsing([]string{"upstream https://proxy.site"}, true) } diff --git a/test/upstreamingproxy/index.html b/test/upstreamingproxy/index.html new file mode 100644 index 0000000..9a32a4c --- /dev/null +++ b/test/upstreamingproxy/index.html @@ -0,0 +1 @@ +I am upstreaming ForwardProxy(don't tell anyone) diff --git a/test/upstreamingproxy/pic.png b/test/upstreamingproxy/pic.png new file mode 100644 index 0000000000000000000000000000000000000000..fa1b2adf60cad874164e933ef02917e48c208105 GIT binary patch literal 2155 zcmb`I`#;l*AIB%vNg7A!au(_chs`az@jxz1sLAE5F}Eq4tht1VebLEfp>V_`=8^~- zeT)rr-<%xe+7M$ZXNE;zj3u^?9_L>;@5kf)ct0NR$LsNYy?%H<9`Dra2z&6spALdR zAh5#~J68~B4+;d5$=WNsySWdMrR)yK^{ei1KgTAO<6hS+N|ClyBWS$s`CK)PARVm; zGvhR?So*PDnOxWvj|dP*q5FHwyi_$vaG{~G}xr+ zfnAgTw(kEB8hSmtD>Cdxs~2@ zKz+g`zyd?Ji^i|s4HJyJyXaD{9}2Fd6#*%2{7t>Zu9UNJkz$fN{Q%i#+Pg0D6+G4n0mVKcO1v6ZXwPnvk;iSiw#|K zu=uU*5uu~twsWME_+w~-Q8_*_^dm1lQfQ>4aW%Eq!%xq#xYg3c{3tM8U z-f-#$B%VXCA{b~Cmai{!Dr)VEMQ_vEqxE8|c^l{j1HU()-V@q-LZmi2xTG&x()@jE z5Yy1xn`tdIH;kzj_Zl_G1w`_J~Nu)WI|hiY8GN-f|0B^7sxrKnWf9>UI9zt43j(HTYD^GJ{$1ho(de z-%7i=@#0Dbhg%fFADRRhq6V5)Sw^>E#RGrT+^A)CriVPHxuphu3dT(!dZQBt!Pr2w z@>HNs#|1b*QQTl z7`JWR7V>Tc6UTu^8j9Go*~CM7!W?oo8)!(M{x=w_k4ZcMwGr?;6`|0)4RMBf=N4u$ zrBT#FlWo=ADhC*n($1!s`_XuGW41$8sm&E8Vy+FK`4oXW^|@fN_WGQ%8p%>FCnW%y z4MCs~QGrV*eR3MMUjx%K$Zkf2DUf>nOz? zv5HytaX<}LUm{Bjfwiq`CTJ%3lUoc*dbis$7%7j{v->8|crMUiXoXzROo(sI1`79A zO%_`X^)WQ^@!(G;yHY00a<*0ld3t|d9jf(~AbCtQA%e>F=W9?9BqCqkXa2n&7|SI~ z;0(JmR}{H{Yqu?bvq@u|6?N-8%q?J>$L=${zpzx#e=u>OK3f-zT`-WVe--1UOATCv z&2(fg8CH_}`xq{4f+F@Qc;DApcgBynW_d*uMLlqTfqkG>cCZCSz!M+`7P$tEkQW2{och5$7 zBbAmO#8WhU6OdT)B`Pq~^iBEPDw1CR0VQ%4)ZdU+@{?7+=2?<~%dr^|!yRH^v@4)Q zI7;XC)6_PMI5mwohRE~g7_}xr)gmcg{K69zXgxV3ex^EYO#fnEtxRf%DZO6bR-|Zs zUF%7ieTp@BJb=46DW@Ezg6CknAP^!`SE*NDhiZgOUtVyvq@Q}F)bMd2?TKq#l|SZr zxjbS_trz*p_ii{VIBoWCig5sw4bdGn^}0SFNh_~!Q6`Ku zR=V=!nT$B!Q@f-7W-3$Qc>wwa>HTCx{DQAP$m^-@R}+@}-Ci18>{BF|2HX>U_d3Zz z59qAlz@y0p+Vwj@okQ9Y;`3e3^GDMo?3;<{G3PE=6%08#PBV0+9Jh3XcA!7(3}olC z-sku2?O9yC4vV@9q#AO|n{^hBQl*-~6xdvJZzxCFs7f63SxYc#E*MMN=ni?rkW$_; zJ+7hVqGh+7MXx(#+gFgv<8C9{Vd2IO-D%E-sx`m8F`KZ0$BDBpQzQ96{qtCzp3lyC hI|=$S|IK%;O^{g&zfU