Add upstream proxy support, fix tests, cosmetics (#27)

README.md

@@ -13,13 +13,15 @@ Open a block for more control; here's an example of all properties in use (note
 ```
 forwardproxy {
     basicauth user1 0NtCL2JPJBgPPMmlPcJ
-    basicauth user2 秘密
+    basicauth user2 密码
     ports     80 443
     hide_ip
+    hide_via
     probe_resistance secretlink.localhost
     serve_pac        /secret-proxy.pac
     response_timeout 30
     dial_timeout     30
+    upstream         https://user:password@extra-upstream-hop.com
 }
 ```
 
@@ -59,6 +61,12 @@ _Default: no timeout (other timeouts will eventually close the connection)._
 Sets timeout (in seconds) for establishing TCP connection to target website. Affects all requests.  
 _Default: 20 seconds._
 
+- **upstream [https://username:password@upstreamproxy.site:443]**  
+Sets upstream proxy to route all forwardproxy requests through it.
+This setting does not affect non-forwardproxy requests nor requests with wrong credentials.  
+Supported schemes to remote host: https.  
+Supported schemes to localhost: socks5, http, https(certificate check is ignored).  
+_Default: no upstream proxy._
 
 ## Client Configuration
 

common_test.go

@@ -4,7 +4,6 @@ import (
 	"crypto/tls"
 	"encoding/hex"
 	"fmt"
-	"github.com/mholt/caddy"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -13,10 +12,13 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/mholt/caddy"
 )
 
 var credentialsEmpty = ""
-var credentialsCorrect = "Basic dGVzdDpwYXNz" // test:pass
+var credentialsCorrect = "Basic dGVzdDpwYXNz"                                 // test:pass
+var credentialsUpstreamCorrect = "basic dXBzdHJlYW10ZXN0OnVwc3RyZWFtcGFzcw==" // upstreamtest:upstreampass
 var credentialsWrong = []string{
 	"",
 	"\"\"",
@@ -53,7 +55,12 @@ var (
 	caddyForwardProxyAuth        caddyTestServer // requires auth
 	caddyForwardProxyProbeResist caddyTestServer // requires auth, and has probing resistance on
 	caddyDummyProbeResist        caddyTestServer // same as caddyForwardProxyProbeResist, but w/o forwardproxy
-	caddyTestTarget              caddyTestServer
+
+	// authenticated server upstreams to authenticated https proxy with different credentials
+	caddyAuthedUpstreamEnter caddyTestServer
+
+	caddyTestTarget     caddyTestServer
+	caddyHTTPTestTarget caddyTestServer
 )
 
 func (c *caddyTestServer) marshal() []byte {
@@ -137,6 +144,17 @@ func TestMain(m *testing.M) {
 		proxyEnabled: false}
 	caddyTestTarget.StartTestServer()
 
+	caddyHTTPTestTarget = caddyTestServer{addr: "localhost:6480", root: "./test/index",
+		directives:   []string{"tls off"},
+		proxyEnabled: false}
+	caddyHTTPTestTarget.StartTestServer()
+
+	caddyAuthedUpstreamEnter = caddyTestServer{addr: "127.0.0.1:6585", root: "./test/upstreamingproxy",
+		directives:   []string{"tls self_signed"},
+		proxyEnabled: true, proxyDirectives: []string{"upstream https://test:pass@127.0.0.1:4891",
+			"basicauth upstreamtest upstreampass"}}
+	caddyAuthedUpstreamEnter.StartTestServer()
+
 	retCode := m.Run()
 
 	caddyForwardProxy.Stop()
@@ -144,6 +162,8 @@ func TestMain(m *testing.M) {
 	caddyForwardProxyProbeResist.Stop()
 	caddyDummyProbeResist.Stop()
 	caddyTestTarget.Stop()
+	caddyHTTPTestTarget.Stop()
+	caddyAuthedUpstreamEnter.Stop()
 
 	os.Exit(retCode)
 }

forwardproxy.go

@@ -45,6 +45,10 @@ type ForwardProxy struct {
 	dialTimeout        time.Duration // for initial tcp connection
 	hostname           string        // do not intercept requests to the hostname (except for hidden link)
 	port               string        // port on which chain with forwardproxy is listening on
+
+	// overridden dial allows to redirect requests to upstream proxy
+	dial     func(network, address string) (net.Conn, error)
+	upstream string // address of upstream proxy
 }
 
 var bufferPool sync.Pool
@@ -263,7 +267,7 @@ func (fp *ForwardProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int,
 			return http.StatusForbidden, errors.New("CONNECT port not allowed for " + r.URL.String())
 		}
 
-		targetConn, err := net.DialTimeout("tcp", r.URL.Hostname()+":"+r.URL.Port(), fp.dialTimeout)
+		targetConn, err := fp.dial("tcp", r.URL.Hostname()+":"+r.URL.Port())
 		if err != nil {
 			return http.StatusBadGateway, errors.New(fmt.Sprintf("Dial %s failed: %v", r.URL.String(), err))
 		}

forwardproxy_test.go

@@ -19,10 +19,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	_ "github.com/mholt/caddy/caddyhttp/header"
-	_ "github.com/mholt/caddy/caddyhttp/httpserver"
-	_ "github.com/mholt/caddy/caddyhttp/redirect"
-	_ "github.com/mholt/caddy/caddyhttp/root"
 	"io"
 	"net"
 	"net/http"
@@ -30,6 +26,11 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	_ "github.com/mholt/caddy/caddyhttp/header"
+	_ "github.com/mholt/caddy/caddyhttp/httpserver"
+	_ "github.com/mholt/caddy/caddyhttp/redirect"
+	_ "github.com/mholt/caddy/caddyhttp/root"
 )
 
 func dial(proxyAddr string, useTls bool) (net.Conn, error) {
@@ -196,10 +197,10 @@ func TestGETNoAuth(t *testing.T) {
 	useTls := true
 	for _, httpTargetVer := range testHttpVersions {
 		for _, resource := range testResources {
-			response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxy.addr, httpTargetVer, credentialsEmpty, useTls)
+			response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxy.addr, httpTargetVer, credentialsEmpty, useTls)
 			if err != nil {
 				t.Fatal(err)
-			} else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil {
+			} else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil {
 				t.Fatal(err)
 			}
 		}
@@ -210,10 +211,10 @@ func TestGETAuthCorrect(t *testing.T) {
 	useTls := true
 	for _, httpTargetVer := range testHttpVersions {
 		for _, resource := range testResources {
-			response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, credentialsCorrect, useTls)
+			response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, credentialsCorrect, useTls)
 			if err != nil {
 				t.Fatal(err)
-			} else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil {
+			} else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil {
 				t.Fatal(err)
 			}
 		}
@@ -225,7 +226,7 @@ func TestGETAuthWrong(t *testing.T) {
 	for _, wrongCreds := range credentialsWrong {
 		for _, httpTargetVer := range testHttpVersions {
 			for _, resource := range testResources {
-				response, err := getViaProxy(caddyTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, wrongCreds, useTls)
+				response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyForwardProxyAuth.addr, httpTargetVer, wrongCreds, useTls)
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -345,3 +346,64 @@ func TestPAC(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestCONNECTViaUpstream(t *testing.T) {
+	useTls := true
+	for _, httpProxyVer := range testHttpVersions {
+		for _, httpTargetVer := range testHttpVersions {
+			for _, resource := range testResources {
+				response, err := connectAndGetViaProxy(caddyTestTarget.addr, resource, caddyAuthedUpstreamEnter.addr,
+					httpTargetVer, credentialsUpstreamCorrect, httpProxyVer, useTls)
+				if err != nil {
+					t.Fatal(err)
+				} else if err = responseExpected(response, caddyTestTarget.contents[resource]); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+	}
+}
+
+func TestGETViaUpstream(t *testing.T) {
+	useTls := true
+	for _, httpTargetVer := range testHttpVersions {
+		for _, resource := range testResources {
+			response, err := getViaProxy(caddyHTTPTestTarget.addr, resource, caddyAuthedUpstreamEnter.addr, httpTargetVer,
+				credentialsUpstreamCorrect, useTls)
+			if err != nil {
+				t.Fatal(err)
+			} else if err = responseExpected(response, caddyHTTPTestTarget.contents[resource]); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}
+
+func TestUpstreamPassthrough(t *testing.T) {
+	// Usptreaming proxy still hosts things as expected
+	tr := &http.Transport{
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
+		ResponseHeaderTimeout: 2 * time.Second,
+	}
+	client := &http.Client{Transport: tr, Timeout: 2 * time.Second}
+	resp, err := client.Get("https://" + caddyAuthedUpstreamEnter.addr)
+	if err != nil {
+		t.Fatal(err)
+	} else if err = responseExpected(resp, caddyAuthedUpstreamEnter.contents[""]); err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err = client.Get("https://" + caddyAuthedUpstreamEnter.addr + "/pic.png")
+	if err != nil {
+		t.Fatal(err)
+	} else if err = responseExpected(resp, caddyAuthedUpstreamEnter.contents["/pic.png"]); err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err = client.Get("https://" + caddyAuthedUpstreamEnter.addr + "/idontexist")
+	if err != nil {
+		t.Fatal(err)
+	} else if resp.StatusCode != http.StatusNotFound {
+		t.Fatalf("Expected: 404 StatusNotFound, got %d. Response: %#v\n", resp.StatusCode, resp)
+	}
+}

setup.go

@@ -17,15 +17,22 @@ package forwardproxy
 import (
 	"encoding/base64"
 	"errors"
-	"github.com/mholt/caddy"
-	"github.com/mholt/caddy/caddyhttp/httpserver"
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
+
+	"bufio"
+	"crypto/tls"
+	"fmt"
+
+	"github.com/mholt/caddy"
+	"github.com/mholt/caddy/caddyhttp/httpserver"
+	"golang.org/x/net/proxy"
 )
 
 func setup(c *caddy.Controller) error {
@@ -151,6 +158,11 @@ func setup(c *caddy.Controller) error {
 				return errors.New("Parse error: dial_timeout cannot be negative.")
 			}
 			fp.dialTimeout = time.Second * time.Duration(timeout)
+		case "upstream":
+			if len(args) != 1 {
+				return c.ArgErr()
+			}
+			fp.upstream = args[0]
 		default:
 			return c.ArgErr()
 		}
@@ -165,11 +177,42 @@ func setup(c *caddy.Controller) error {
 		}
 	}
 
-	fp.httpTransport.DialContext = (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   fp.dialTimeout,
 		KeepAlive: 30 * time.Second,
 		DualStack: true,
-	}).DialContext
+	}
+
+	if fp.upstream != "" {
+		upstreamURL, err := url.Parse(fp.upstream)
+		if err != nil {
+			return errors.New("failed to parse upstream address: " + err.Error())
+		}
+
+		if !isLocalhost(upstreamURL) && upstreamURL.Scheme != "https" {
+			return errors.New("insecure schemes are only allowed to localhost upstreams")
+		}
+
+		// TODO: remove homebrewed Dialer when https://go-review.googlesource.com/c/net/+/111135 gets merged
+		proxy.RegisterDialerType("https", func(u *url.URL, _ proxy.Dialer) (proxy.Dialer, error) {
+			// CONNECT request is proxied as-is, so we don't care about target url, but it could be
+			// useful in future to implement policies of choosing between multiple upstream servers.
+			// Given dialer is not used, since it's the same dialer provided by us.
+			return NewHTTPDialer(dialer, true, upstreamURL), nil
+		})
+		proxy.RegisterDialerType("http", func(u *url.URL, _ proxy.Dialer) (proxy.Dialer, error) {
+			return NewHTTPDialer(dialer, false, upstreamURL), nil
+		})
+		newDialer, err := proxy.FromURL(upstreamURL, dialer)
+		if err != nil {
+			return errors.New("failed to create proxy to upstream: " + err.Error())
+		}
+		fp.dial = newDialer.Dial
+		fp.httpTransport.Dial = newDialer.Dial
+	} else {
+		fp.dial = dialer.Dial
+		fp.httpTransport.DialContext = dialer.DialContext
+	}
 
 	httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler {
 		fp.Next = next
@@ -187,3 +230,65 @@ func init() {
 		Action:     setup,
 	})
 }
+
+type HTTPDialer struct {
+	dialer       *net.Dialer
+	upstreamUrl  string
+	tlsConf      *tls.Config
+	extraHeaders string // empty or whole lines together with \r\n\r\n
+}
+
+func NewHTTPDialer(dialer *net.Dialer, useHTTPS bool, upstream *url.URL) *HTTPDialer {
+	d := &HTTPDialer{
+		dialer:      dialer,
+		upstreamUrl: upstream.Host,
+		tlsConf:     nil,
+	}
+	if useHTTPS {
+		d.tlsConf = &tls.Config{ServerName: upstream.Hostname()}
+		if isLocalhost(upstream) {
+			log.Println("Localhost upstream detected, disabling verification of TLS ceritifcate")
+			d.tlsConf.InsecureSkipVerify = true
+		}
+	}
+	if upstream.User != nil {
+		d.extraHeaders = fmt.Sprintf("Proxy-Authorization: basic %s\r\n",
+			base64.StdEncoding.EncodeToString([]byte(upstream.User.String())))
+	}
+
+	return d
+}
+
+func (d *HTTPDialer) Dial(network, addr string) (net.Conn, error) {
+	var err error
+	var c net.Conn
+	if d.tlsConf == nil {
+		c, err = d.dialer.Dial(network, d.upstreamUrl)
+	} else {
+		c, err = tls.DialWithDialer(d.dialer, network, d.upstreamUrl, d.tlsConf)
+	}
+	if err != nil {
+		return nil, err
+	}
+	// TODO: multiplexed http/2 to upstream, also will eventually be added to x/net/proxy
+	_, err = fmt.Fprintf(c, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n%s\r\n", addr, addr, d.extraHeaders)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.ReadResponse(bufio.NewReader(c), nil)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("Upstream responded with " + resp.Status)
+	}
+	return c, nil
+}
+
+func isLocalhost(u *url.URL) bool {
+	if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" ||
+		u.Hostname() == "::1" {
+		return true
+	}
+	return false
+}

setup_test.go

@@ -15,8 +15,9 @@
 package forwardproxy
 
 import (
-	"github.com/mholt/caddy"
 	"testing"
+
+	"github.com/mholt/caddy"
 )
 
 func TestSetup(t *testing.T) {
@@ -111,4 +112,12 @@ func TestSetup(t *testing.T) {
 	testParsing([]string{"dial_timeout 1 2"}, false)
 	testParsing([]string{"dial_timeout seven"}, false)
 	testParsing([]string{"dial_timeout 2"}, true)
+
+	testParsing([]string{"upstream proxy.site"}, false)
+	testParsing([]string{"upstream https://proxy.site https://proxy.site"}, false)
+	testParsing([]string{"upstream http://localhost:1230"}, true)
+	testParsing([]string{"upstream socks5://127.0.0.1:999"}, true)
+	testParsing([]string{"upstream http://proxy.site"}, false)
+	testParsing([]string{"upstream https://proxy.site https://proxy.site"}, false)
+	testParsing([]string{"upstream https://proxy.site"}, true)
 }

test/upstreamingproxy/index.html

@@ -0,0 +1 @@
+I am upstreaming ForwardProxy(don't tell anyone)