plugin/forward: Add http(2) host/authority header and TO server resolution (#8233)

plugin/forward/resolve.go

@@ -16,8 +16,8 @@ import (
 // hostEntry represents a hostname-based TO address that needs DNS resolution.
 type hostEntry struct {
 	hostname  string // the hostname to resolve (e.g., "rbldnsd.rbldnsd.svc.cluster.local")
-	port      string // port (e.g., "53", "853")
+	port      string // port (e.g., "53", "443", "853")
-	transport string // "dns" or "tls"
+	transport string // "dns", "tls", or "https"
 	zone      string // TLS server name zone (from %zone syntax)
 }
 
@@ -67,15 +67,18 @@ func parseAsHostEntry(h string) (hostEntry, bool) {
 	cleanH, zone := splitZone(h)
 	trans, host := parse.Transport(cleanH)
 
-	// Only dns and tls transports are supported for hostname resolution
+	// Only dns, tls, and https transports are supported for hostname resolution
-	if trans != transport.DNS && trans != transport.TLS {
+	if trans != transport.DNS && trans != transport.TLS && trans != transport.HTTPS {
 		return hostEntry{}, false
 	}
 
 	hostname := host
 	port := transport.Port
-	if trans == transport.TLS {
+	switch trans {
+	case transport.TLS:
 		port = transport.TLSPort
+	case transport.HTTPS:
+		port = transport.HTTPSPort
 	}
 
 	// Check if there's a port
@@ -161,14 +164,14 @@ func formatResolvedAddr(ip, port, trans, zone string) string {
 	isIPv6 := strings.Contains(ip, ":")
 
 	switch trans {
-	case transport.TLS:
+	case transport.TLS, transport.HTTPS:
 		if zone != "" {
 			if isIPv6 {
-				return transport.TLS + "://[" + ip + "%" + zone + "]:" + port
+				return trans + "://[" + ip + "%" + zone + "]:" + port
 			}
-			return transport.TLS + "://" + ip + "%" + zone + ":" + port
+			return trans + "://" + ip + "%" + zone + ":" + port
 		}
-		return transport.TLS + "://" + net.JoinHostPort(ip, port)
+		return trans + "://" + net.JoinHostPort(ip, port)
 	default: // transport.DNS
 		return net.JoinHostPort(ip, port)
 	}

plugin/forward/resolve_test.go

@@ -79,6 +79,11 @@ func TestClassifyToAddrs(t *testing.T) {
 			input:       []string{"tls://dns.example.com"},
 			wantDynamic: 1,
 		},
+		{
+			name:        "HTTPS hostname",
+			input:       []string{"https://dns.example.com"},
+			wantDynamic: 1,
+		},
 		{
 			name:        "k8s service name",
 			input:       []string{"rbldnsd.rbldnsd.svc.cluster.local"},
@@ -166,12 +171,15 @@ func TestParseAsHostEntry(t *testing.T) {
 		{"tls://dns.example.com", true, "dns.example.com", "853", transport.TLS, ""},
 		{"tls://dns.example.com:8853", true, "dns.example.com", "8853", transport.TLS, ""},
 		{"tls://dns.example.com%servername.example.com", true, "dns.example.com", "853", transport.TLS, "servername.example.com"},
+		{"https://dns.example.com", true, "dns.example.com", "443", transport.HTTPS, ""},
+		{"https://dns.example.com:8443", true, "dns.example.com", "8443", transport.HTTPS, ""},
+		{"https://dns.example.com%servername.example.com", true, "dns.example.com", "443", transport.HTTPS, "servername.example.com"},
 		{"rbldnsd.rbldnsd.svc.cluster.local", true, "rbldnsd.rbldnsd.svc.cluster.local", "53", transport.DNS, ""},
 		// Should fail for IPs
 		{"127.0.0.1", false, "", "", "", ""},
 		{"::1", false, "", "", "", ""},
 		// Should fail for unsupported transports
-		{"https://example.com", false, "", "", "", ""},
+		{"grpc://example.com", false, "", "", "", ""},
 		// Should fail for empty
 		{"", false, "", "", "", ""},
 	}
@@ -209,9 +217,13 @@ func TestFormatResolvedAddr(t *testing.T) {
 		{"10.0.0.1", "53", transport.DNS, "", "10.0.0.1:53"},
 		{"10.0.0.1", "853", transport.TLS, "", "tls://10.0.0.1:853"},
 		{"10.0.0.1", "853", transport.TLS, "example.com", "tls://10.0.0.1%example.com:853"},
+		{"10.0.0.1", "443", transport.HTTPS, "", "https://10.0.0.1:443"},
+		{"10.0.0.1", "443", transport.HTTPS, "example.com", "https://10.0.0.1%example.com:443"},
 		{"::1", "53", transport.DNS, "", "[::1]:53"},
 		{"::1", "853", transport.TLS, "", "tls://[::1]:853"},
 		{"::1", "853", transport.TLS, "example.com", "tls://[::1%example.com]:853"},
+		{"::1", "443", transport.HTTPS, "", "https://[::1]:443"},
+		{"::1", "443", transport.HTTPS, "example.com", "https://[::1%example.com]:443"},
 	}
 
 	for _, tc := range tests {
@@ -576,6 +588,10 @@ func TestExpandAndDedupTLS(t *testing.T) {
 		{static: false, entry: hostEntry{hostname: "dns2.example.com", port: "853", transport: "tls"}},
 		{static: true, addrs: []string{"tls://149.112.112.112:853"}},
 		{static: true, addrs: []string{"tls://9.9.9.10:853"}},
+		{static: false, entry: hostEntry{hostname: "dns1.example.com", port: "443", transport: "https"}},
+		{static: false, entry: hostEntry{hostname: "dns2.example.com", port: "443", transport: "https"}},
+		{static: true, addrs: []string{"https://149.112.112.112:443"}},
+		{static: true, addrs: []string{"https://9.9.9.10:443"}},
 	}
 
 	result, err := expandAndDedup(entries, []string{s.Addr})
@@ -583,7 +599,7 @@ func TestExpandAndDedupTLS(t *testing.T) {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
-	expected := []string{"9.9.9.9:853", "149.112.112.112:853", "9.9.9.10:853"}
+	expected := []string{"9.9.9.9:853", "149.112.112.112:853", "9.9.9.10:853", "9.9.9.9:443", "149.112.112.112:443", "9.9.9.10:443"}
 	if len(result) != len(expected) {
 		t.Fatalf("expected %d addresses after dedup, got %d: %v", len(expected), len(result), result)
 	}

plugin/forward/setup.go

@@ -224,20 +224,8 @@ func parseStanza(c *caddy.Controller) (*Forward, error) {
 	f.tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(len(f.proxies))
 
 	for i := range f.proxies {
-		// Only set this for proxies that need it.
-		if transports[i] == transport.TLS {
-			if tlsConfig, ok := perServerNameTlsConfig[tlsServerNames[i]]; ok {
-				f.proxies[i].SetTLSConfig(tlsConfig)
-			} else {
-				f.proxies[i].SetTLSConfig(f.tlsConfig)
-			}
-		}
-
 		if transports[i] == transport.HTTPS {
 			httpTransport := http.DefaultTransport.(*http.Transport).Clone()
-			httpTransport.TLSClientConfig = f.tlsConfig
-			httpTransport.MaxIdleConns = f.maxIdleConns
-			httpTransport.MaxIdleConnsPerHost = f.maxIdleConns
 
 			c := http.Client{
 				Transport: httpTransport,
@@ -248,6 +236,15 @@ func parseStanza(c *caddy.Controller) (*Forward, error) {
 			f.proxies[i].SetDOHRequestOptions(f.dohMethod)
 		}
 
+		// Only set this for proxies that need it.
+		if transports[i] == transport.TLS {
+			if tlsConfig, ok := perServerNameTlsConfig[tlsServerNames[i]]; ok {
+				f.proxies[i].SetTLSConfig(tlsConfig)
+			} else {
+				f.proxies[i].SetTLSConfig(f.tlsConfig)
+			}
+		}
+
 		f.proxies[i].SetExpire(f.expire)
 		f.proxies[i].SetMaxAge(f.maxAge)
 		f.proxies[i].SetMaxIdleConns(f.maxIdleConns)

plugin/forward/setup_test.go

@@ -143,14 +143,22 @@ func TestSplitZone(t *testing.T) {
 			"tls://127.0.0.1%example.net:854", "tls://127.0.0.1:854", "example.net",
 		}, {
 			"tls://127.0.0.1%example.net", "tls://127.0.0.1", "example.net",
+		}, {
+			"https://127.0.0.1%example.net:443", "https://127.0.0.1:443", "example.net",
+		}, {
+			"https://127.0.0.1%example.net", "https://127.0.0.1", "example.net",
 		}, {
 			"tls://127.0.0.1:854", "tls://127.0.0.1:854", "",
+		}, {
+			"https://127.0.0.1:443", "https://127.0.0.1:443", "",
 		}, {
 			"dns://127.0.0.1", "dns://127.0.0.1", "",
 		}, {
 			"foo%bar:baz", "foo:baz", "bar",
 		}, {
 			"tls://[::1%example.net]:853", "tls://[::1]:853", "example.net",
+		}, {
+			"https://[::1%example.net]:443", "https://[::1]:443", "example.net",
 		},
 	}
 	for i, test := range tests {

plugin/pkg/doh/doh.go

@@ -25,11 +25,11 @@ const Path = "/dns-query"
 // The URL should not have a path, so please exclude /dns-query. The URL will
 // be prefixed with https:// by default, unless it's already prefixed with
 // either http:// or https://.
-func NewRequest(method, url string, m *dns.Msg) (*http.Request, error) {
+func NewRequest(method, url, host string, m *dns.Msg) (*http.Request, error) {
-	return NewRequestWithContext(context.Background(), method, url, m)
+	return NewRequestWithContext(context.Background(), method, url, host, m)
 }
 
-func NewRequestWithContext(ctx context.Context, method, url string, m *dns.Msg) (*http.Request, error) {
+func NewRequestWithContext(ctx context.Context, method, url, host string, m *dns.Msg) (*http.Request, error) {
 	buf, err := m.Pack()
 	if err != nil {
 		return nil, err
@@ -55,6 +55,7 @@ func NewRequestWithContext(ctx context.Context, method, url string, m *dns.Msg)
 
 		req.Header.Set("Content-Type", MimeType)
 		req.Header.Set("Accept", MimeType)
+		req.Host = host
 		return req, nil
 
 	case http.MethodPost:
@@ -70,6 +71,7 @@ func NewRequestWithContext(ctx context.Context, method, url string, m *dns.Msg)
 
 		req.Header.Set("Content-Type", MimeType)
 		req.Header.Set("Accept", MimeType)
+		req.Host = host
 		return req, nil
 
 	default:

plugin/pkg/doh/doh_test.go

@@ -25,7 +25,7 @@ func TestDoH(t *testing.T) {
 			m := new(dns.Msg)
 			m.SetQuestion("example.org.", dns.TypeDNSKEY)
 
-			req, err := NewRequest(test.method, test.url, m)
+			req, err := NewRequest(test.method, test.url, "example.org", m)
 			if err != nil {
 				t.Errorf("Failure to make request: %s", err)
 			}

plugin/pkg/proxy/connect.go

@@ -228,6 +228,13 @@ func (p *Proxy) lookupDoH(ctx context.Context, state request.Request, _ Options)
 	// DoH always runs over TCP (HTTPS), regardless of the downstream
 	// client's protocol.
 	const proto = "tcp"
+	// records the origin Id before upstream.
+	originId := state.Req.Id
+	// RFC8484 has DNS ID of 0 as a SHOULD
+	state.Req.Id = 0
+	defer func() {
+		state.Req.Id = originId
+	}()
 
 	var localAddr net.Addr
 	trace := &httptrace.ClientTrace{
@@ -237,7 +244,7 @@ func (p *Proxy) lookupDoH(ctx context.Context, state request.Request, _ Options)
 	}
 	ctx = httptrace.WithClientTrace(ctx, trace)
 
-	req, err := doh.NewRequestWithContext(ctx, p.dohMethod, p.addr, state.Req)
+	req, err := doh.NewRequestWithContext(ctx, p.dohMethod, p.addr, p.dohHost, state.Req)
 	if err != nil {
 		return nil, nil, proto, err
 	}
@@ -253,6 +260,10 @@ func (p *Proxy) lookupDoH(ctx context.Context, state request.Request, _ Options)
 		return nil, localAddr, proto, err
 	}
 
+	// recovery the origin Id after upstream.
+	if ret != nil {
+		ret.Id = originId
+	}
 	return ret, localAddr, proto, nil
 }
 

plugin/pkg/proxy/health.go

@@ -205,7 +205,7 @@ type dohHc struct {
 }
 
 func (h *dohHc) Check(p *Proxy) error {
-	err := h.send(p.addr)
+	err := h.send(p.addr, p.dohHost)
 	if err != nil {
 		healthcheckFailureCount.WithLabelValues(p.proxyName, p.addr).Add(1)
 		p.incrementFails()
@@ -216,7 +216,7 @@ func (h *dohHc) Check(p *Proxy) error {
 	return nil
 }
 
-func (h *dohHc) send(addr string) error {
+func (h *dohHc) send(addr, host string) error {
 	ping := new(dns.Msg)
 	ping.SetQuestion(h.domain, dns.TypeNS)
 	ping.RecursionDesired = h.recursionDesired
@@ -224,7 +224,7 @@ func (h *dohHc) send(addr string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), h.client.Timeout)
 	defer cancel()
 
-	req, err := doh.NewRequestWithContext(ctx, http.MethodPost, addr, ping)
+	req, err := doh.NewRequestWithContext(ctx, http.MethodPost, addr, host, ping)
 	if err != nil {
 		return err
 	}

plugin/pkg/proxy/proxy.go

@@ -22,6 +22,7 @@ type Proxy struct {
 	protocol  string
 
 	dohMethod string
+	dohHost   string
 
 	readTimeout time.Duration
 
@@ -40,6 +41,7 @@ func NewProxy(proxyName, addr, protocol string) *Proxy {
 		transport:   newTransport(proxyName, addr),
 		protocol:    protocol,
 		dohMethod:   http.MethodPost,
+		dohHost:     "",
 		health:      NewHealthChecker(proxyName, protocol, true, "."),
 		proxyName:   proxyName,
 	}
@@ -68,7 +70,13 @@ func (p *Proxy) SetMaxAge(maxAge time.Duration) { p.transport.SetMaxAge(maxAge)
 
 // SetMaxIdleConns sets the maximum idle connections per transport type.
 // A value of 0 means unlimited (default).
-func (p *Proxy) SetMaxIdleConns(n int) { p.transport.SetMaxIdleConns(n) }
+func (p *Proxy) SetMaxIdleConns(n int) {
+	p.transport.SetMaxIdleConns(n)
+	if p.transport.httpClient != nil {
+		p.transport.httpClient.Transport.(*http.Transport).MaxIdleConns = n
+		p.transport.httpClient.Transport.(*http.Transport).MaxIdleConnsPerHost = n
+	}
+}
 
 func (p *Proxy) SetHTTPClient(client *http.Client) {
 	p.transport.httpClient = client
@@ -78,6 +86,12 @@ func (p *Proxy) SetDOHRequestOptions(method string) {
 	p.dohMethod = method
 }
 
+func (p *Proxy) SetDOHHost(host string) {
+	p.dohHost = host
+}
+
+func (p *Proxy) DoHHost() string { return p.dohHost }
+
 func (p *Proxy) GetHealthchecker() HealthChecker {
 	return p.health
 }