HTTP cache: force caching even if server forbids (#26107)

plugin/http.go

@@ -101,6 +101,23 @@ func NewHTTP(log *util.Logger, method, uri string, insecure bool, cache time.Dur
 		method: method,
 	}
 
+	// override the transport to accept self-signed certificates
+	if insecure {
+		p.Client.Transport = request.NewTripper(log, transport.Insecure())
+	}
+
+	if cache > 0 {
+		// remove no-cache response headers
+		p.Client.Transport = &transport.Modifier{
+			Modifier: func(resp *http.Response) error {
+				dropNoCache(resp, "Cache-Control")
+				dropNoCache(resp, "Pragma")
+				return nil
+			},
+			Base: p.Client.Transport,
+		}
+	}
+
 	// http cache
 	p.Client.Transport = &httpcache.Transport{
 		Cache:     mc,
@@ -122,14 +139,27 @@ func NewHTTP(log *util.Logger, method, uri string, insecure bool, cache time.Dur
 		}
 	}
 
-	// ignore the self signed certificate
-	if insecure {
-		p.Client.Transport = request.NewTripper(log, transport.Insecure())
-	}
-
 	return p
 }
 
+func dropNoCache(resp *http.Response, header string) {
+	if h := resp.Header.Get(header); h != "" {
+		var hh []string
+
+		for _, h := range strings.Split(h, ",") {
+			if s := strings.TrimSpace(h); strings.ToLower(s) != "no-cache" {
+				hh = append(hh, s)
+			}
+		}
+
+		if len(hh) == 0 {
+			resp.Header.Del(header)
+		} else {
+			resp.Header.Set(header, strings.Join(hh, ", "))
+		}
+	}
+}
+
 // WithBody adds request body
 func (p *HTTP) WithBody(body string) *HTTP {
 	if body != "" {

util/transport/decorator.go

@@ -1,7 +1,6 @@
 package transport
 
 import (
-	"errors"
 	"net/http"
 )
 
@@ -12,12 +11,15 @@ type Decorator struct {
 	Decorator func(*http.Request) error
 
 	// Base is the base RoundTripper used to make HTTP requests.
-	// If nil, http.DefaultTransport is used.
 	Base http.RoundTripper
 }
 
 // RoundTrip decorates the request using the Decorator.
 func (t *Decorator) RoundTrip(req *http.Request) (*http.Response, error) {
+	if t.Decorator == nil {
+		return t.Base.RoundTrip(req)
+	}
+
 	reqBodyClosed := false
 	if req.Body != nil {
 		defer func() {
@@ -27,10 +29,6 @@ func (t *Decorator) RoundTrip(req *http.Request) (*http.Response, error) {
 		}()
 	}
 
-	if t.Decorator == nil {
-		return nil, errors.New("transport: Transport's Decorator is nil")
-	}
-
 	req2 := cloneRequest(req) // per RoundTripper contract
 	if err := t.Decorator(req2); err != nil {
 		return nil, err
@@ -38,14 +36,7 @@ func (t *Decorator) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	// req.Body is assumed to be closed by the base RoundTripper.
 	reqBodyClosed = true
-	return t.base().RoundTrip(req2)
-}
-
-func (t *Decorator) base() http.RoundTripper {
-	if t.Base != nil {
-		return t.Base
-	}
-	return Default()
+	return t.Base.RoundTrip(req2)
 }
 
 // cloneRequest returns a clone of the provided *http.Request.

util/transport/modifier.go

@@ -0,0 +1,26 @@
+package transport
+
+import (
+	"net/http"
+)
+
+// Modifier is an http.RoundTripper that makes HTTP responses,
+// wrapping a base RoundTripper and modifying given responses.
+type Modifier struct {
+	// Modifier modifies the incoming response
+	Modifier func(*http.Response) error
+
+	// Base is the base RoundTripper used to make HTTP responses.
+	Base http.RoundTripper
+}
+
+// RoundTrip modifies the response using the Modifier.
+func (t *Modifier) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.Base.RoundTrip(req)
+	if err != nil || t.Modifier == nil {
+		return resp, err
+	}
+
+	err = t.Modifier(resp)
+	return resp, err
+}