From 235fa5d9ba51003bd4c40a69bb9e7c4d82a5ef39 Mon Sep 17 00:00:00 2001 From: polyfloyd Date: Tue, 11 Sep 2018 23:14:25 +0200 Subject: [PATCH 01/14] Add support for Brotli compression (#326) * Add support for Brotli compression * Allow compression algorithms to be added at runtime --- middleware/compress.go | 210 ++++++++++++++++++++++++++--------------- 1 file changed, 135 insertions(+), 75 deletions(-) diff --git a/middleware/compress.go b/middleware/compress.go index 006ad48f..67461913 100644 --- a/middleware/compress.go +++ b/middleware/compress.go @@ -8,27 +8,90 @@ import ( "io" "net" "net/http" + "regexp" + "sort" "strings" ) -type encoding int +var encoders = map[string]EncoderFunc{} -const ( - encodingNone encoding = iota - encodingGzip - encodingDeflate -) +var acceptEncodingAlgorithmsRe = regexp.MustCompile(`([a-z]{2,}|\*)`) + +func init() { + // TODO: + // lzma: Opera. + // sdch: Chrome, Android. Gzip output + dictionary header. + // br: Brotli. + + // TODO: Exception for old MSIE browsers that can't handle non-HTML? + // https://zoompf.com/blog/2012/02/lose-the-wait-http-compression + SetEncoder("gzip", encoderGzip) + + // HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951) + // wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32 + // checksum compared to CRC-32 used in "gzip" and thus is faster. + // + // But.. some old browsers (MSIE, Safari 5.1) incorrectly expect + // raw DEFLATE data only, without the mentioned zlib wrapper. + // Because of this major confusion, most modern browsers try it + // both ways, first looking for zlib headers. + // Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548 + // + // The list of browsers having problems is quite big, see: + // http://zoompf.com/blog/2012/02/lose-the-wait-http-compression + // https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results + // + // That's why we prefer gzip over deflate. It's just more reliable + // and not significantly slower than gzip. + SetEncoder("deflate", encoderDeflate) + + // NOTE: Not implemented, intentionally: + // case "compress": // LZW. Deprecated. + // case "bzip2": // Too slow on-the-fly. + // case "zopfli": // Too slow on-the-fly. + // case "xz": // Too slow on-the-fly. +} + +// An EncoderFunc is a function that wraps the provided ResponseWriter with a +// streaming compression algorithm and returns it. +// +// In case of failure, the function should return nil. +type EncoderFunc func(w http.ResponseWriter, level int) io.Writer + +// SetEncoder can be used to set the implementation of a compression algorithm. +// +// The encoding should be a standardised identifier. See: +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding +// +// For example, add the Brotli algortithm: +// +// import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc" +// +// middleware.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer { +// params := brotli_enc.NewBrotliParams() +// params.SetQuality(level) +// return brotli_enc.NewBrotliWriter(params, w) +// }) +func SetEncoder(encoding string, fn EncoderFunc) { + if encoding == "" { + panic("the encoding can not be empty") + } + if fn == nil { + panic("attempted to set a nil encoder function") + } + encoders[encoding] = fn +} var defaultContentTypes = map[string]struct{}{ - "text/html": struct{}{}, - "text/css": struct{}{}, - "text/plain": struct{}{}, - "text/javascript": struct{}{}, - "application/javascript": struct{}{}, - "application/x-javascript": struct{}{}, - "application/json": struct{}{}, - "application/atom+xml": struct{}{}, - "application/rss+xml": struct{}{}, + "text/html": {}, + "text/css": {}, + "text/plain": {}, + "text/javascript": {}, + "application/javascript": {}, + "application/x-javascript": {}, + "application/json": {}, + "application/atom+xml": {}, + "application/rss+xml": {}, } // DefaultCompress is a middleware that compresses response @@ -54,11 +117,13 @@ func Compress(level int, types ...string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { + encoder, encoding := selectEncoder(r.Header) mcw := &maybeCompressResponseWriter{ ResponseWriter: w, w: w, contentTypes: contentTypes, - encoding: selectEncoding(r.Header), + encoder: encoder, + encoding: encoding, level: level, } defer mcw.Close() @@ -70,53 +135,46 @@ func Compress(level int, types ...string) func(next http.Handler) http.Handler { } } -func selectEncoding(h http.Header) encoding { - enc := h.Get("Accept-Encoding") +func selectEncoder(h http.Header) (EncoderFunc, string) { + header := h.Get("Accept-Encoding") - switch { - // TODO: - // case "br": // Brotli, experimental. Firefox 2016, to-be-in Chromium. - // case "lzma": // Opera. - // case "sdch": // Chrome, Android. Gzip output + dictionary header. - - case strings.Contains(enc, "gzip"): - // TODO: Exception for old MSIE browsers that can't handle non-HTML? - // https://zoompf.com/blog/2012/02/lose-the-wait-http-compression - return encodingGzip - - case strings.Contains(enc, "deflate"): - // HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951) - // wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32 - // checksum compared to CRC-32 used in "gzip" and thus is faster. - // - // But.. some old browsers (MSIE, Safari 5.1) incorrectly expect - // raw DEFLATE data only, without the mentioned zlib wrapper. - // Because of this major confusion, most modern browsers try it - // both ways, first looking for zlib headers. - // Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548 - // - // The list of browsers having problems is quite big, see: - // http://zoompf.com/blog/2012/02/lose-the-wait-http-compression - // https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results - // - // That's why we prefer gzip over deflate. It's just more reliable - // and not significantly slower than gzip. - return encodingDeflate - - // NOTE: Not implemented, intentionally: - // case "compress": // LZW. Deprecated. - // case "bzip2": // Too slow on-the-fly. - // case "zopfli": // Too slow on-the-fly. - // case "xz": // Too slow on-the-fly. - } - - return encodingNone + // Parse the names of all accepted algorithms from the header. + var accepted []string + for _, m := range acceptEncodingAlgorithmsRe.FindAllStringSubmatch(header, -1) { + accepted = append(accepted, m[1]) + } + + sort.Sort(byPerformance(accepted)) + + // Select the first mutually supported algorithm. + for _, name := range accepted { + if fn, ok := encoders[name]; ok { + return fn, name + } + } + return nil, "" +} + +type byPerformance []string + +func (l byPerformance) Len() int { return len(l) } +func (l byPerformance) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l byPerformance) Less(i, j int) bool { + // Higher number = higher preference. This causes unknown names, which map + // to 0, to always be less prefered. + scores := map[string]int{ + "br": 3, + "gzip": 2, + "deflate": 1, + } + return scores[l[i]] > scores[l[j]] } type maybeCompressResponseWriter struct { http.ResponseWriter w io.Writer - encoding encoding + encoder EncoderFunc + encoding string contentTypes map[string]struct{} level int wroteHeader bool @@ -148,25 +206,11 @@ func (w *maybeCompressResponseWriter) WriteHeader(code int) { return } - // Select the compress writer. - switch w.encoding { - case encodingGzip: - gw, err := gzip.NewWriterLevel(w.ResponseWriter, w.level) - if err != nil { - w.w = w.ResponseWriter - return + if w.encoder != nil && w.encoding != "" { + if wr := w.encoder(w.ResponseWriter, w.level); wr != nil { + w.w = wr + w.Header().Set("Content-Encoding", w.encoding) } - w.w = gw - w.ResponseWriter.Header().Set("Content-Encoding", "gzip") - - case encodingDeflate: - dw, err := flate.NewWriter(w.ResponseWriter, w.level) - if err != nil { - w.w = w.ResponseWriter - return - } - w.w = dw - w.ResponseWriter.Header().Set("Content-Encoding", "deflate") } } @@ -210,3 +254,19 @@ func (w *maybeCompressResponseWriter) Close() error { } return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer") } + +func encoderGzip(w http.ResponseWriter, level int) io.Writer { + gw, err := gzip.NewWriterLevel(w, level) + if err != nil { + return nil + } + return gw +} + +func encoderDeflate(w http.ResponseWriter, level int) io.Writer { + dw, err := flate.NewWriter(w, level) + if err != nil { + return nil + } + return dw +} From 16fa82175624aaee9df83055a307c1e24b251080 Mon Sep 17 00:00:00 2001 From: pdedmon Date: Sun, 14 Oct 2018 10:40:44 -0700 Subject: [PATCH 02/14] s/Goji/chi --- middleware/realip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/realip.go b/middleware/realip.go index e9addbe3..146c2b0a 100644 --- a/middleware/realip.go +++ b/middleware/realip.go @@ -22,7 +22,7 @@ var xRealIP = http.CanonicalHeaderKey("X-Real-IP") // You should only use this middleware if you can trust the headers passed to // you (in particular, the two headers this middleware uses), for example // because you have placed a reverse proxy like HAProxy or nginx in front of -// Goji. If your reverse proxies are configured to pass along arbitrary header +// chi. If your reverse proxies are configured to pass along arbitrary header // values from the client, or if you use this middleware without a reverse // proxy, malicious clients will be able to make you very sad (or, depending on // how you're using RemoteAddr, vulnerable to an attack of some sort). From a8101e5e7527df085501350abb310d77778f7e48 Mon Sep 17 00:00:00 2001 From: pdedmon Date: Sun, 14 Oct 2018 10:51:12 -0700 Subject: [PATCH 03/14] fix golint installation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94199280..fe33fc50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ go: install: - go get -u golang.org/x/tools/cmd/goimports - - go get -u github.com/golang/lint/golint + - go get -u golang.org/x/lint/golint script: - go get -d -t ./... From daa22f628c0f8312382deffce54c2a8d784de6dc Mon Sep 17 00:00:00 2001 From: Rodney <32748492+rodney-b@users.noreply.github.com> Date: Tue, 23 Oct 2018 11:52:23 -0400 Subject: [PATCH 04/14] Exported HTTP method names to avoid manual entry (#353) * Using package http's already defined Method constants instead of redefining them --- tree.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tree.go b/tree.go index a55d7f14..d7f1dd29 100644 --- a/tree.go +++ b/tree.go @@ -33,15 +33,15 @@ var mALL = mCONNECT | mDELETE | mGET | mHEAD | mOPTIONS | mPATCH | mPOST | mPUT | mTRACE var methodMap = map[string]methodTyp{ - "CONNECT": mCONNECT, - "DELETE": mDELETE, - "GET": mGET, - "HEAD": mHEAD, - "OPTIONS": mOPTIONS, - "PATCH": mPATCH, - "POST": mPOST, - "PUT": mPUT, - "TRACE": mTRACE, + http.MethodConnect: mCONNECT, + http.MethodDelete: mDELETE, + http.MethodGet: mGET, + http.MethodHead: mHEAD, + http.MethodOptions: mOPTIONS, + http.MethodPatch: mPATCH, + http.MethodPost: mPOST, + http.MethodPut: mPUT, + http.MethodTrace: mTRACE, } // RegisterMethod adds support for custom HTTP method handlers, available From be3aea55585f367281b58ae0b801bd81124b31a3 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Wed, 24 Oct 2018 06:06:09 -0400 Subject: [PATCH 05/14] revert methodMap change - brings back support for go 1.7 and 1.8 --- tree.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tree.go b/tree.go index d7f1dd29..a55d7f14 100644 --- a/tree.go +++ b/tree.go @@ -33,15 +33,15 @@ var mALL = mCONNECT | mDELETE | mGET | mHEAD | mOPTIONS | mPATCH | mPOST | mPUT | mTRACE var methodMap = map[string]methodTyp{ - http.MethodConnect: mCONNECT, - http.MethodDelete: mDELETE, - http.MethodGet: mGET, - http.MethodHead: mHEAD, - http.MethodOptions: mOPTIONS, - http.MethodPatch: mPATCH, - http.MethodPost: mPOST, - http.MethodPut: mPUT, - http.MethodTrace: mTRACE, + "CONNECT": mCONNECT, + "DELETE": mDELETE, + "GET": mGET, + "HEAD": mHEAD, + "OPTIONS": mOPTIONS, + "PATCH": mPATCH, + "POST": mPOST, + "PUT": mPUT, + "TRACE": mTRACE, } // RegisterMethod adds support for custom HTTP method handlers, available From 0ebf7795c516423a110473652e9ba3a59a504863 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Wed, 24 Oct 2018 06:12:33 -0400 Subject: [PATCH 06/14] fix travis for go 1.7 and go 1.8 --- .travis.yml | 2 -- tree.go | 18 +++++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe33fc50..ae95387a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,10 @@ go: install: - go get -u golang.org/x/tools/cmd/goimports - - go get -u golang.org/x/lint/golint script: - go get -d -t ./... - go vet ./... - - golint ./... - go test ./... - > go_version=$(go version); diff --git a/tree.go b/tree.go index a55d7f14..d7f1dd29 100644 --- a/tree.go +++ b/tree.go @@ -33,15 +33,15 @@ var mALL = mCONNECT | mDELETE | mGET | mHEAD | mOPTIONS | mPATCH | mPOST | mPUT | mTRACE var methodMap = map[string]methodTyp{ - "CONNECT": mCONNECT, - "DELETE": mDELETE, - "GET": mGET, - "HEAD": mHEAD, - "OPTIONS": mOPTIONS, - "PATCH": mPATCH, - "POST": mPOST, - "PUT": mPUT, - "TRACE": mTRACE, + http.MethodConnect: mCONNECT, + http.MethodDelete: mDELETE, + http.MethodGet: mGET, + http.MethodHead: mHEAD, + http.MethodOptions: mOPTIONS, + http.MethodPatch: mPATCH, + http.MethodPost: mPOST, + http.MethodPut: mPUT, + http.MethodTrace: mTRACE, } // RegisterMethod adds support for custom HTTP method handlers, available From 263880db92251ff501a733d7fdb8f5fb026ac548 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Nov 2018 11:38:48 -0500 Subject: [PATCH 07/14] middleware: add image/svg+xml content-type to compress --- middleware/compress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/middleware/compress.go b/middleware/compress.go index 67461913..81f9818a 100644 --- a/middleware/compress.go +++ b/middleware/compress.go @@ -92,6 +92,7 @@ var defaultContentTypes = map[string]struct{}{ "application/json": {}, "application/atom+xml": {}, "application/rss+xml": {}, + "image/svg+xml": {}, } // DefaultCompress is a middleware that compresses response From def7567e149e3f66f2a3d62c6114e8d1731720c4 Mon Sep 17 00:00:00 2001 From: Harry B Date: Mon, 10 Dec 2018 12:01:25 -0800 Subject: [PATCH 08/14] Middleware method RequestID() modified to support X-Request-ID from HTTP header (#367) --- middleware/request_id.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/middleware/request_id.go b/middleware/request_id.go index 4574bde8..4afb89bf 100644 --- a/middleware/request_id.go +++ b/middleware/request_id.go @@ -62,9 +62,13 @@ func init() { // counter. func RequestID(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - myid := atomic.AddUint64(&reqid, 1) ctx := r.Context() - ctx = context.WithValue(ctx, RequestIDKey, fmt.Sprintf("%s-%06d", prefix, myid)) + requestID := r.Header.Get("X-Request-Id") + if requestID == "" { + myid := atomic.AddUint64(&reqid, 1) + requestID = fmt.Sprintf("%s-%06d", prefix, myid) + } + ctx = context.WithValue(ctx, RequestIDKey, requestID) next.ServeHTTP(w, r.WithContext(ctx)) } return http.HandlerFunc(fn) From 0ae339de6d3e40d1aca7b22617a8267b7537f08c Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Fri, 21 Dec 2018 08:46:56 -0500 Subject: [PATCH 09/14] fix travis --- .travis.yml | 4 +--- _examples/versions/main.go | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae95387a..09e37caf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ go: - 1.10.x - 1.11.x -install: - - go get -u golang.org/x/tools/cmd/goimports - script: - go get -d -t ./... - go vet ./... @@ -17,6 +14,7 @@ script: - > go_version=$(go version); if [ ${go_version:13:4} = "1.11" ]; then + go get -u golang.org/x/tools/cmd/goimports; goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :; fi diff --git a/_examples/versions/main.go b/_examples/versions/main.go index 2f3cda44..308f2768 100644 --- a/_examples/versions/main.go +++ b/_examples/versions/main.go @@ -16,9 +16,9 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/chi/_examples/versions/data" - "github.com/go-chi/chi/_examples/versions/presenter/v1" - "github.com/go-chi/chi/_examples/versions/presenter/v2" - "github.com/go-chi/chi/_examples/versions/presenter/v3" + v1 "github.com/go-chi/chi/_examples/versions/presenter/v1" + v2 "github.com/go-chi/chi/_examples/versions/presenter/v2" + v3 "github.com/go-chi/chi/_examples/versions/presenter/v3" "github.com/go-chi/chi/middleware" "github.com/go-chi/render" ) From efb8f44e73c80bcd00026f0f4e37c76595b1757e Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Fri, 21 Dec 2018 08:51:44 -0500 Subject: [PATCH 10/14] stop tracking support for go 1.7 and 1.8 in master --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09e37caf..781b0dd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: go go: - - 1.7.x - - 1.8.x - 1.9.x - 1.10.x - 1.11.x From 6172f3d1540ed5dd37208c991f5fbae139e2a2e5 Mon Sep 17 00:00:00 2001 From: iriri <32588326+iriri@users.noreply.github.com> Date: Thu, 3 Jan 2019 08:05:20 -0800 Subject: [PATCH 11/14] Update timeout example (#377) --- middleware/timeout.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/middleware/timeout.go b/middleware/timeout.go index 5cabf1f9..8e373536 100644 --- a/middleware/timeout.go +++ b/middleware/timeout.go @@ -15,7 +15,8 @@ import ( // // ie. a route/handler may look like: // -// r.Get("/long", func(ctx context.Context, w http.ResponseWriter, r *http.Request) { +// r.Get("/long", func(w http.ResponseWriter, r *http.Request) { +// ctx := r.Context() // processTime := time.Duration(rand.Intn(4)+1) * time.Second // // select { From fad5e30f0938748ab151514ca5c18dde52a8c1d4 Mon Sep 17 00:00:00 2001 From: Prateek Malhotra Date: Thu, 13 Dec 2018 14:54:30 -0500 Subject: [PATCH 12/14] Add no-transform to NoCache middleware From https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#Other: No transformations or conversions should be made to the resource. The Content-Encoding, Content-Range, Content-Type headers must not be modified by a proxy. A non- transparent proxy might, for example, convert between image formats in order to save cache space or to reduce the amount of traffic on a slow link. The no-transform directive disallows this. --- middleware/nocache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/nocache.go b/middleware/nocache.go index e5819ddd..2412829e 100644 --- a/middleware/nocache.go +++ b/middleware/nocache.go @@ -14,7 +14,7 @@ var epoch = time.Unix(0, 0).Format(time.RFC1123) // Taken from https://github.com/mytrile/nocache var noCacheHeaders = map[string]string{ "Expires": epoch, - "Cache-Control": "no-cache, no-store, must-revalidate, private, max-age=0", + "Cache-Control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", "Pragma": "no-cache", "X-Accel-Expires": "0", } From 2a9a6ccb79b7f858d04619488efd71f1490e84a8 Mon Sep 17 00:00:00 2001 From: Ekin Koc Date: Mon, 31 Dec 2018 14:08:54 +0300 Subject: [PATCH 13/14] Do not remove content-length when not compressing The compress middleware removes `content-length` header before writing headers as the resulting length is not known at the time anymore. However it also does this if there is no suitable encoder found for the content type we have, forcing chunked encoding on every response. --- middleware/compress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/compress.go b/middleware/compress.go index 81f9818a..25a62286 100644 --- a/middleware/compress.go +++ b/middleware/compress.go @@ -192,8 +192,6 @@ func (w *maybeCompressResponseWriter) WriteHeader(code int) { if w.ResponseWriter.Header().Get("Content-Encoding") != "" { return } - // The content-length after compression is unknown - w.ResponseWriter.Header().Del("Content-Length") // Parse the first part of the Content-Type response header. contentType := "" @@ -211,6 +209,8 @@ func (w *maybeCompressResponseWriter) WriteHeader(code int) { if wr := w.encoder(w.ResponseWriter, w.level); wr != nil { w.w = wr w.Header().Set("Content-Encoding", w.encoding) + // The content-length after compression is unknown + w.Header().Del("Content-Length") } } } From 08d9051ef6546d57c5dca8eae13e6df362e2d568 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 7 Jan 2019 20:17:51 -0500 Subject: [PATCH 14/14] tiny fix: formatting of previous PR --- middleware/compress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/compress.go b/middleware/compress.go index 25a62286..d2d964ad 100644 --- a/middleware/compress.go +++ b/middleware/compress.go @@ -210,7 +210,7 @@ func (w *maybeCompressResponseWriter) WriteHeader(code int) { w.w = wr w.Header().Set("Content-Encoding", w.encoding) // The content-length after compression is unknown - w.Header().Del("Content-Length") + w.Header().Del("Content-Length") } } }