Merge pull request #171 from mumugoah/master

asciimoo · web-flow · commit 2e03a396ce30 · 2018-06-27T10:38:04.000+02:00
Read gzipped response and request marshal include headers
diff --git a/colly.go b/colly.go
@@ -459,7 +459,7 @@ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error) {
 		Body:      bytes.NewReader(req.Body),
 		Ctx:       ctx,
 		ID:        atomic.AddUint32(&c.requestCount, 1),
-		Headers:   &http.Header{},
+		Headers:   &req.Headers,
 		collector: c,
 	}, nil
 }
diff --git a/http_backend.go b/http_backend.go
@@ -28,6 +28,8 @@ import (
 	"sync"
 	"time"
 
+	"compress/gzip"
+
 	"github.com/gobwas/glob"
 )
 
@@ -187,6 +189,12 @@ func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error)
 	if bodySize > 0 {
 		bodyReader = io.LimitReader(bodyReader, int64(bodySize))
 	}
+	if res.Uncompressed && res.Header.Get("Content-Encoding") == "gzip" {
+		bodyReader, err = gzip.NewReader(bodyReader)
+		if err != nil {
+			return nil, err
+		}
+	}
 	body, err := ioutil.ReadAll(bodyReader)
 	defer res.Body.Close()
 	if err != nil {
diff --git a/request.go b/request.go
@@ -51,11 +51,12 @@ type Request struct {
 }
 
 type serializableRequest struct {
-	URL    string
-	Method string
-	Body   []byte
-	ID     uint32
-	Ctx    map[string]interface{}
+	URL     string
+	Method  string
+	Body    []byte
+	ID      uint32
+	Ctx     map[string]interface{}
+	Headers http.Header
 }
 
 // New creates a new request with the context of the original request
@@ -164,10 +165,11 @@ func (r *Request) Marshal() ([]byte, error) {
 		}
 	}
 	return json.Marshal(&serializableRequest{
-		URL:    r.URL.String(),
-		Method: r.Method,
-		Body:   body,
-		ID:     r.ID,
-		Ctx:    ctx,
+		URL:     r.URL.String(),
+		Method:  r.Method,
+		Body:    body,
+		ID:      r.ID,
+		Ctx:     ctx,
+		Headers: *r.Headers,
 	})
 }

Original file line number	Diff line number	Diff line change
`@@ -459,7 +459,7 @@ func (c Collector) UnmarshalRequest(r []byte) (Request, error) {`
`459`	`459`	`Body: bytes.NewReader(req.Body),`
`460`	`460`	`Ctx: ctx,`
`461`	`461`	`ID: atomic.AddUint32(&c.requestCount, 1),`
`462`		`- Headers: &http.Header{},`
	`462`	`+ Headers: &req.Headers,`
`463`	`463`	`collector: c,`
`464`	`464`	`}, nil`
`465`	`465`	`}`