[fix] handle xml/html parsing errors

asciimoo · asciimoo · commit d8c4e07fe2cc · 2018-05-08T21:13:47.000+02:00
fixes #135
diff --git a/colly.go b/colly.go
@@ -587,13 +587,19 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
 
 	c.handleOnResponse(response)
 
-	c.handleOnHTML(response)
+	err = c.handleOnHTML(response)
+	if err != nil {
+		c.handleOnError(response, err, request, ctx)
+	}
 
-	c.handleOnXML(response)
+	err = c.handleOnXML(response)
+	if err != nil {
+		c.handleOnError(response, err, request, ctx)
+	}
 
 	c.handleOnScraped(response)
 
-	return nil
+	return err
 }
 
 func (c *Collector) requestCheck(u, method string, depth int, checkRevisit bool) error {
@@ -912,13 +918,13 @@ func (c *Collector) handleOnResponse(r *Response) {
 	}
 }
 
-func (c *Collector) handleOnHTML(resp *Response) {
+func (c *Collector) handleOnHTML(resp *Response) error {
 	if len(c.htmlCallbacks) == 0 || !strings.Contains(strings.ToLower(resp.Headers.Get("Content-Type")), "html") {
-		return
+		return nil
 	}
 	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(resp.Body))
 	if err != nil {
-		return
+		return err
 	}
 	if href, found := doc.Find("base[href]").Attr("href"); found {
 		resp.Request.baseURL, _ = url.Parse(href)
@@ -937,21 +943,22 @@ func (c *Collector) handleOnHTML(resp *Response) {
 			}
 		})
 	}
+	return nil
 }
 
-func (c *Collector) handleOnXML(resp *Response) {
+func (c *Collector) handleOnXML(resp *Response) error {
 	if len(c.xmlCallbacks) == 0 {
-		return
+		return nil
 	}
 	contentType := strings.ToLower(resp.Headers.Get("Content-Type"))
 	if !strings.Contains(contentType, "html") && !strings.Contains(contentType, "xml") {
-		return
+		return nil
 	}
 
 	if strings.Contains(contentType, "html") {
 		doc, err := htmlquery.Parse(bytes.NewBuffer(resp.Body))
 		if err != nil {
-			return
+			return err
 		}
 		if e := htmlquery.FindOne(doc, "//base/@href"); e != nil {
 			for _, a := range e.Attr {
@@ -977,7 +984,7 @@ func (c *Collector) handleOnXML(resp *Response) {
 	} else if strings.Contains(contentType, "xml") {
 		doc, err := xmlquery.Parse(bytes.NewBuffer(resp.Body))
 		if err != nil {
-			return
+			return err
 		}
 
 		for _, cc := range c.xmlCallbacks {
@@ -993,13 +1000,14 @@ func (c *Collector) handleOnXML(resp *Response) {
 			})
 		}
 	}
+	return nil
 }
 
 func (c *Collector) handleOnError(response *Response, err error, request *Request, ctx *Context) error {
 	if err == nil && (c.ParseHTTPErrorResponse || response.StatusCode < 203) {
 		return nil
 	}
-	if err == nil {
+	if err == nil && response.StatusCode >= 203 {
 		err = errors.New(http.StatusText(response.StatusCode))
 	}
 	if response == nil {

Original file line number	Diff line number	Diff line change
`@@ -587,13 +587,19 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct`
`587`	`587`
`588`	`588`	`c.handleOnResponse(response)`
`589`	`589`
`590`		`- c.handleOnHTML(response)`
	`590`	`+ err = c.handleOnHTML(response)`
	`591`	`+ if err != nil {`
	`592`	`+ c.handleOnError(response, err, request, ctx)`
	`593`	`+ }`
`591`	`594`
`592`		`- c.handleOnXML(response)`
	`595`	`+ err = c.handleOnXML(response)`
	`596`	`+ if err != nil {`
	`597`	`+ c.handleOnError(response, err, request, ctx)`
	`598`	`+ }`
`593`	`599`
`594`	`600`	`c.handleOnScraped(response)`
`595`	`601`
`596`		`- return nil`
	`602`	`+ return err`
`597`	`603`	`}`
`598`	`604`
`599`	`605`	`func (c *Collector) requestCheck(u, method string, depth int, checkRevisit bool) error {`
`@@ -912,13 +918,13 @@ func (c Collector) handleOnResponse(r Response) {`
`912`	`918`	`}`
`913`	`919`	`}`
`914`	`920`
`915`		`-func (c Collector) handleOnHTML(resp Response) {`
	`921`	`+func (c Collector) handleOnHTML(resp Response) error {`
`916`	`922`	`if len(c.htmlCallbacks) == 0 \|\| !strings.Contains(strings.ToLower(resp.Headers.Get("Content-Type")), "html") {`
`917`		`- return`
	`923`	`+ return nil`
`918`	`924`	`}`
`919`	`925`	`doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(resp.Body))`
`920`	`926`	`if err != nil {`
`921`		`- return`
	`927`	`+ return err`
`922`	`928`	`}`
`923`	`929`	`if href, found := doc.Find("base[href]").Attr("href"); found {`
`924`	`930`	`resp.Request.baseURL, _ = url.Parse(href)`
`@@ -937,21 +943,22 @@ func (c Collector) handleOnHTML(resp Response) {`
`937`	`943`	`}`
`938`	`944`	`})`
`939`	`945`	`}`
	`946`	`+ return nil`
`940`	`947`	`}`
`941`	`948`
`942`		`-func (c Collector) handleOnXML(resp Response) {`
	`949`	`+func (c Collector) handleOnXML(resp Response) error {`
`943`	`950`	`if len(c.xmlCallbacks) == 0 {`
`944`		`- return`
	`951`	`+ return nil`
`945`	`952`	`}`
`946`	`953`	`contentType := strings.ToLower(resp.Headers.Get("Content-Type"))`
`947`	`954`	`if !strings.Contains(contentType, "html") && !strings.Contains(contentType, "xml") {`
`948`		`- return`
	`955`	`+ return nil`
`949`	`956`	`}`
`950`	`957`
`951`	`958`	`if strings.Contains(contentType, "html") {`
`952`	`959`	`doc, err := htmlquery.Parse(bytes.NewBuffer(resp.Body))`
`953`	`960`	`if err != nil {`
`954`		`- return`
	`961`	`+ return err`
`955`	`962`	`}`
`956`	`963`	`if e := htmlquery.FindOne(doc, "//base/@href"); e != nil {`
`957`	`964`	`for _, a := range e.Attr {`
`@@ -977,7 +984,7 @@ func (c Collector) handleOnXML(resp Response) {`
`977`	`984`	`} else if strings.Contains(contentType, "xml") {`
`978`	`985`	`doc, err := xmlquery.Parse(bytes.NewBuffer(resp.Body))`
`979`	`986`	`if err != nil {`
`980`		`- return`
	`987`	`+ return err`
`981`	`988`	`}`
`982`	`989`
`983`	`990`	`for _, cc := range c.xmlCallbacks {`
`@@ -993,13 +1000,14 @@ func (c Collector) handleOnXML(resp Response) {`
`993`	`1000`	`})`
`994`	`1001`	`}`
`995`	`1002`	`}`
	`1003`	`+ return nil`
`996`	`1004`	`}`
`997`	`1005`
`998`	`1006`	`func (c Collector) handleOnError(response Response, err error, request Request, ctx Context) error {`
`999`	`1007`	`if err == nil && (c.ParseHTTPErrorResponse \|\| response.StatusCode < 203) {`
`1000`	`1008`	`return nil`
`1001`	`1009`	`}`
`1002`		`- if err == nil {`
	`1010`	`+ if err == nil && response.StatusCode >= 203 {`
`1003`	`1011`	`err = errors.New(http.StatusText(response.StatusCode))`
`1004`	`1012`	`}`
`1005`	`1013`	`if response == nil {`