Merge pull request #128 from OscarScholten/master

asciimoo · web-flow · commit 69f7bf5edf80 · 2018-04-10T17:18:53.000+02:00
Connection error when getting robots.txt leads to runtime error
diff --git a/colly.go b/colly.go
@@ -565,17 +565,16 @@ func (c *Collector) isDomainAllowed(domain string) bool {
 }
 
 func (c *Collector) checkRobots(u *url.URL) error {
-	// var robot *robotstxt.RobotsData
-	// var ok bool
-	var err error
-
 	c.lock.RLock()
 	robot, ok := c.robotsMap[u.Host]
 	c.lock.RUnlock()
 
 	if !ok {
 		// no robots file cached
-		resp, _ := c.backend.Client.Get(u.Scheme + "://" + u.Host + "/robots.txt")
+		resp, err := c.backend.Client.Get(u.Scheme + "://" + u.Host + "/robots.txt")
+		if err != nil {
+			return err
+		}
 		robot, err = robotstxt.FromResponse(resp)
 		if err != nil {
 			return err
diff --git a/colly_test.go b/colly_test.go
@@ -544,6 +544,19 @@ func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
 
 }
 
+func TestConnectionErrorOnRobotsTxtResultsInError(t *testing.T) {
+	ts := newTestServer()
+	ts.Close() // immediately close the server to force a connection error
+
+	c := NewCollector()
+	c.IgnoreRobotsTxt = false
+	err := c.Visit(ts.URL)
+
+	if err == nil {
+		t.Fatal("Error expected")
+	}
+}
+
 func TestEnvSettings(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()
