Merge pull request #479 from WGH-/fix-relative-base

asciimoo · web-flow · commit 6581387f98e6 · 2020-05-11T00:54:15.000+02:00
Fix relative &lt;base&gt; URL
diff --git a/colly.go b/colly.go
@@ -1057,7 +1057,7 @@ func (c *Collector) handleOnHTML(resp *Response) error {
 		return err
 	}
 	if href, found := doc.Find("base[href]").Attr("href"); found {
-		resp.Request.baseURL, _ = url.Parse(href)
+		resp.Request.baseURL, _ = resp.Request.URL.Parse(href)
 	}
 	for _, cc := range c.htmlCallbacks {
 		i := 0
@@ -1096,7 +1096,7 @@ func (c *Collector) handleOnXML(resp *Response) error {
 		if e := htmlquery.FindOne(doc, "//base"); e != nil {
 			for _, a := range e.Attr {
 				if a.Key == "href" {
-					resp.Request.baseURL, _ = url.Parse(a.Val)
+					resp.Request.baseURL, _ = resp.Request.URL.Parse(a.Val)
 					break
 				}
 			}
diff --git a/colly_test.go b/colly_test.go
@@ -139,6 +139,21 @@ func newTestServer() *httptest.Server {
 		`))
 	})
 
+	mux.HandleFunc("/base_relative", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		w.Write([]byte(`<!DOCTYPE html>
+<html>
+<head>
+<title>Test Page</title>
+<base href="/foobar/" />
+</head>
+<body>
+<a href="z">link</a>
+</body>
+</html>
+		`))
+	})
+
 	mux.HandleFunc("/large_binary", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/octet-stream")
 		ww := bufio.NewWriter(w)
@@ -767,6 +782,31 @@ func TestBaseTag(t *testing.T) {
 	c2.Visit(ts.URL + "/base")
 }
 
+func TestBaseTagRelative(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+
+	c := NewCollector()
+	c.OnHTML("a[href]", func(e *HTMLElement) {
+		u := e.Request.AbsoluteURL(e.Attr("href"))
+		expected := ts.URL + "/foobar/z"
+		if u != expected {
+			t.Errorf("Invalid <base /> tag handling in OnHTML: expected %q, got %q", expected, u)
+		}
+	})
+	c.Visit(ts.URL + "/base_relative")
+
+	c2 := NewCollector()
+	c2.OnXML("//a", func(e *XMLElement) {
+		u := e.Request.AbsoluteURL(e.Attr("href"))
+		expected := ts.URL + "/foobar/z"
+		if u != expected {
+			t.Errorf("Invalid <base /> tag handling in OnXML: expected %q, got %q", expected, u)
+		}
+	})
+	c2.Visit(ts.URL + "/base_relative")
+}
+
 func TestCollectorCookies(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()

Original file line number	Diff line number	Diff line change
`@@ -1057,7 +1057,7 @@ func (c Collector) handleOnHTML(resp Response) error {`
`1057`	`1057`	`return err`
`1058`	`1058`	`}`
`1059`	`1059`	`if href, found := doc.Find("base[href]").Attr("href"); found {`
`1060`		`- resp.Request.baseURL, _ = url.Parse(href)`
	`1060`	`+ resp.Request.baseURL, _ = resp.Request.URL.Parse(href)`
`1061`	`1061`	`}`
`1062`	`1062`	`for _, cc := range c.htmlCallbacks {`
`1063`	`1063`	`i := 0`
`@@ -1096,7 +1096,7 @@ func (c Collector) handleOnXML(resp Response) error {`
`1096`	`1096`	`if e := htmlquery.FindOne(doc, "//base"); e != nil {`
`1097`	`1097`	`for _, a := range e.Attr {`
`1098`	`1098`	`if a.Key == "href" {`
`1099`		`- resp.Request.baseURL, _ = url.Parse(a.Val)`
	`1099`	`+ resp.Request.baseURL, _ = resp.Request.URL.Parse(a.Val)`
`1100`	`1100`	`break`
`1101`	`1101`	`}`
`1102`	`1102`	`}`