Index: scrapy/contrib/downloadermiddleware/redirect.py =================================================================== --- scrapy/contrib/downloadermiddleware/redirect.py (revision 1421:0f9ee0e39384) +++ scrapy/contrib/downloadermiddleware/redirect.py (revision 1804:8d03689ef662) @@ -15,6 +15,4 @@ def process_response(self, request, response, spider): - domain = spider.domain_name - if response.status in [302, 303] and 'Location' in response.headers: redirected_url = urljoin_rfc(request.url, response.headers['location']) @@ -30,6 +28,6 @@ interval, url = get_meta_refresh(response) - if url and int(interval) < self.max_metarefresh_delay: - redirected = request.replace(url=urljoin_rfc(request.url, url)) + if url and interval < self.max_metarefresh_delay: + redirected = request.replace(url=url) return self._redirect(redirected, request, spider, 'meta refresh') Index: scrapy/tests/test_utils_response.py =================================================================== --- scrapy/tests/test_utils_response.py (revision 1437:bb36d272ab74) +++ scrapy/tests/test_utils_response.py (revision 1804:8d03689ef662) @@ -43,5 +43,5 @@ """ response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) @@ -53,5 +53,5 @@ url=http://example.org/newpage" />""" response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines @@ -61,5 +61,36 @@ CONTENT="1; URL=http://example.org/newpage">""" response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) + + # entities in the redirect url + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) + + # relative redirects + body = """""" + response = Response(url='http://example.com/page/this.html', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) + + # non-standard encodings (utf-16) + body = """""" + body = body.decode('ascii').encode('utf-16') + response = TextResponse(url='http://example.com', body=body, encoding='utf-16') + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect')) + + # non-ascii chars in the url (default encoding - utf8) + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) + + # non-ascii chars in the url (custom encoding - latin1) + body = """""" + response = TextResponse(url='http://example.com', body=body, encoding='latin1') + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) + + # wrong encodings (possibly caused by truncated chunks) + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) def test_response_httprepr(self): Index: scrapy/utils/response.py =================================================================== --- scrapy/utils/response.py (revision 1592:14c9408de227) +++ scrapy/utils/response.py (revision 1804:8d03689ef662) @@ -14,4 +14,6 @@ from twisted.web.http import RESPONSES +from scrapy.utils.markup import remove_entities +from scrapy.utils.url import safe_url_string, urljoin_rfc from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse @@ -35,14 +37,27 @@ return _baseurl_cache[response] -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): - """ Return a tuple of two strings containing the interval and url included - in the http-equiv parameter of the HTML meta element. If no url is included - (None, None) is returned [instead of (interval, None)] + """Parse the http-equiv parameter of the HTML meta element from the given + response and return a tuple (interval, url) where interval is an integer + containing the delay in seconds (or zero if not present) and url is a + string with the absolute url to redirect. + + If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - match = META_REFRESH_RE.search(response.body[0:4096]) - _metaref_cache[response] = match.groups() if match else (None, None) + encoding = getattr(response, 'encoding', 'utf-8') + body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ + errors='ignore')) + match = META_REFRESH_RE.search(body_chunk) + if match: + interval = int(match.group('int')) + url = safe_url_string(match.group('url').strip(' "\'')) + url = urljoin_rfc(response.url, url) + _metaref_cache[response] = (interval, url) + else: + _metaref_cache[response] = (None, None) + #_metaref_cache[response] = match.groups() if match else (None, None) return _metaref_cache[response]