Changeset 1804:8d03689ef662

Timestamp:

10/21/09 09:37:04 (9 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

make get_meta_refresh() function more robust and changed interface to return (int, str) as (interval, absolute_url)

Location:

scrapy

Files:

• contrib/downloadermiddleware/redirect.py (modified) (2 diffs)
• tests/test_utils_response.py (modified) (3 diffs)
• utils/response.py (modified) (2 diffs)

scrapy/contrib/downloadermiddleware/redirect.py

@@ -15 +15 @@
 
     def process_response(self, request, response, spider):
-        domain = spider.domain_name
-
         if response.status in [302, 303] and 'Location' in response.headers:
             redirected_url = urljoin_rfc(request.url, response.headers['location'])
@@ -30 +28 @@
 
         interval, url = get_meta_refresh(response)
-        if url and int(interval) < self.max_metarefresh_delay:
-            redirected = request.replace(url=urljoin_rfc(request.url, url))
+        if url and interval < self.max_metarefresh_delay:
+            redirected = request.replace(url=url)
             return self._redirect(redirected, request, spider, 'meta refresh')
 

scrapy/tests/test_utils_response.py

@@ -43 +43 @@
             </html>"""
         response = Response(url='http://example.org', body=body)
-        self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
+        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
 
         # refresh without url should return (None, None)
@@ -53 +53 @@
             url=http://example.org/newpage" /></head>"""
         response = Response(url='http://example.org', body=body)
-        self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
+        self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
 
         # meta refresh in multiple lines
@@ -61 +61 @@
                CONTENT="1; URL=http://example.org/newpage">"""
         response = Response(url='http://example.org', body=body)
-        self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage'))
+        self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage'))
+
+        # entities in the redirect url
+        body = """<meta http-equiv="refresh" content="3; url=&#39;http://www.example.com/other&#39;">"""
+        response = Response(url='http://example.com', body=body)
+        self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other'))
+
+        # relative redirects
+        body = """<meta http-equiv="refresh" content="3; url=other.html">"""
+        response = Response(url='http://example.com/page/this.html', body=body)
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html'))
+
+        # non-standard encodings (utf-16)
+        body = """<meta http-equiv="refresh" content="3; url=http://example.com/redirect">"""
+        body = body.decode('ascii').encode('utf-16')
+        response = TextResponse(url='http://example.com', body=body, encoding='utf-16')
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect'))
+
+        # non-ascii chars in the url (default encoding - utf8)
+        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">"""
+        response = Response(url='http://example.com', body=body)
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
+
+        # non-ascii chars in the url (custom encoding - latin1)
+        body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xa3">"""
+        response = TextResponse(url='http://example.com', body=body, encoding='latin1')
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
+
+        # wrong encodings (possibly caused by truncated chunks)
+        body = """<meta http-equiv="refresh" content="3; url=http://example.com/this\xc2_THAT">"""
+        response = Response(url='http://example.com', body=body)
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT'))
 
     def test_response_httprepr(self):

scrapy/utils/response.py

@@ -14 +14 @@
 from twisted.web.http import RESPONSES
 
+from scrapy.utils.markup import remove_entities
+from scrapy.utils.url import safe_url_string, urljoin_rfc
 from scrapy.xlib.BeautifulSoup import BeautifulSoup
 from scrapy.http import Response, HtmlResponse
@@ -35 +37 @@
     return _baseurl_cache[response]
 
-META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE)
+META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', re.DOTALL | re.IGNORECASE)
 _metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
-    """ Return a tuple of two strings containing the interval and url included
-    in the http-equiv parameter of the HTML meta element. If no url is included
-    (None, None) is returned [instead of (interval, None)]
+    """Parse the http-equiv parameter of the HTML meta element from the given
+    response and return a tuple (interval, url) where interval is an integer
+    containing the delay in seconds (or zero if not present) and url is a
+    string with the absolute url to redirect.
+
+    If no meta redirect is found, (None, None) is returned.
     """
     if response not in _metaref_cache:
-        match = META_REFRESH_RE.search(response.body[0:4096])
-        _metaref_cache[response] = match.groups() if match else (None, None)
+        encoding = getattr(response, 'encoding', 'utf-8')
+        body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \
+            errors='ignore'))
+        match = META_REFRESH_RE.search(body_chunk)
+        if match:
+            interval = int(match.group('int'))
+            url = safe_url_string(match.group('url').strip(' "\''))
+            url = urljoin_rfc(response.url, url)
+            _metaref_cache[response] = (interval, url)
+        else:
+            _metaref_cache[response] = (None, None)
+        #_metaref_cache[response] = match.groups() if match else (None, None)
     return _metaref_cache[response]