Changeset 1936:a32f3fe4fe6d

Timestamp:

02/24/10 14:01:29 (5 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

Fixed encoding issue (reported in #135) when the encoding declared in the HTTP header is unknown. This is the patch proposed by Rolando, with an update to the Request/Response documentation.

Files:

• docs/topics/request-response.rst (modified) (1 diff)
• scrapy/http/response/text.py (modified) (2 diffs)
• scrapy/tests/test_http_response.py (modified) (2 diffs)

docs/topics/request-response.rst

@@ -467 +467 @@
     .. attribute:: TextResponse.encoding
 
-       A string with the encoding of this response. The encoding is resolved in the
-       following order: 
+       A string with the encoding of this response. The encoding is resolved by
+       trying the following mechanisms, in order:
 
        1. the encoding passed in the constructor `encoding` argument
 
-       2. the encoding declared in the Content-Type HTTP header
+       2. the encoding declared in the Content-Type HTTP header. If this
+          encoding is not valid (ie. unknown), it is ignored and the next
+          resolution mechanism is tried.
 
        3. the encoding declared in the response body. The TextResponse class

scrapy/http/response/text.py

@@ -6 +6 @@
 """
 
+import codecs
 import re
 
@@ -65 +66 @@
             encoding = self._ENCODING_RE.search(content_type)
             if encoding:
-                return encoding.group(1)
+                enc = encoding.group(1)
+                try:
+                    codecs.lookup(enc) # check if the encoding is valid
+                    return enc
+                except LookupError:
+                    pass
 
     @memoizemethod_noargs

scrapy/tests/test_http_response.py

@@ -176 +176 @@
         r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
         r4 = self.response_class("http://www.example.com", body="\xa2\xa3")
+        r5 = self.response_class("http://www.example.com",
+        headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3")
 
         self.assertEqual(r1.headers_encoding(), "utf-8")
@@ -183 +185 @@
         self.assertEqual(r3.encoding, 'iso-8859-1')
         self.assertEqual(r4.headers_encoding(), None)
+        self.assertEqual(r5.headers_encoding(), None)
+        self.assertEqual(r5.encoding, "utf-8")
         assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii'
         self._assert_response_values(r1, 'utf-8', u"\xa3")