Changeset 1809:ad834af83918

Timestamp:

10/21/09 16:13:41 (9 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

added DEFAULT_RESPONSE_ENCODING setting

Files:

• docs/topics/settings.rst (modified) (1 diff)
• scrapy/conf/default_settings.py (modified) (1 diff)
• scrapy/http/response/text.py (modified) (2 diffs)
• scrapy/tests/test_http_response.py (modified) (4 diffs)

docs/topics/settings.rst

@@ -339 +339 @@
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`.
+
+.. setting:: DEFAULT_RESPONSE_ENCODING
+
+DEFAULT_RESPONSE_ENCODING
+-------------------------
+
+Default: ``'ascii'``
+
+The default encoding to use for :class:`~scrapy.http.TextResponse` objects (and
+subclasses) when no encoding is declared and no encoding could be inferred from
+the body.
 
 .. setting:: DEPTH_LIMIT

scrapy/conf/default_settings.py

@@ -37 +37 @@
     'Accept-Language': 'en',
 }
+
+DEFAULT_RESPONSE_ENCODING = 'ascii'
 
 DEPTH_LIMIT = 0

scrapy/http/response/text.py

@@ -12 +12 @@
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs
+from scrapy.conf import settings
 
 class TextResponse(Response):
 
+    _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING']
     _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I)
 
@@ -72 +74 @@
         dammit = UnicodeDammit(self.body, possible_encodings)
         self._body_inferred_encoding = dammit.originalEncoding
+        if self._body_inferred_encoding in ('ascii', None):
+            self._body_inferred_encoding = self._DEFAULT_ENCODING
         return dammit.unicode
 

scrapy/tests/test_http_response.py

@@ -3 +3 @@
 
 from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers
+from scrapy.conf import settings
 
 
@@ -139 +140 @@
 
     def test_unicode_url(self):
-        # instantiate with unicode url without encoding
-        self.assertRaises(TypeError, self.response_class, u"http://www.example.com/")
+        # instantiate with unicode url without encoding (should set default encoding)
+        resp = self.response_class(u"http://www.example.com/")
+        self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING'])
+
         # make sure urls are converted to str
         resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
@@ -188 +191 @@
         self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
 
-
 class HtmlResponseTest(TextResponseTest):
 
@@ -230 +232 @@
         body = "<xml></xml>"
         r1 = self.response_class("http://www.example.com", body=body)
-        # XXX: we may want to swtich default XmlResponse encoding to utf-8
-        self._assert_response_values(r1, 'ascii', body)
+        self._assert_response_values(r1, settings['DEFAULT_RESPONSE_ENCODING'], body)
 
         body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""