Changeset 1843:133d5e60dded

Timestamp:

11/13/09 14:25:47 (9 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

Refactored HttpCache? middleware:

* simplified code
* performance improvements
* removed awkward/unused domain sectorization
* it can now receive Settings on constructor
* added unittests
* added documentation about filesystem storage structure

Also made scrapy.conf.Settings objects instantiable with a dict which is used to override default settings.

Files:

• docs/topics/downloader-middleware.rst (modified) (2 diffs)
• docs/topics/settings.rst (modified) (1 diff)
• scrapy/conf/__init__.py (modified) (2 diffs)
• scrapy/conf/default_settings.py (modified) (1 diff)
• scrapy/contrib/downloadermiddleware/httpcache.py (modified) (3 diffs)
• scrapy/tests/test_downloadermiddleware_httpcache.py (added)

docs/topics/downloader-middleware.rst

@@ -215 +215 @@
 
     This middleware provides low-level cache to all HTTP requests and responses.
-    Every request and its corresponding response are cached and then, when that
-    same request is seen again, the response is returned without transferring
+    Every request and its corresponding response are cached. When the same
+    request is seen again, the response is returned without transferring
     anything from the Internet.
 
@@ -223 +223 @@
     an Internet connection.
 
-    The :class:`HttpCacheMiddleware` can be configured through the following
-    settings (see the settings documentation for more info):
-
-    * :setting:`HTTPCACHE_DIR` - this one actually enables the cache besides
-      settings the cache dir
-    * :setting:`HTTPCACHE_IGNORE_MISSING` - ignoring missing requests instead
-      of downloading them
-    * :setting:`HTTPCACHE_SECTORIZE` - split HTTP cache in several directories
-      (for performance reasons)
-    * :setting:`HTTPCACHE_EXPIRATION_SECS` - how many secs until the cache is
-      considered out of date
+File system storage
+~~~~~~~~~~~~~~~~~~~
+
+By default, the :class:`HttpCacheMiddleware` uses a file system storage  with the following structure:
+
+Each request/response pair is stored in a different directory containing with
+the following files:
+
+ * ``request_body`` - the plain request body
+ * ``request_headers`` - the request headers (in raw HTTP format)
+ * ``response_body`` - the plain response body
+ * ``response_headers`` - the request headers (in raw HTTP format)
+ * ``meta`` - some metadata of this cache resource in Python ``repr()`` format
+   (for easy grepeability)
+ * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
+   efficient deserialization
+
+The directory name is made from the request fingerprint (see
+``scrapy.utils.request.fingerprint``), and one level of subdirectories is
+used to avoid creating too many files into the same directory (which is
+inefficient in many file systems). An example directory could be::
+
+   /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+
+The cache storage backend can be changed with the :setting:`HTTPCACHE_STORAGE`
+setting, but no other backend is provided with Scrapy yet.
+
+Settings
+~~~~~~~~
+
+The :class:`HttpCacheMiddleware` can be configured through the following
+settings:
+
+.. setting:: HTTPCACHE_DIR
+
+HTTPCACHE_DIR
+^^^^^^^^^^^^^
+
+Default: ``''`` (empty string)
+
+The directory to use for storing the (low-level) HTTP cache. If empty the HTTP
+cache will be disabled.
+
+.. setting:: HTTPCACHE_EXPIRATION_SECS
+
+HTTPCACHE_EXPIRATION_SECS
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``0``
+
+Number of seconds to use for HTTP cache expiration. Requests that were cached
+before this time will be re-downloaded. If zero, cached requests will always
+expire. Negative numbers means requests will never expire.
+
+.. setting:: HTTPCACHE_IGNORE_MISSING
+
+HTTPCACHE_IGNORE_MISSING
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``False``
+
+If enabled, requests not found in the cache will be ignored instead of downloaded. 
+
+.. setting:: HTTPCACHE_STORAGE
+
+HTTPCACHE_STORAGE
+^^^^^^^^^^^^^^^^^
+
+Default: ``'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'``
+
+The class which implements the cache storage backend.
+
 
 .. _topics-dlmw-robots:

docs/topics/settings.rst

@@ -220 +220 @@
 The version of the bot implemented by this Scrapy project. This will be used to
 construct the User-Agent by default.
-
-.. setting:: HTTPCACHE_DIR
-
-HTTPCACHE_DIR
--------------
-
-Default: ``''`` (empty string)
-
-The directory to use for storing the (low-level) HTTP cache. If empty the HTTP
-cache will be disabled.
-
-.. setting:: HTTPCACHE_EXPIRATION_SECS
-
-HTTPCACHE_EXPIRATION_SECS
--------------------------
-
-Default: ``0``
-
-Number of seconds to use for HTTP cache expiration. Requests that were cached
-before this time will be re-downloaded. If zero, cached requests will always
-expire. Negative numbers means requests will never expire.
-
-.. setting:: HTTPCACHE_IGNORE_MISSING
-
-HTTPCACHE_IGNORE_MISSING
-------------------------
-
-Default: ``False``
-
-If enabled, requests not found in the cache will be ignored instead of downloaded. 
-
-.. setting:: HTTPCACHE_SECTORIZE
-
-HTTPCACHE_SECTORIZE
--------------------
-
-Default: ``True``
-
-Whether to split HTTP cache storage in several dirs for performance.
 
 .. setting:: COMMANDS_MODULE

scrapy/conf/__init__.py

@@ -14 +14 @@
 class Settings(object):
 
-    def __init__(self):
+    def __init__(self, overrides=None):
         self.defaults = {}
         self.global_defaults = default_settings
@@ -25 +25 @@
         pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
         self.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
+        if overrides:
+            self.overrides.update(overrides)
 
     def __getitem__(self, opt_name):

scrapy/conf/default_settings.py

@@ -95 +95 @@
 HTTPCACHE_DIR = ''
 HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_SECTORIZE = True
+HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
 

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -1 +1 @@
 from __future__ import with_statement
 
-import errno
 import os
-import hashlib
-import datetime
+from os.path import join, exists
+from time import time
 import cPickle as pickle
+
 from scrapy.xlib.pydispatch import dispatcher
-
 from scrapy.core import signals
-from scrapy import log
 from scrapy.http import Headers
 from scrapy.core.exceptions import NotConfigured, IgnoreRequest
@@ -16 +14 @@
 from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.conf import settings
+from scrapy.utils.misc import load_object
+from scrapy import conf
 
 
 class HttpCacheMiddleware(object):
-    def __init__(self):
-        if not settings['HTTPCACHE_DIR']:
-            raise NotConfigured
-        self.cache = Cache(settings['HTTPCACHE_DIR'], sectorize=settings.getbool('HTTPCACHE_SECTORIZE'))
+
+    def __init__(self, settings=conf.settings):
+        self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
         self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
-        dispatcher.connect(self.open_domain, signal=signals.spider_opened)
+        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
+        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
 
-    def open_domain(self, spider):
-        self.cache.open_domain(spider.domain_name)
+    def spider_opened(self, spider):
+        self.storage.open_spider(spider)
+
+    def spider_closed(self, spider):
+        self.storage.close_spider(spider)
 
     def process_request(self, request, spider):
-        if not is_cacheable(request):
+        if not self.is_cacheable(request):
             return
-
-        key = request_fingerprint(request)
-        domain = spider.domain_name
-
-        try:
-            response = self.cache.retrieve_response(domain, key)
-        except:
-            log.msg("Corrupt cache for %s" % request.url, log.WARNING)
-            response = False
-
+        response = self.storage.retrieve_response(spider, request)
         if response:
+            response.flags.append('cached')
             return response
         elif self.ignore_missing:
@@ -49 +43 @@
 
     def process_response(self, request, response, spider):
-        if is_cacheable(request):
-            key = request_fingerprint(request)
-            self.cache.store(spider.domain_name, key, request, response)
-
+        if self.is_cacheable(request):
+            self.storage.store_response(spider, request, response)
         return response
 
-
-def is_cacheable(request):
-    return urlparse_cached(request).scheme in ['http', 'https']
+    def is_cacheable(self, request):
+        return urlparse_cached(request).scheme in ['http', 'https']
 
 
-class Cache(object):
-    DOMAIN_SECTORDIR = 'data'
-    DOMAIN_LINKDIR = 'domains'
+class FilesystemCacheStorage(object):
 
-    def __init__(self, cachedir, sectorize=False):
+    def __init__(self, settings=conf.settings):
+        cachedir = settings['HTTPCACHE_DIR']
+        if not cachedir:
+            raise NotConfigured
         self.cachedir = cachedir
-        self.sectorize = sectorize
+        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
 
-        self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR)
-        if not os.path.exists(self.baselinkpath):
-            os.makedirs(self.baselinkpath)
+    def open_spider(self, spider):
+        pass
 
-        self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR)
-        if not os.path.exists(self.basesectorpath):
-            os.makedirs(self.basesectorpath)
+    def close_spider(self, spider):
+        pass
 
-    def domainsectorpath(self, domain):
-        sector = hashlib.sha1(domain).hexdigest()[0]
-        return os.path.join(self.basesectorpath, sector, domain)
-
-    def domainlinkpath(self, domain):
-        return os.path.join(self.baselinkpath, domain)
-
-    def requestpath(self, domain, key):
-        linkpath = self.domainlinkpath(domain)
-        return os.path.join(linkpath, key[0:2], key)
-
-    def open_domain(self, domain):
-        if domain:
-            linkpath = self.domainlinkpath(domain)
-            if self.sectorize:
-                sectorpath = self.domainsectorpath(domain)
-                if not os.path.exists(sectorpath):
-                    os.makedirs(sectorpath)
-                if not os.path.exists(linkpath):
-                    try:
-                        os.symlink(sectorpath, linkpath)
-                    except:
-                        os.makedirs(linkpath) # windows filesystem
-            else:
-                if not os.path.exists(linkpath):
-                    os.makedirs(linkpath)
-
-    def read_meta(self, domain, key):
-        """Return the metadata dictionary (possibly empty) if the entry is
-        cached, None otherwise.
-        """
-        requestpath = self.requestpath(domain, key)
-        try:
-            with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
-                metadata = pickle.load(f)
-        except IOError, e:
-            if e.errno != errno.ENOENT:
-                raise
-            return None
-        expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        if expiration_secs >= 0:
-            expiration_date = metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs)
-            if datetime.datetime.utcnow() > expiration_date:
-                log.msg('dropping old cached response from %s' % metadata['timestamp'], \
-                    level=log.DEBUG, domain=domain)
-                return None
-        return metadata
-
-    def retrieve_response(self, domain, key):
-        """
-        Return response dictionary if request has correspondent cache record;
-        return None if not.
-        """
-        metadata = self.read_meta(domain, key)
+    def retrieve_response(self, spider, request):
+        """Return response if present in cache, or None otherwise."""
+        metadata = self._read_meta(spider, request)
         if metadata is None:
-            return None # not cached
-
-        requestpath = self.requestpath(domain, key)
-        responsebody = responseheaders = None
-        with open(os.path.join(requestpath, 'response_body')) as f:
-            responsebody = f.read()
-        with open(os.path.join(requestpath, 'response_headers')) as f:
-            responseheaders = f.read()
-
+            return # not cached
+        rpath = self._get_request_path(spider, request)
+        with open(join(rpath, 'response_body'), 'rb') as f:
+            body = f.read()
+        with open(join(rpath, 'response_headers'), 'rb') as f:
+            rawheaders = f.read()
         url = metadata['url']
-        headers = Headers(headers_raw_to_dict(responseheaders))
         status = metadata['status']
-
+        headers = Headers(headers_raw_to_dict(rawheaders))
         respcls = responsetypes.from_args(headers=headers, url=url)
-        response = respcls(url=url, headers=headers, status=status, body=responsebody)
-        response.meta['cached'] = True
-        response.flags.append('cached')
+        response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
-    def store(self, domain, key, request, response):
-        requestpath = self.requestpath(domain, key)
-        if not os.path.exists(requestpath):
-            os.makedirs(requestpath)
+    def store_response(self, spider, request, response):
+        """Store the given response in the cache."""
+        rpath = self._get_request_path(spider, request)
+        if not exists(rpath):
+            os.makedirs(rpath)
+        metadata = {
+            'url': request.url,
+            'method': request.method,
+            'status': response.status,
+            'timestamp': time(),
+        }
+        with open(join(rpath, 'meta'), 'wb') as f:
+            f.write(repr(metadata))
+        with open(join(rpath, 'pickled_meta'), 'wb') as f:
+            pickle.dump(metadata, f, protocol=2)
+        with open(join(rpath, 'response_headers'), 'wb') as f:
+            f.write(headers_dict_to_raw(response.headers))
+        with open(join(rpath, 'response_body'), 'wb') as f:
+            f.write(response.body)
+        with open(join(rpath, 'request_headers'), 'wb') as f:
+            f.write(headers_dict_to_raw(request.headers))
+        with open(join(rpath, 'request_body'), 'wb') as f:
+            f.write(request.body)
 
-        metadata = {
-                'url':request.url,
-                'method': request.method,
-                'status': response.status,
-                'domain': domain,
-                'timestamp': datetime.datetime.utcnow(),
-            }
+    def _get_request_path(self, spider, request):
+        key = request_fingerprint(request)
+        return join(self.cachedir, spider.domain_name, key[0:2], key)
 
-        # metadata
-        with open(os.path.join(requestpath, 'meta_data'), 'w') as f:
-            f.write(repr(metadata))
-        # pickled metadata (to recover without using eval)
-        with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f:
-            pickle.dump(metadata, f)
-        # response
-        with open(os.path.join(requestpath, 'response_headers'), 'w') as f:
-            f.write(headers_dict_to_raw(response.headers))
-        with open(os.path.join(requestpath, 'response_body'), 'w') as f:
-            f.write(response.body)
-        # request
-        with open(os.path.join(requestpath, 'request_headers'), 'w') as f:
-            f.write(headers_dict_to_raw(request.headers))
-        if request.body:
-            with open(os.path.join(requestpath, 'request_body'), 'w') as f:
-                f.write(request.body)
+    def _read_meta(self, spider, request):
+        rpath = self._get_request_path(spider, request)
+        metapath = join(rpath, 'pickled_meta')
+        if not exists(metapath):
+            return # not found
+        mtime = os.stat(rpath).st_mtime
+        if 0 <= self.expiration_secs < time() - mtime:
+            return # expired
+        with open(metapath, 'rb') as f:
+            return pickle.load(f)