Changeset 1849:1d0ac164cf62 for scrapy/contrib/itemsampler.py

Timestamp:

11/14/09 20:28:59 (10 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

StatsCollector?: ported methods to receive spider instances (closes #113), removed list_domains() method, added iter_spider_stats() method

Files:

• scrapy/contrib/itemsampler.py (modified) (4 diffs)

scrapy/contrib/itemsampler.py

@@ -54 +54 @@
 
     def process_item(self, spider, item):
-        sampled = stats.get_value("items_sampled", 0, domain=spider.domain_name)
+        sampled = stats.get_value("items_sampled", 0, spider=spider)
         if sampled < items_per_spider:
             self.items[item.guid] = item
             sampled += 1
-            stats.set_value("items_sampled", sampled, domain=spider.domain_name)
+            stats.set_value("items_sampled", sampled, spider=spider)
             log.msg("Sampled %s" % item, spider=spider, level=log.INFO)
             if close_spider and sampled == items_per_spider:
@@ -72 +72 @@
 
     def spider_closed(self, spider, reason):
-        if reason == 'finished' and not stats.get_value("items_sampled", domain=spider.domain_name):
+        if reason == 'finished' and not stats.get_value("items_sampled", spider=spider):
             self.empty_domains.add(spider.domain_name)
         self.spiders_count += 1
@@ -88 +88 @@
 
     def process_spider_input(self, response, spider):
-        if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_spider:
+        if stats.get_value("items_sampled", spider=spider) >= items_per_spider:
             return []
         elif max_response_size and max_response_size > len(response_httprepr(response)):  
@@ -101 +101 @@
                 items.append(r)
 
-        if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_spider:
+        if stats.get_value("items_sampled", spider=spider) >= items_per_spider:
             return []
         else: