Changeset 1849:1d0ac164cf62 for scrapy/contrib/corestats.py

Timestamp:

11/14/09 20:28:59 (10 months ago)

Author:

Pablo Hoffman <pablo@…>

Branch:

default

Message:

StatsCollector?: ported methods to receive spider instances (closes #113), removed list_domains() method, added iter_spider_stats() method

Files:

• scrapy/contrib/corestats.py (modified) (2 diffs)

scrapy/contrib/corestats.py

@@ -11 +11 @@
 from scrapy.core import signals
 from scrapy.stats import stats
-from scrapy.stats.signals import stats_domain_opened, stats_domain_closing
+from scrapy.stats.signals import stats_spider_opened, stats_spider_closing
 from scrapy.conf import settings
 
@@ -23 +23 @@
         stats.set_value('envinfo/pid', os.getpid())
 
-        dispatcher.connect(self.stats_domain_opened, signal=stats_domain_opened)
-        dispatcher.connect(self.stats_domain_closing, signal=stats_domain_closing)
+        dispatcher.connect(self.stats_spider_opened, signal=stats_spider_opened)
+        dispatcher.connect(self.stats_spider_closing, signal=stats_spider_closing)
         dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
         dispatcher.connect(self.item_passed, signal=signals.item_passed)
         dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
 
-    def stats_domain_opened(self, domain):
-        stats.set_value('start_time', datetime.datetime.utcnow(), domain=domain)
-        stats.set_value('envinfo/host', stats.get_value('envinfo/host'), domain=domain)
-        stats.inc_value('domain_count/opened')
+    def stats_spider_opened(self, spider):
+        stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+        stats.set_value('envinfo/host', stats.get_value('envinfo/host'), spider=spider)
+        stats.inc_value('spider_count/opened')
 
-    def stats_domain_closing(self, domain, reason):
-        stats.set_value('finish_time', datetime.datetime.utcnow(), domain=domain)
-        stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, domain=domain)
-        stats.inc_value('domain_count/%s' % reason, domain=domain)
+    def stats_spider_closing(self, spider, reason):
+        stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+        stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, spider=spider)
+        stats.inc_value('spider_count/%s' % reason, spider=spider)
 
     def item_scraped(self, item, spider):
-        stats.inc_value('item_scraped_count', domain=spider.domain_name)
+        stats.inc_value('item_scraped_count', spider=spider)
         stats.inc_value('item_scraped_count')
 
     def item_passed(self, item, spider):
-        stats.inc_value('item_passed_count', domain=spider.domain_name)
+        stats.inc_value('item_passed_count', spider=spider)
         stats.inc_value('item_passed_count')
 
     def item_dropped(self, item, spider, exception):
         reason = exception.__class__.__name__
-        stats.inc_value('item_dropped_count', domain=spider.domain_name)
-        stats.inc_value('item_dropped_reasons_count/%s' % reason, domain=spider.domain_name)
+        stats.inc_value('item_dropped_count', spider=spider)
+        stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
         stats.inc_value('item_dropped_count')