Run Scrapy crawler in a thread

When you run the Scrapy crawler from a program, the code blocks until the Scrapy crawer is finished. This is due to how Twisted (the underlying asynchronous network library) works. This prevents using the Scrapy crawler from scripts or other code.

To circumvent this issue you can run the Scrapy crawler in a thread with the following code:

CrawlerThread

"""
Code to run Scrapy crawler in a thread - works on Scrapy 0.8
"""

import threading, Queue

from twisted.internet import reactor

from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.manager import scrapymanager
from scrapy.core.engine import scrapyengine
from scrapy.core import signals

class CrawlerThread(threading.Thread):

    def __init__(self): 
        threading.Thread.__init__(self)
        self.running = False

    def run(self):
        self.running = True
        scrapymanager.configure(control_reactor=False)
        scrapymanager.start()
        reactor.run(installSignalHandlers=False)

    def crawl(self, *args):
        if not self.running:
            raise RuntimeError("CrawlerThread not running")
        self._call_and_block_until_signal(signals.spider_closed, \
            scrapymanager.crawl, *args)

    def stop(self):
        reactor.callFromThread(scrapyengine.stop)

    def _call_and_block_until_signal(self, signal, f, *a, **kw):
        q = Queue.Queue()
        def unblock():
            q.put(None)
        dispatcher.connect(unblock, signal=signal)
        reactor.callFromThread(f, *a, **kw)
        q.get()

Usage example

Here's a simple script which uses CrawlerThread to crawl a couple of spiders and print the names of the items found.

import os
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'myproject.settings')

from scrapy.xlib.pydispatch import dispatcher
from scrapy.core import signals
from scrapy.conf import settings
from scrapy.crawler import CrawlerThread

settings.overrides['LOG_ENABLED'] = False # avoid log noise

def item_passed(item):
    print "Just scraped item:", item

dispatcher.connect(item_passed, signal=signals.item_passed)

crawler = CrawlerThread()
print "Starting crawler thread..."
crawler.start()
print "Crawling somedomain.com...."
crawler.crawl('somedomain.com) # blocking call
print "Crawling anotherdomain.com..."
crawler.crawl('anotherdomain.com') # blocking call
print "Stopping crawler thread..."
crawler.stop()

The output of running of that script would be something like this:

Starting crawler thread...
Crawling somedomain.com...
Just scraped item: ...some item here...
Just scraped item: ...some item here...
...
Crawling anotherdomain.com
Just scraped item: ...some item here...
Just scraped item: ...some item here...
...
Stopping crawler thread...