Recording the total time taken for running a spider in scrapy
This could be useful:
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.stats import stats
from datetime import datetime
def handle_spider_closed(spider, reason):
print 'Spider closed:', spider.name, stats.get_stats(spider)
print 'Work time:', datetime.now() - stats.get_stats(spider)['start_time']
dispatcher.connect(handle_spider_closed, signals.spider_closed)