how to run scrapy spider code example
Example 1: scrapy spider shell from spider
from scrapy.shell import inspect_response
def parse_details(self, response, item=None):
if item:
return item
else:
inspect_response(response, self)
Example 2: how to run scrapy inside a nm
import scrapy
from scrapy.crawler import CrawlerProcess
class MySpider(scrapy.Spider):
...
process = CrawlerProcess(settings={
"FEEDS": {
"items.json": {"format": "json"},
},
})
process.crawl(MySpider)
process.start()
Example 3: simple scrapy spider file
import scrapy
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
]
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)