scrapy python tutorial code example
Example 1: python web crawler
import scrapy
class BlogSpider(scrapy.Spider):
name = 'blogspider'
start_urls = ['https://blog.scrapinghub.com']
def parse(self, response):
for title in response.css('.post-header>h2'):
yield {'title': title.css('a ::text').get()}
for next_page in response.css('a.next-posts-link'):
yield response.follow(next_page, self.parse)
Example 2: scrapy create project
scrapy startproject projectname
Example 3: scrapy tutorial
import scrapy
class AliexpressTabletsSpider(scrapy.Spider):
name = 'aliexpress_tablets'
allowed_domains = ['aliexpress.com']
start_urls = ['https://www.aliexpress.com/category/200216607/tablets.html',
'https://www.aliexpress.com/category/200216607/tablets/2.html?site=glo&g=y&tag=']
def parse(self, response):
print("procesing:"+response.url)
product_name=response.css('.product::text').extract()
price_range=response.css('.value::text').extract()
orders=response.xpath("//em[@title='Total Orders']/text()").extract()
company_name=response.xpath("//a[@class='store $p4pLog']/text()").extract()
row_data=zip(product_name,price_range,orders,company_name)
for item in row_data:
scraped_info = {
'page':response.url,
'product_name' : item[0],
'price_range' : item[1],
'orders' : item[2],
'company_name' : item[3],
}
yield scraped_info