how to use beautifulsoup to scrape web pages code example
Example 1: webbscraping website with beautifulsoup
import requests
from bs4 import BeautifulSoup
URL = 'https://www.monster.com/jobs/search/?q=Software-Developer&where=Australia'
page = requests.get(URL)
soup = BeautifulSoup(page.content, 'html.parser')
Example 2: Use Beautifulsoup or Scrapy to Scrape a Book Store
import scrapy
class bookScraper(scrapy.Spider):
name = "bookscrape"
start_urls = [
'http://books.toscrape.com/'
]
def parse(self, response):
all_books = response.css('.col-lg-3 ')
for book in all_books:
img_link = book.css('a img::attr(src)').extract()
title = book.css('h3 a::attr(title)').extract()
price = book.css('div.product_price p.price_color::text').extract()
yield {
'image_url' : img_link,
'book_title' : title,
'product_price' : price
}
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, callback = self.parse)