web scraping google news with python
You can use awesome requests library:
import requests
URL = 'https://www.google.com/search?pz=1&cf=all&ned=us&hl=en&tbm=nws&gl=us&as_q={query}&as_occt=any&as_drrb=b&as_mindate={month}%2F%{from_day}%2F{year}&as_maxdate={month}%2F{to_day}%2F{year}&tbs=cdr%3A1%2Ccd_min%3A3%2F1%2F13%2Ccd_max%3A3%2F2%2F13&as_nsrc=Gulf%20Times&authuser=0'
def run(**params):
response = requests.get(URL.format(**params))
print response.content, response.status_code
run(query="Egypt", month=3, from_day=2, to_day=2, year=13)
And you'll get status_code=200.
And, btw, take a look at scrapy project. Nothing makes web-scraping more simple than this tool.
You can use google-search-results package to extract data from Google News. It's a paid API with a free trial.
Check a demo on Repl.it
from serpapi import GoogleSearch
import os
month = 4
from_day = 2
to_day = 3
year = 2020
params = {
"engine": "google",
"q": "Trump",
"google_domain": "google.com",
"tbm": "nws",
"tbs": f"cdr:1,cd_min:{month}/{from_day}/{year},cd_max:{month}/{to_day}/{year}",
"api_key": os.getenv("API_KEY"),
}
client = GoogleSearch(params)
data = client.get_dict()
print(f"Raw HTML: {data['search_metadata']['raw_html_file']}")
print(f"JSON endpoint: {data['search_metadata']['json_endpoint']}")
print()
print("News results")
for result in data['news_results']:
print(f"""
Title: {result['title']}
Snippet: {result['snippet']}
Date: {result['date']}
""")
Part of JSON response
{
"news_results": [
{
"position": 1,
"title": "Trump Promotes Oil Deal That May Not Exist",
"link": "https://www.nytimes.com/2020/04/02/us/politics/trump-russia-saudi-arabia-oil.html",
"source": "The New York Times",
"date": "15 hours ago",
"snippet": "WASHINGTON — When oil prices crashed in early March after a dispute between \nRussia and Saudi Arabia, President Trump put a positive spin on the news.",
"thumbnail": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBwgHBgkIBwgKCgkLDRYPDQwMDRsUFRAWIB0iIiAdHx8kKDQsJCYxJx8fLT0tMTU3Ojo6Iys/RD84QzQ5OjcBCgoKDQwNGg8PGjclHyU3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3N//AABEIAHAAcAMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAACBAEDBQAGB//EADkQAAIBAwMCAgcHAwMFAAAAAAECAwAEEQUSITFBE1EGFCJhcYGhIzJSkbHB8EJi0STh8RUlc5Ky/8QAGQEAAwEBAQAAAAAAAAAAAAAAAQIDAAQF/8QAJBEAAgICAgEEAwEAAAAAAAAAAAECEQMxEiFhIjJBcRMUUQT/2gAMAwEAAhEDEQA/APQZrs1FdQMFmuzQ1NYxOa6ozXZrGJzU0GakGsYMVmaneXttcTRWMIlIGGBAJQ7N2Rkj49xxWiDzXmrmeSyuLm5VHQ3MjeMWPQeHIqlSRjqVqWS+iuOjas7i4kuZY5oyvh/ezjjIU9vjTlYujSFfU4AHdhFIJpyCBId2fL4D5VsE0cbuIMiXLoKoNRmozVCZXu4rkBkuLeMNtDyqp4zkE/8ANTaqkl/aLIPZ8dAeSMDPP899DaKraoix7huvSACxJGHP7CpzltFIxuidrR3VyjS71SZlTgcAUeaqfaLq6Ef3fHfjy5oqMPagSXbDzXZoM1O6mBQVdmhzXZrGoLNZnpHPD6nHH9puEyF/axyTwR8q0qw/SWHfCrbAd0iDt7Q+NJNdDwNXTpFeyg2tkrEoIxjB8qYzSWmR+FaKgXAwpHH9o/fNNZpo6Fkuwia7NBmozRBQenH/ALnZ8Z+3Tj5/5xQ2B8XVYg3s7rs57f1mrNH51ez/APMP3odKO7WYNx63Lcn4moT3L6KR0vspO1Z7jbwDO5+v+1EKqJzLMfOZ/wD6NGDVYe1Cy9zDrs0OaquJ0t4zJKWCDrtUt+lMKXg1xNVI4dVYZAI4yCD9aLdWCWA0lr7AabAGQSBn4Rs44I5/n6cUzmkdYvYo444wwLKjZDJuHUH9x8KSY0B62bdYWj/igUjNFmlNMmSbTrXaW3LEobI70zmmjoD2FmoqM11MKXaIc6zZ8gfa/sap0mdY9VtZCQwM7HOe3tV1raahaahbS3NtcIqYdpFKdSMYA56Zqq0tXs3QzKRJGzFRIMgZ6Hp86455U7f9R0xxtIC3lNyGZAAzSPgDtliRVgZWAMbBlIBB88jNV2Jit7k7LNlDOJHwxyPyX8h76suptHsxm59ZRRGGHO7HTphcnyorPQHisqZJBsnEhILlNm7jpnPSmtP2tqNsG6AnP/qaIXGjPM1kst140D5IKHqV8yvPHarbdtOt7iKX1iQlcsv2Z5GDk0HlTi09jLH6k1ozrUgQJtGOO5zVu6m4bbTjENk0y49nlDwfmBQvaRIrO12oVfvExkAdutWjmjSRKWKVi+6srUkjlMu5efCkI575H+DW3Hbwypviu0dcE5VSaSvNMWRrzbKsuy23AFMEE7s/tyaznF6NGLWxfQxssI1AA9kdD8a0M+dJ6ZYRLpsSreIgfcikRkZIdunHOc+Qrli8CKHbcqY2HsuiDDdjkEKRgg9aX88V0N+Fy7G6lQzZ2jOBk1mxslqrCe5IAY75dg9nJyM4z2opL6B7O4a3aVWO0hw/OSyAED41v2F/DPC72D/1yNrsK9/FdQycFI0IKDuT5cE9vKgSW4utQmiudPi9XZmXxRexDCjOMKoBweOM8ce+lvR+wk1OKeDSnsnkYF5XgeQnAI4G9iBzwcYqr0djhmvNVub26lfw5iYo0vmZQpJzkA+ycdj07dKg4pX4K23QxZsZZ3nayZX8OUHOooeNhP3gxHu6cU5ckSQNNFC/hNIyLHJMsgbBU7h5DDHjHGKWktrYpKInY24WXHizEhxsbGcnJ49/1zSzLNNokzxqvqVtGEQRgDazMuSSGJJbA6/CsldMEurQxpuoetPuubFTvGCWtwwLFRkjGPLp/itZ5HPhKumSAlXxi3PsnL/3cZ+fWvnaWM1pNEbm3CCaEug38KNp94/hrY1e60+2uLeC5hBURlVYTOnshiM8HmnlBX0LGTrs9NeanYxmQyeGwWZ42BIwGGPNh5j+c0T61p3gB9kAU7Sc8A5Ix/V8O9ZWi6MdS9G31S0toEhspXlEcztuO1Q2c8/XyrN0y/F5Be3VnZCJ5ZzkFva3nbn4cbfzNLxG5Weng1q0MrpH6rsMRkLRl9wHTz4599W6g0Nw+oBfV1PhiMbpNvtYIyCD15qzStMt5DcH1xEeaHw9nhocAg8DD9uaG20W9uEuJ5U8XxIxiTwUAbK54PxJ6eRqfJWPx6FNMu4LfTI5XjV5IpZsM8m053DsG4PPcU0ZbKSVVNrJ/ptyxuZJB97k5y/6/SlvRvSJLiyjhumddrS+xJFkqVdeCDz5dfKnzoVvPctJHqWGUEKPCwBkEcH/AH7Clco2MouhGbUNMBgPhKwJEvM7DaQTyCW56VVI+nxxwv6ozOeV/wBW56MrDksc8kf8VoXvozb30UsEl4GLIVXcudu4seBtPalL70ItvAif1iGFIQWZ2jfGNiqT90fgBNFSg/kVxkZ3o/f6PpeRHZRbhE0SoL0nG7rnk8ZAHmP1us5NKjaWT1J4ZbhQZgJ9xxnqeeOp7diPdWPF6RXLAeKYZPcyFfqOKZj1vw1GzTzhORs2Mq/SrvFNCLJA2pbHSbi1KSWzpERyvPGewwf0qbPRLCCD1G3t5VSR/us7AsQRjHn93+ZrFb00Yn7SzLKOxI7fOmIPSnT32hrcxkdeAM/lSPHkSGU4N7NqbQFuVjjaGdNqbFCufu4xVR9G0uoUM9tcOrdAVDdyeo95NUwa5ZOwaOYLweFmGT+eKjCTQReFeXcLqDnwnG089MHjvU/UvBT0/Bpadp9vHHe2fgu03iAABSojBVT2FFa+jVvbWi28do6RgkkBwM5x/gVjW8F2s05N1cRgkBZDLkvjv7qaPrqrtivrgt+Itk/pQryZLwOXmgSmEpaRKjbQoLYOBu3EcnnPPXz71NjY6pZwmBZZmTHC+IvsjGAMA4I+I7Vmlr9CFfVroN+HeB9MUtcXuoWUsPi6ndESNt++PgO3T2vpRp0A9HC+uAmSSGBHcHdsjQ8nqcgjyFUZ1JWdJtIhmUYw+1Pa8xjcPdWHcX17YD1q41q8kjbI8Pcu08E8YAPbzpiW4v4QGfV7yVRknIHUeWBQ4BUjRmknDFW9HVZdwBeOJeB8N5z5dKsT0gtdy2cUKSyNkPBGsoIbGdpGeuOeprOt9R1GVNyajOTkqctkcHy69K8rpTrbelq2iXl3LdWQcRtOoaKEFPLduJwcZyPnTRxJpiym10YK26/jP5VbFH4bbkZw3mDijAo1UsQAMk16TbODiMx3QbC3UQnXuTw351dc6SHiFxZEyRNnK4wwx1+Pw/WjttJmcbrki3j6ktjOPh2+eKYl1KCziEOnDO3P2h7E8Z95+WPdUm++iqXXZgNCR1FQjPE2Y2ZD5qauZ/PgeVVN76oTHIdZv4eFuGYf3AGm4fSOVT9rCjeZHFY4GTUOoFI8cX8Dqclpnp4fSOE9WeM+8E4rRg1RJipV0lI9wzXhgfLiiVipz0NTl/ni9FFnktnujPCWBliiJx/UmQKlrlW2Zdtq5wAOPhXlbTU5YsLI3iL7+v51rxSwzoHTB+hFQlj47LRyXo0knt1fduXd2O0ZFYtpbxQ+ll1f2l2zXXiKJYmjARYmABGc8kAdseVPDlccMPI813gJjIDRnzVyPpWUXG6ZnJOrP//Z"
},
{
"position": 2,
"title": "Trump’s Oil Summit",
"link": "https://www.wsj.com/articles/trumps-oil-summit-11585870063",
"source": "Wall Street Journal",
"date": "Opinion · 16 hours ago",
"snippet": "Trump's Oil Summit. Tariffs and quotas won't solve a price shock caused by \na pandemic and a Saudi Arabia-Russia feud.",
"thumbnail": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBwgHBgkIBwgKCgkLDRYPDQwMDRsUFRAWIB0iIiAdHx8kKDQsJCYxJx8fLT0tMTU3Ojo6Iys/RD84QzQ5OjcBCgoKDQwNGg8PGjclHyU3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3N//AABEIAHAAcAMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAAFBgIDBAABB//EADwQAAIBAgQEBAQDBgQHAAAAAAECAwQRAAUSIRMxQVEGFCJhMnGBkRWhwSNCYrHR8AczUnIWJEOCksLx/8QAFwEBAQEBAAAAAAAAAAAAAAAAAQACA//EABsRAQEBAAMBAQAAAAAAAAAAAAARARIhMQJB/9oADAMBAAIRAxEAPwD6D5od8d5od8LhrnFzwGAHMkMAMQ/EVbkQD21Y6RjkZ/NjvjvNjvhX/EQote56bnHLX3Nhc/fbBFyNHmx3x55sd8LXnxe1unOxxIZgLkaVuO5OKGmI1gHM48NYvfC2a1msQm9+g2v98ctedyFW/wDffFFTEa1e+ImsXvhefMZBv6ee22I/ibG3+Xb5YYqYfODvj0VgPXC9HXTyH9nHxL/6UvjmrqiPd4tIvsShF8UVMRq8eecthc/FJL3Gj/xx34lIbn0bew3xQUM/FJGGmoaVkYbgn9MZDVq8hKoyL2AH6Yr/ABN29DcOMhbEsL7jEC8czsFlJcdNgB/L3xnN1uDNJX5dDpNTTs3oKMqoLX6G5N7/AGwQgnDU0tXFQxmmQCyFbE/I2sR074TJaqT1IugyH0kagpwZ8K5rRxSSUOZUkEhkY6ZJUBZTt6QencYtDf52krxIIUSjqNV0JbUjDse2LJJVpL64lnQp6nRijD2K3sR8sbsxqkXJ5VzGninKORGEI1AX9NiLHlhPZppuH+ySMhdClYwmr3Nhv/PBm07gtFmtBxCy0ztp3ALkaflY4spp5a1mWkyiKVxc3jVyR731YBxJX08qrxkAJuVJsW/LF5qmp0kVJJIw5P8AlOwKnocOgWkgzNxpny6oijvsI4Dt9Tc48EVQhI4FULc2aK1/qbYDjMswiW8ecZip6ASE/wDsMd/xLnEABGeZhb+NT+pwcjBWTNayL0LVSxBeSgm1vvjNHWLMAauq1Mp2BJwKq8xmrv21TUNPUNYPIR8VuX1A264z8YsGuEH/AG2wclBmpqYQf2DHbnpJP64pNdJyWWSw5nUcCjMEU2sT3vj2Nysd99Xflg5iNUVDmVPKkcqnWtpHMMq3VDz33F9j3xjzLNIpagcGReEi+lCyltxvdlAuet7Ypr5Wq3RvPz1U5XRYRgbdrDE6fJ5WhZi/D4al3GqzqAR0G/XGj0jPPTzMDFI5QOdJDX1e9jjx5NUgIgYs3UkEfYAYI+HqChrM2jpZ0ldTHqIBtc+9t/zwxZ14cy4LTxUkb0c88iosjs7La1zccumBFV55JUtNGrlb7am3vzOLKGameVVqF4aNaxRltfcerbbnzwczTwlHRUUlTBmKvwoyxHB59ed8VeH3oM9pFyqsPlqlDeGWNlXXZTckkcu+LdUTyPL8urKiaCtM9BMgW+4fUTz08tvvzGCld4RqKfMkYmonysKXaUaEaMAH08zfpvbrhZcmgn/Cs9MLCL4JYZQeHcA3jJ5juuD+V+JTlwY5rHDVUlRtTzmPWrr17WPK4ti2/ixPO8xymjoFp6KAO7LotIQxC22Y+nGHL6jIIMsiFZTu0oFme8huepFjbDO1HltZV0telFliRiNrw+XsH1WsSO4sfvhfzbLVzjMpFyyOgy5aX0TDQY0nJAIIAB6dcWeGlII0gfnYG5B6DEo42ndI6aMySuQEWIamY9rdcHqbN6rwjUyUCTRF5DxbRNcNcAczbtyONOUx5XnLvmddVy0WY8dnUwIlwLW3B59emCKlvMcszGhVTmFJUQg/C0sLJq68zjIZkfb32uvI+/2wU8U1eZ5lUPRrO9ZRU8t45IodybWuQpOnmR2wutI8eqCfVvtvsRjO4c0xJVMhMNKrU7auUUVtQ5/Fa/8A9x5JrosulErK1TWMEUI3wx3BYmxtcmw788VzVMcczHy4ZpLk3Omxv8Vha+MubRtBV07zcFQItQCltWw/eDAEewxvWTX/AIdI8smZzCpjhRnWPSWsTYG1vbfBXxq/lMhaVKy7RyI2kN8N2tsw5c8Cv8O0olyVpiHWoeUiQg6gLctuXXF3jURHIKplkNlKH1AC5DC3tg7xNOUeIMubLIlqqqnMmgCTiShWv9d8DfEOV1r+IKdcpoTE81KXUw2UmxsWX3sV++FPwhDSVniCmjrxeE6i66Qxb0k9flh2zSWgop6mvhqCtRSuBCmpUXSbAhl69d7i1sG/W5vjWZk9dSVMHiemkyfxDLUU80UqiFEdAzELYn1i/cWPaxwr66rJKqfL62JJaZnKvTzuo4libMN9m7MMTz/MZJswpsxyuNxMoYysiEqG2G/QE7jF/iHManxBRUMrUJR/+o76VcDppc2Jt0+uD525V9ZNi2kq6jLKiDMsrrp6rKIJLzJ8bwfwsttxuNxtthwj/wASKOoVYxpl4g+FaV2v9MfPMriraCokq8vnSSWJdLmJhpYWvaQWPpsD/UYIRwy5nl7yeHZjR1aAmoy9ban90PUe3689Aa8KZtPlC1xqpLvNOX1yJqUKQLC9th8u+AXjTNqXM83jYS0BjWLS+hlW7XOxsb8rYVhR0zsyVhmSbkryLqCt11at7c+WBrQ1CMOF6gdgUFsIOuT5/FkkTpAWdHN2VRa23IH977DA/wAQ+IYM6lWcxVCtHHpR2ZQSb9d9xy6X98A48tq9N6qRIiOSu1z9hc42JRU6MwEQYKbB7c8FMMeV0yvMwyqj1zL6WaMiw/3S7Af7VF/fBVvAivSmoq8x4MlmLXW8Sg89ybn54bad4qeDg0FGVMYtplXhpF8zy+18LufeK6LK40aomSqmINxpvGD10r13tucHiLGZ5PmfhhWmgrtMDEC8MlrnkLg/0OANdV1lSAKqolmW97O2oX/kPoMbJ86zDPJalpyI6aYC/E7A3H9du2MJWKmE0SMsrfvWXe3TFTEopSra+FAL2N9JFvpvi+Od1eSVZOGDcDSbW+uMrUjpCk2rZ+QCnYDqcVVFLOrBZgy3ANmBHTbbCH0TwjlMXjHKGhnzZ1elbQ0QXU2jmLm4uLjt+mF3xdkP/DmZ+VqV1RSeunqNNhIvXlyI6j+uB/h2trshzTzmXVMUcwQqVYEhge4/vlhhnzWs8VwimzeoSQQHilQyoLgG5ACagLe/XDAV6rMEgWnqssjNLURsSZ4JGJ1dOZIt7YL0GYUOYJLXxJJT5lSoJHp6aVYlcD4pEuDb3XFGYRHK5SsNJQrGwJDeqVuZG4ZiAdjtbGbLc+lo6svIbIo9McUKxgnsQtrjn98FUMOYy02bq088UBr0UcRKdyZFW9gzmwHbpt9sLU3ladCJKmNSSdi4GGgx/jdJFnfhpzHW0kaJU0RILxAC3psPUtvbe3e4wInpaXOG4lHHFTZkd3praVm/ijPf+E/TliXgJ5uMSqqjiA82juw+d+WNBkvt07DGdw0bskilWU2KkWsceasI3aavEfjCvrKhqekQyyQNqlDxmKGNehbURve3M9MKU1JX18zVFVLHUSE81mQgDsAG2HsMCpamsaGOnnmmlp4/gTiEhfocQpn1kojR3tcBhzPbfrjEbzR3yVYJFeanWKIDeaS6RqP4jyH5XxfPPWPIdVAauFUsjwxF4zfe4Zdjb8sL0wZWN0RSTcAAbYtin8usa00syAoOKGAsHueXta2HMVhhfNs4MFPQNS1FPShwVhMRVXe/xam3v9bDpbFFVFm1bVI06mbc3eKVZAlzvqYGwPXc4ENmdbLxFWoqFjOzIkr6bH2v79cFPC+S1NbLLUQTRU0cC+qSQXFiOVh7b/bEkxl88iyJA8c0kak8KOeN2I62AJOMsAnUniMVHIdTgwVyCiqvM1HiBpatGDWpYtNiPnf+e+IPnuRoztR5XLUhr6zIDz7jUdsFUwO9BlSKsqzBEWGplQMVHQ6bjV98SzCGmlqoosqfjpwlszLZ3Y9CovY/0xsk8QVcbFKXK6OnZVDDcMdNr9Ljl0xRP4nziSjWFHRUjHwRIBq9+XQYuzMM/hLK6qgzapkpKadWEKiCtltaMlRrBW4DDUeRF9h156PE+X0cqrXtUQQV5sZ4IZBuw/eUqPr0P6K2S1VNWuj5vUzzrL+xMUkrDhuSLPsRcbG3Q7jGiLw5Q1HiNKRs2jjpJCwSd11nUDstr7A9z8sams7uWNbRxZ8P+YkhhrD6YKgAqs9h8Dk8n/n74X62lnoah4KhSrrzwazLIKrwpmklJXRNU01YoSGoRSsR3FywB2K87c+2NcZpayiFBmgeOK5FNXNCVZN7APcDUpPJ/wAwcQhBJxE6SLWFjzxG98dfAEo5XhXSpdkO2kWNvlfHQxS1AbgAtbmGFrfU7fniN8RIazBXYBviAOzfMYjRbXFBTChkh4cgIPFdQNQ3323t2+mG7KkgpfBSGN7GqJZmC7n1G9uXRR98fOkEkaaI5To/0MNS/Y7Y+g+DqalzTIK6SSiiEtOuhTe9/Te/3B2+mCNUpUUdFxRBUVCU0e5eURl9O3YbnfHLKuqXy8wdFHMfvC/O3TvjFMF4sqr8Gvlaw29rn+/tiKgKbrsR1GGM1qaRiUcOV0kWN+R98e+ZdKg641ViRdlxRT1DU6ujRCaNt7X3B/XE+PSCGP8AZTNIZLyq0Oyr7MJN/sPyxHBSZS7pLHrifk+gE3H0/v8APEGqPJvFJTwhXVSole+mx5owFtj2PfAd5X4jeXV0jvdQz2P2AOI8Wqv6XVQOlzy7dMWYt2nnMfHEs/h4ZXUQRywstmM6F2W3JQwYcjuG58vqs1OeZhKlCxZf+Uplp0G+hkW+5UmxJvvgaaio06ZArqeg3t9CcTR4HYIYmueRjNifof0xrexnT//Z"
}
]
}
Output
News results
Title: Trump Promotes Oil Deal That May Not Exist
Snippet: WASHINGTON — When oil prices crashed in early March after a dispute between
Russia and Saudi Arabia, President Trump put a positive spin on the news.
Date: 15 hours ago
Title: Trump’s Oil Summit
Snippet: Trump's Oil Summit. Tariffs and quotas won't solve a price shock caused by
a pandemic and a Saudi Arabia-Russia feud.
Date: Opinion · 16 hours ago
Title: OPEC and allies reportedly set for video meeting as analysts pour
skepticism on Trump's intervention
Snippet: “Donald Trump's tweet … It's nonsense, really,” Patrick Armstrong, chief
investment officer at Plurimi Investment Managers, told CNBC's “Squawk Box
Europe” on ...
Date: 5 hours ago
Title: Trump again tests negative for coronavirus
Snippet: President Donald Trump on Thursday again tested negative for the
coronavirus after being tested by the White House physician, according to
two White House ...
Date: 17 hours ago
Title: Trump passes the buck as deadly ventilator shortage looms
Snippet: (CNN) President Donald Trump is pinning the blame on states for a shortage
of ventilators that governors warn could effectively condemn coronavirus
patients to ...
Date: 10 hours ago
If you want more information, check out SerpApi documentation or live playground.
Disclosure: I work for SerpApi.
hi you can scrap like this with easy way
from bs4 import BeautifulSoup
import requests
url="https://news.google.co.in/"
code=requests.get(url)
soup=BeautifulSoup(code.text,'html5lib')
for title in soup.find_all('span',class_="titletext"):
print title.text