Unable to log in to ASP.NET website with requests module of Python
I was initially using requests+bs4 as well however I was running into similar issues with the ASPX site I'm scrapping. I found another library called robobrowser that wraps requests+bs4. With this you no longer have to manually set items such as "__VIEWSTATE" and friends when interacting with ASPX sites.
from robobrowser import RoboBrowser
url = ' http://www11.davidsonsinc.com'
login_url = url + '/Login/Login.aspx'
username = "username"
password = "password"
browser = RoboBrowser(history=True)
# This retrieves __VIEWSTATE and friends
browser.open(login_url)
signin = browser.get_form(id='aspnetForm')
signin["ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$UserName"].value = username
signin["ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$Password"].value = password
signin["ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$LoginButton"].value = "Log In"
browser.submit_form(signin)
print browser.url
import requests
from bs4 import BeautifulSoup
URL="http://www11.davidsonsinc.com/Login/Login.aspx"
headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36"}
username="username"
password="password"
s=requests.Session()
s.headers.update(headers)
r=s.get(URL)
soup=BeautifulSoup(r.content)
VIEWSTATE=soup.find(id="__VIEWSTATE")['value']
VIEWSTATEGENERATOR=soup.find(id="__VIEWSTATEGENERATOR")['value']
EVENTVALIDATION=soup.find(id="__EVENTVALIDATION")['value']
login_data={"__VIEWSTATE":VIEWSTATE,
"__VIEWSTATEGENERATOR":VIEWSTATEGENERATOR,
"__EVENTVALIDATION":EVENTVALIDATION,
"ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$UserName":username,
"ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$Password":password,
"ctl00$ContentPlaceHolderNavPane$LeftSection$UserLogin$LoginButton":"Log In"}
r=s.post(URL, data=login_data)
print r.url