做爬虫的同学应该经常遇到登录验证的问题，在这里我使用phantomejs模拟登录并且保存cookies，方便再次登录。废话不多说，直接切入正题。
-
模拟登录
def login(username, password, driver=None):
url = 'https://www.jd.com/productDetail.html'
driver.get(url)
driver.find_element_by_xpath('//input[@id="loginname"]').send_keys(username)
logger.info ('已输入账号')
driver.find_element_by_xpath('//input[@id="nloginpwd"]').send_keys(password)
logger.info ('已输入密码')
driver.find_element_by_xpath('//a[@id="loginsubmit"]').click()
logger.info ('已点击登录按钮')
return driver
-
从网上获取cookies
def get_cookies_from_network():
driver = login(username, password, driver)
# 获取cookies保存到本地并且返回此cookies
cookies = driver.get_cookies()
cookie_dict = {}
logger.info(cookies)
for cookie in cookies:
if 'name' in cookie.keys() and 'value' in cookie.keys():
cookie_dict[cookie['name']] = cookie['value']
logger.info(cookie_dict)
with open('cookies.txt', 'w') as f:
# 保存cookies到本地
f.write(json.dumps(cookies))
logger.info("保存cookis成功.")
return cookie_dict
-
从本地缓存获取cookies
def get_cookies_from_cache():
cookies_dict = {}
if os.path.exists('cookies.txt'):
# 如果本地有cookies文件，则读取本地cookies，否则返回空
with open('cookies.txt', 'r') as f:
for i in json.loads(f.read()):
if 'name' in i.keys() and 'value' in i.keys():
if 'expiry' in i.keys():
expiry_date = int(i['expiry'])
# 判断cookies是否过期，过期则返回空
if expiry_date > (int)(time.time()):
cookies_dict[i['name']] = i['value']
else:
logger.info('过期啦：{}'.format(i['name']))
return {}
else:
cookies_dict[i['name']] = i['value']
else:
return cookies_dict
logger.info(cookies_dict)
return cookies_dict
-
获取cookies时，优先访问本地的cookies，如果cookies过期则再次获取。
def get_cookies():
# 先从本地获取cookies
cookie_dict = get_cookies_from_cache()
if not cookie_dict:
# 从本地返回的cookies为空则从网上获取cookies
cookie_dict = self.get_cookies_from_network()
return cookie_dict
-
带着cookies登录网站
def get_content(driver=None):
#获取cookies
cookies = get_cookies()
# 先访问一遍目标网站
driver.get(url)
# 删除当前的cookies
driver.delete_all_cookies()
for k,v in cookies.items():
# 添加cookies
driver.add_cookie({'name': k, 'value': v})
time.sleep(1)
logger.info('get url again')
# 再次访问目标网站，模拟登录成功
driver.get(self.url)
logger.info(driver.page_source)
