参考文档：
python selenium实现智联招聘数据爬取python脚本之家 (jb51.net)
安装selenium 和chromeDriver文档：
selenium 安装与 chromedriver安装 - Rogn - 博客园 (cnblogs.com)
注意：
· 因为页面存在动态加载，通过调整页面大小规避
· 智联页面无法自动输入账号密码，会被判断用户名错误，所以要手动输入，前提保证睡眠时间够用，可适当加长
· 此代码需要在个人电脑调试一下，并不是万能的，cookie ，页面大小，驱动地址都需要更改调试
代码：
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 11 19:14:00 2021
@author:
"""
from urllib.request import quote, unquote
import csv
import os
import re
import json
import time
import requests
from selenium.webdriver import Chrome
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.common.by import By
from selenium.webdriver.support import ui
from selenium.webdriver.support import expected_conditions
from lxml import etree
#驱动地址
chrome =
Chrome(executable_path=r'C:\Users\XX\AppData\Local\Google\Chrome\Application\chromedriver.exe')
# 因为页面存在动态加载，适当更改页面比例，规避动态显示
chrome.maximize_window()
chrome.execute_script("document.body.style.zoom='0.75'") #缩小
# 模拟登录
def login(url):
# 获取登录页面
chrome.get(url)
##下面代码自动输入，会判断密码错误，需要手动输入，点击登录，滑块
##如果时间不够可以令睡眠时间长一点
time.sleep(20)
get_allcity('https://www.zhaopin.com/citymap')
# 在登录状态下进行所有城市信息的获取
def get_allcity(url):
resp = requests.get(url, headers=headers)
if resp.status_code == 200:
html = resp.text
json_data = re.search(r'<script>__INITIAL_STATE__=(.*?)</script>', html).groups()[0]
data = json.loads(json_data)
cityMapList = data['cityList']['cityMapList'] # dict
for letter, citys in cityMapList.items():
for city in citys: # citys 是个列表，里面嵌套的字典
city_name = city['name']
city_url = 'https:' + city['url']
# 筛选城市
query_citys = ('成都')
if city_name in query_citys:
print(f'正在获取{city_name}的信息')
get_city_job(city_url)
time.sleep(3)
else:
# print(f'{city_name} 不在搜索范围内！')
pass
else:
print('网页获取失败')
def get_city_job(url):
chrome.get(url) # 打开城市信息
# 根据class_name 查询WebElement找出输入的位置
input_seek: WebElement = chrome.find_element_by_class_name('zp-search__input')
#输入查询岗位
input_seek.send_keys('Python')
# 找出搜索按钮并点击
click: WebElement =
chrome.find_element_by_xpath('//div[@class="zp-search__common"]//a')
click.click()
# 切换到第二个页面
chrome.switch_to.window(chrome.window_handles[1])
time.sleep(1)
time.sleep(1)
# 等待class_name为“sou-main__list” div元素出现
ui.WebDriverWait(chrome, 30).until(
expected_conditions.visibility_of_all_elements_located((By.CLASS_NAME, 'sou-main__list')),
'查找的元素一直没有出现'
)
# 判断当前查询结果是否不存在
no_content = chrome.find_elements_by_class_name('positionlist')
if not no_content:
print('当前城市未查找到Python岗位')
else:
# 提取查找结果
parse(chrome.page_source)
def parse(html):
root = etree.HTML(html)
divs = root.xpath('//div[@class="positionlist"]') # element对象
items = {}
for div in divs:
# 岗位
position = div.xpath('.//a//div[@class="iteminfo__line1__jobname"]/span[1]')
# 公司
company = div.xpath('//a//div[@class="iteminfo__line1__compname"]/span/text()')
# 薪资
money = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/p/text()')
# 位置
city = div.xpath('//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[1]/text()')
# 经验
experience = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[2]/text()')
# 学历
education = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[3]/text()')
# 规模
scale = div.xpath('.//a//div[@class="iteminfo__line2__compdesc"]/span[1]/text()')
# 人数
people = div.xpath('.//a//div[@class="iteminfo__line2__compdesc"]/span[2]/text()')
for position_, company_, money_, city_, experience_, education_, scale_, people_ in zip(position, company,
money, city, experience,
education, scale,
people):
# title="python爬虫工程师" 获取它的title属性值
string = position_.attrib.get('title')
items['position'] = string
items['company'] = company_
items['money'] = money_.strip()
items['city'] = city_
items['experience'] = experience_
items['education'] = education_
items['scale'] = scale_
items['people'] = people_
itempipeline(items)
#print(items)
# 获取下一页
next_page()
def itempipeline(items):
has_header = os.path.exists(save_csv) # 文件头
with open(save_csv, 'a', encoding='utf-8-sig') as file:
writer = csv.DictWriter(file, fieldnames=items.keys())
if not has_header:
writer.writeheader() # 写入文件头
writer.writerow(items)
def next_page():
# 找到下一页按钮
time.sleep(1)
# 页面有动态加载无法直接从顶部开始拉滚动条，所以选择底部
js="window.scrollTo(document.body.scollHeight,100)"
chrome.execute_script(js)
#找到下一页按钮
button = chrome.find_elements_by_xpath('/div[@class="soupager"]/button[@class="btn soupager__btn"]')
if not button:
print(f'获取完毕，请在 {save_csv} 里查看!!')
exit()
else:
time.sleep(1)
button[0].click() # 点击下一页
#chrome.execute_script("arguments[0].click();", button)
time.sleep(3)
parse(chrome.page_source)
if __name__ == '__main__':
n = 0
####cookies 是网址“https://www.zhaopin.com/citymap” 这一页的
cookies ='"x-zp-client-id=f1e50b3c-66e1-4026-b065-e91d60707ea6; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross={"distinct_id":"17c706bcc1754f-0f0b96a673e2808-b7a1438-1024000-17c706bcc18593","first_id":"","props":{"$latest_traffic_source_type":"直接流量","$latest_search_keyword":"未取到值_直接打开","$latest_referrer":""},"$device_id":"17c706bcc1754f-0f0b96a673e2808-b7a1438-1024000-17c706bcc18593"}; zp_passport_deepknow_sessionId=bd94ba2bs00e1943148a588024abfb073d03; acw_tc=2760826216339736981194712e27e6f8af94db9c58e502ce71ac5d5b427b8e; sts_deviceid=17c706bfdd1365-045fa83fea5838-b7a1438-1024000-17c706bfdd218; LastCity_id=801; LastCity=成都; sts_sg=1; sts_sid=17c706bfedc683-071b312edb9e43-b7a1438-1024000-17c706bfedd817; sts_chnlsid=Unknown; ZL_REPORT_GLOBAL={"//www":{"seid":"","actionid":"932b2668-11af-450d-a229-f10072031ebc-cityPage"}}; Hm_lvt_363368edd8b243d3ad4afde198719c4a=1633973699; 1420ba6bb40c9512e9642a1f8c243891=5930d939-a8aa-4835-af18-1b800a782a12; locationInfo_search={"code":"606","name":"营口","message":"匹配到市级编码"}; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1633973701; Hm_lpvt_38ba284938d5eddca645bb5e02a02006=1633973701; Hm_lpvt_363368edd8b243d3ad4afde198719c4a=1633973947; sts_evtseq=7"'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
'Cookie': cookies.encode('utf-8').decode('latin-1'),
'content-type':'text/html; charset=utf-8'
}
save_csv = 'chengdu-python.csv'
login(
"https://passport.zhaopin.com/login?bkUrl=%2F%2Fi.zhaopin.com%2Fblank%3Fhttps%3A%2F%2Fsou.zhaopin.com%2F%3Fjl%3D801%26kw%3DPython%26kt%3D3%26srccode%3D")
结果
结果
