这几天在学爬虫，就想着尝试爬取一下网站，一下是爬取51job的过程
分析网站
image.png
这是首页，大家可以尝试各种方法抓取接口。
查看变化，尝试直接在get头改变信息
[图片上传中...(image.png-c093dd-1589098280977-0)]
image.png
image.png
由此看到，搜索接口
在查看网页第一，二页
image.png
由此，可以看到页数接口
获取信息
比如职位信息：由于我最近在学xpath，所以用xpath来定点
image.png
右键复制xpath，就可以拿到想要的节点了，但需要注意的是，要循环的话就不能用原始的xpath，可以自行改一下。
献上代码
image.png
不多bb，直接上代码
import random
from lxml import etree
import requests
import re
import pymysql
# 使用随机agent
def rand_user():
ua_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
"Opera/8.0 (Windows NT 5.1; U; en)",
"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 ",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
]
ua = random.choice(ua_list)
headers = {
"User-Agent": ua
}
return headers
# 使用代理ip
def proxy():
proxy_list = [
{
"HTTP": "92.247.142.14:53281",
},
{
"HTTP": "203.150.160.157:8080",
},
{
"HTTP": "91.211.107.204:41258",
}
]
pr = random.choice(proxy_list)
return pr
# 获取岗位数据
def get_detail():
work = input("你想要的职位")
page = 1
url = "https://search.51job.com/list/000000,000000,0000,00,9,99,{},2,{}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=".format(work,page)
response = requests.get(url, headers=rand_user(), proxies=proxy())
content = response.content.decode('gbk')
# 获取最后一页
mytree = etree.HTML(content)
job_page = mytree.xpath('//*[@id="resultList"]/div[55]/div/div/div/span[1]/text()')
job_page_str = str(job_page)
num=re.findall(r'\d+', job_page_str)
num1 = int(num[0])
# print(num1)
# 翻页爬
for i in range(1, num1+1):
url += "https://search.51job.com/list/000000,000000,0000,00,9,99,{},2,{}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=".format(work,i)
try:
# 先获取页面数据
response = requests.get(url, headers=rand_user(), proxies=proxy())
content = response.content.decode('gbk')
# 使用xpath
mytree = etree.HTML(content)
job_list = mytree.xpath('//*[@id="resultList"]/div[@class="el"]')
# print(job_list)
# 获取当前页的相关数据
for job in job_list:
job_name = job.xpath('./p/span/a/text()')[0]
# print(job_name)
# 公司
job_company = job.xpath('./span/a/text()')[0] # //*[@id="resultList"]/div[5]/span[1]/a
# print(job_company)
# 城市
job_city = job.xpath('./span/text()')[0]
# print(job_city)
# 薪资
job_salary = job.xpath('./span/text()')[1]
# print(job_salary)
job_salary1 = job.xpath('./span[@class="t5"]/text()')[0]
# print(job_salary1)
job_url = job.xpath('./p/span/a/@href')[0]
# print(job_url)
if job_salary != job_salary1:
pass
else:
job_salary = "面议"
str_job_name = job_name.strip()
str_job_company=job_company.strip()
str_job_city=job_city.strip()
str_job_salary=job_salary.strip()
str_job_url=job_url.strip()
# 存入数据库
sql = 'insert into job_table values(null,"%s", "%s", "%s", "%s", "%s")'%(str_job_name, str_job_company, str_job_city, str_job_salary,str_job_url)
cursor.execute(sql)
db.commit()
except Exception as e:
pass
print("*" * 40)
print("%s 第%d页 爬取异常" % (url, i))
print(e)
print("*" * 40)
if __name__ == '__main__':
# 连接MySQL
db = pymysql.connect(
host="localhost",
port=3306,
user="root",
password="123123",
database="job",
charset="utf8"
)
cursor = db.cursor()
get_detail()
cursor.close()
db.close()
image.png
image.png
image.png
到此结束，希望大家新手发帖，大佬勿喷，没加多线程的原因是不想改代码了。
