爬虫之 beautifulsoup
Beautiful Soup 3目前已经停止开发，推荐现在的项目使用Beautiful Soup
利用beautiful 爬取
import requests
from bs4 import BeautifulSoup
import json
'''发送请求,获取响应的内容'''
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
}
base_url = 'https://hr.tencent.com/position.php'
keywords = input('输入职位：')
begin_page = int(input('起始页：'))
end_page = int(input('结束页：'))
job_list = []
for page in range(begin_page, end_page + 1):
params = {
'keywords': keywords,
'start': (page - 1) * 10
}
print('%s爬取中...' % page)
response1 = requests.get(url=base_url, params=params,headers=headers)
content = response1.content
# with open('./tencent-%s.html'%page, 'wb') as file:
# file.write(content)
content = content.decode('utf-8')
'''数据提取'''
bs = BeautifulSoup(content,'lxml')
# tr_list = bs.select('tr[class="odd"],tr[class="even"]')
tr_list = bs.find_all(name='tr',attrs={'class':['even','odd']})
for tr in tr_list:
job={}
job['job_name'] = tr.a.text.strip()
job['job_href'] = tr.a['href']
job['job_type'] = tr.find_all('td')[1].text.strip()
job['job_person'] = tr.find_all('td')[2].text.strip()
job['job_address'] = tr.find_all('td')[3].text.strip()
job['job_time'] = tr.find_all('td')[4].text.strip()
job_list.append(job)
#转成json ensure_asci=False----默认是True,改成False,才能显示中文， ensure_ascii=False 来禁用ascii编码
#dump 和 dumps两种写法，
# job_json_string = json.dumps(job_list,ensure_ascii=False)
# with open('./tencent.json', 'w',encoding='utf-8') as file:
# file.write(job_json_string)
json.dump(job_list,open('./tencent.json', 'w',encoding='utf-8'),ensure_ascii=False)
