最近学爬虫，照着学长爬中国大学排名写了个豆瓣读书小说书名的爬虫，没多爬，600本书，主要是熟悉熟悉bs4
- Python版本：3.6
- 所需库：
requests
bs4
time
xlwt
首先用request获取目标页面源代码
def get_page(url):
headers = {
'Uesr-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
try:
res = requests.get(url, headers)
# 访问失败则抛出异常
res.raise_for_status()
res.encoding = res.apparent_encoding
return res.text
except:
return ""
第二步用美丽汤解析html，我用的解析器是lxml
def fillBooklist(blist, html):
soup = BeautifulSoup(html, 'lxml')
for li in soup.find("ul", "subject-list").children:
if isinstance(li, bs4.element.Tag):
div = li('div')[1] # 定位到书籍信息
book = div.find('h2').a['title'] # 定位书名
# print(book)
pub = div.find('div').string.strip() # 定位出版信息，去除多余空格回车
# print(pub)
blist.append([book, pub])
获得书籍后填入excel，比较懒，没设置太多样式
def fillBookXls(blist):
# 创建excel
# 太乱了，但是懒得写多个函数设置样式嘻嘻嘻
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('BookSheet')
worksheet.col(0).width = 8000
worksheet.col(1).width = 20000
style1 = xlwt.XFStyle()
style2 = xlwt.XFStyle()
font = xlwt.Font()
font.name = "宋体"
font.height = 13 * 20
alignment1 = xlwt.Alignment()
alignment2 = xlwt.Alignment()
alignment1.horz = xlwt.Alignment.HORZ_CENTER
alignment1.vert = xlwt.Alignment.VERT_CENTER
alignment2.horz = xlwt.Alignment.HORZ_LEFT
alignment2.vert = xlwt.Alignment.VERT_CENTER
style1.font = font
style1.alignment = alignment1
style2.font = font
style2.alignment = alignment2
# 写入excel
worksheet.write(0, 0, "书名", style1)
worksheet.write(0, 1, "出版信息", style1)
for i in range(len(blist)):
worksheet.write(i + 1, 0, blist[i][0], style1)
worksheet.write(i + 1, 1, blist[i][1], style2)
workbook.save('小说.xls')
完整代码
import requests
from bs4 import BeautifulSoup
import bs4
import xlwt
import time
def get_page(url):
headers = {
'Uesr-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
try:
res = requests.get(url, headers)
# 访问失败则抛出异常
res.raise_for_status()
res.encoding = res.apparent_encoding
return res.text
except:
return ""
def fillBooklist(blist, html):
soup = BeautifulSoup(html, 'lxml')
for li in soup.find("ul", "subject-list").children:
if isinstance(li, bs4.element.Tag):
div = li('div')[1] # 定位到书籍信息
book = div.find('h2').a['title'] # 定位书名
# print(book)
pub = div.find('div').string.strip() # 定位出版信息
# print(pub)
blist.append([book, pub])
def fillBookXls(blist):
# 创建excel
# 太乱了，但是懒得写多个函数设置样式嘻嘻嘻
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('BookSheet')
worksheet.col(0).width = 8000
worksheet.col(1).width = 20000
style1 = xlwt.XFStyle()
style2 = xlwt.XFStyle()
font = xlwt.Font()
font.name = "宋体"
font.height = 13 * 20
alignment1 = xlwt.Alignment()
alignment2 = xlwt.Alignment()
alignment1.horz = xlwt.Alignment.HORZ_CENTER
alignment1.vert = xlwt.Alignment.VERT_CENTER
alignment2.horz = xlwt.Alignment.HORZ_LEFT
alignment2.vert = xlwt.Alignment.VERT_CENTER
style1.font = font
style1.alignment = alignment1
style2.font = font
style2.alignment = alignment2
# 写入excel
worksheet.write(0, 0, "书名", style1)
worksheet.write(0, 1, "出版信息", style1)
for i in range(len(blist)):
worksheet.write(i + 1, 0, blist[i][0], style1)
worksheet.write(i + 1, 1, blist[i][1], style2)
workbook.save('小说.xls')
if __name__ == '__main__':
blist = []
for i in range(30):
page = i * 20
url = "https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4" + "?start=" + str(page) + "&type=T"
html = get_page(url)
fillBooklist(blist, html)
fillBookXls(blist)
time.sleep(2) # 慢点爬，怕被封ip
效果
部分书名
