简书首页爬取的关键是分页和找准入口。数据应该是有15页，试了好久请求添加了id，最后还是只爬取了20条数据。
代码
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
import xlwt
import sys
reload(sys)
sys.setdefaultencoding('utf8')
#获取源码
def getHtml(url):
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',}
html = requests.get(url,headers=headers).content
#print html
return html
def getDate(url):
html = getHtml(url)
soup = BeautifulSoup(html, 'html.parser')
#先找到整体文章列表
datas = soup.find('ul',class_='note-list').find_all('li')
results = []
for data in datas:
author = data.find('a', class_="blue-link").text
title = data.find('a', class_="title").text
pubtime = data.find('span',class_='time').get('data-shared-at')
abstract = data.find('p',class_='abstract').text
meta = data.find('div',class_='meta')
#判断文章是否收入专题
if meta.find('a',class_="collection-tag"):
collect_tag = meta.find_all('a',target="_blank")[0].text
read_count = meta.find_all('a',target="_blank")[1].text
comment_count = meta.find_all('a',target="_blank")[2].text
like_count = meta.find_all('span')[0].text
#判断有没有打赏
if meta.find('i',class_="iconfont ic-list-money"):
money_count = meta.find_all('span')[1].text
else:money_count=0
else:
collect_tag = u'这篇文章还没有收入任何专题~~~'
read_count = meta.find_all('a', target="_blank")[0].text
comment_count = meta.find_all('a', target="_blank")[1].text
like_count = meta.find_all('span')[0].text
if meta.find('i', class_="iconfont ic-list-money"):
money_count = meta.find_all('span')[1].text
else:
money_count = 0
result = [title,author,pubtime,abstract,read_count,comment_count,like_count,money_count,collect_tag]
print title,author,pubtime,abstract,read_count,comment_count,like_count,money_count,collect_tag
results.append(result)
return results
# a = getDate(url)
#写入excel
def excel_write(results):
newTable = 'jianshu.xls'
wb = xlwt.Workbook(encoding='utf-8')
ws = wb.add_sheet('HomePage')
headData = ['标题', '作者', '发表时间', '摘要', '阅读量','评论数','点赞数','打赏数','收录专题']
for colnum in range(0, 9):
ws.write(0, colnum, headData[colnum], xlwt.easyxf('font: bold on')) # 行 列 数据
index = 1
for result in results:
for i in range(0, 9):
print result[i]
ws.write(index, i, result[i])
index += 1
wb.save(newTable)
if __name__ == "__main__":
baseUrl = 'http://www.jianshu.com/'
data = '?page=15&seen_snote_ids%5B%5D=12658273&seen_snote_ids%5B%5D=12689156&seen_snote_ids%5B%5D=12681559&seen_snote_ids%5B%5D=12484933&seen_snote_ids%5B%5D=12700981&seen_snote_ids%5B%5D=12691138&seen_snote_ids%5B%5D=12692643&seen_snote_ids%5B%5D=12658708&seen_snote_ids%5B%5D=12688277&seen_snote_ids%5B%5D=12704483&seen_snote_ids%5B%5D=12595741&seen_snote_ids%5B%5D=12631433&seen_snote_ids%5B%5D=12640738&seen_snote_ids%5B%5D=12624548&seen_snote_ids%5B%5D=12619420&seen_snote_ids%5B%5D=12658768&seen_snote_ids%5B%5D=12658377&seen_snote_ids%5B%5D=12540326&seen_snote_ids%5B%5D=12656357&seen_snote_ids%5B%5D=12518815&seen_snote_ids%5B%5D=12711009&seen_snote_ids%5B%5D=12338074&seen_snote_ids%5B%5D=12698315&seen_snote_ids%5B%5D=12702095&seen_snote_ids%5B%5D=12695554&seen_snote_ids%5B%5D=12665024&seen_snote_ids%5B%5D=12700059&seen_snote_ids%5B%5D=12702828&seen_snote_ids%5B%5D=12658142&seen_snote_ids%5B%5D=12670518&seen_snote_ids%5B%5D=12679524&seen_snote_ids%5B%5D=12681445&seen_snote_ids%5B%5D=12660821&seen_snote_ids%5B%5D=12661012&seen_snote_ids%5B%5D=12695747&seen_snote_ids%5B%5D=12663860&seen_snote_ids%5B%5D=12679518&seen_snote_ids%5B%5D=12582752&seen_snote_ids%5B%5D=12643448&seen_snote_ids%5B%5D=12648303&seen_snote_ids%5B%5D=12706868&seen_snote_ids%5B%5D=12599997&seen_snote_ids%5B%5D=12684922&seen_snote_ids%5B%5D=12669401&seen_snote_ids%5B%5D=12680783&seen_snote_ids%5B%5D=12523099&seen_snote_ids%5B%5D=12564012&seen_snote_ids%5B%5D=12686358&seen_snote_ids%5B%5D=12650691&seen_snote_ids%5B%5D=12663630&seen_snote_ids%5B%5D=12608069&seen_snote_ids%5B%5D=12654093&seen_snote_ids%5B%5D=12300224&seen_snote_ids%5B%5D=12682055&seen_snote_ids%5B%5D=12663246&seen_snote_ids%5B%5D=12643880&seen_snote_ids%5B%5D=12648040&seen_snote_ids%5B%5D=12642421&seen_snote_ids%5B%5D=12636423&seen_snote_ids%5B%5D=12628359&seen_snote_ids%5B%5D=12623170&seen_snote_ids%5B%5D=12636074&seen_snote_ids%5B%5D=12665569&seen_snote_ids%5B%5D=12703498&seen_snote_ids%5B%5D=12685253&seen_snote_ids%5B%5D=12701187&seen_snote_ids%5B%5D=12703984&seen_snote_ids%5B%5D=12634364&seen_snote_ids%5B%5D=12602649&seen_snote_ids%5B%5D=12218345&seen_snote_ids%5B%5D=12492535&seen_snote_ids%5B%5D=12660271&seen_snote_ids%5B%5D=12691473&seen_snote_ids%5B%5D=12687211&seen_snote_ids%5B%5D=12699375&seen_snote_ids%5B%5D=12703102&seen_snote_ids%5B%5D=12657872&seen_snote_ids%5B%5D=12659742&seen_snote_ids%5B%5D=12604652&seen_snote_ids%5B%5D=11166785&seen_snote_ids%5B%5D=12657143&seen_snote_ids%5B%5D=12662330&seen_snote_ids%5B%5D=12640661&seen_snote_ids%5B%5D=12708110&seen_snote_ids%5B%5D=12657911&seen_snote_ids%5B%5D=12564405&seen_snote_ids%5B%5D=12632846&seen_snote_ids%5B%5D=12580583&seen_snote_ids%5B%5D=12623918&seen_snote_ids%5B%5D=12647490&seen_snote_ids%5B%5D=12662155&seen_snote_ids%5B%5D=12684822&seen_snote_ids%5B%5D=12683266&seen_snote_ids%5B%5D=12662006&seen_snote_ids%5B%5D=12694356&seen_snote_ids%5B%5D=12663263&seen_snote_ids%5B%5D=12659372&seen_snote_ids%5B%5D=12565648&seen_snote_ids%5B%5D=12617037&seen_snote_ids%5B%5D=12612930&seen_snote_ids%5B%5D=12563831&seen_snote_ids%5B%5D=11777918&seen_snote_ids%5B%5D=12694557&seen_snote_ids%5B%5D=12703379&seen_snote_ids%5B%5D=12609864&seen_snote_ids%5B%5D=12606620&seen_snote_ids%5B%5D=12687239&seen_snote_ids%5B%5D=12642303&seen_snote_ids%5B%5D=12691171&seen_snote_ids%5B%5D=7663876&seen_snote_ids%5B%5D=12688009&seen_snote_ids%5B%5D=12642396&seen_snote_ids%5B%5D=12607554&seen_snote_ids%5B%5D=12669692&seen_snote_ids%5B%5D=12696410&seen_snote_ids%5B%5D=12677520&seen_snote_ids%5B%5D=12584716&seen_snote_ids%5B%5D=12660054&seen_snote_ids%5B%5D=12530825&seen_snote_ids%5B%5D=12681038&seen_snote_ids%5B%5D=12704140&seen_snote_ids%5B%5D=12686932&seen_snote_ids%5B%5D=12630368&seen_snote_ids%5B%5D=12695703&seen_snote_ids%5B%5D=12582230&seen_snote_ids%5B%5D=12689421&seen_snote_ids%5B%5D=12663769&seen_snote_ids%5B%5D=11820725&seen_snote_ids%5B%5D=12520519&seen_snote_ids%5B%5D=12672594&seen_snote_ids%5B%5D=12656074&seen_snote_ids%5B%5D=12512244&seen_snote_ids%5B%5D=12699112&seen_snote_ids%5B%5D=12656727&seen_snote_ids%5B%5D=12698991&seen_snote_ids%5B%5D=12688923&seen_snote_ids%5B%5D=12544436&seen_snote_ids%5B%5D=12656603&seen_snote_ids%5B%5D=12530052&seen_snote_ids%5B%5D=12702798&seen_snote_ids%5B%5D=12687284&seen_snote_ids%5B%5D=12448888&seen_snote_ids%5B%5D=12663782&seen_snote_ids%5B%5D=12606930&seen_snote_ids%5B%5D=12588667&seen_snote_ids%5B%5D=12665002&seen_snote_ids%5B%5D=10688540&seen_snote_ids%5B%5D=12585769&seen_snote_ids%5B%5D=12668881&seen_snote_ids%5B%5D=12677358&seen_snote_ids%5B%5D=12676116&seen_snote_ids%5B%5D=12700404&seen_snote_ids%5B%5D=12661935&seen_snote_ids%5B%5D=12707133&seen_snote_ids%5B%5D=12656134&seen_snote_ids%5B%5D=12478683&seen_snote_ids%5B%5D=12709169&seen_snote_ids%5B%5D=12679225&seen_snote_ids%5B%5D=12663030&seen_snote_ids%5B%5D=12698974&seen_snote_ids%5B%5D=12689032&seen_snote_ids%5B%5D=12680340&seen_snote_ids%5B%5D=12693738&seen_snote_ids%5B%5D=12679164&seen_snote_ids%5B%5D=12637667&seen_snote_ids%5B%5D=12699423&seen_snote_ids%5B%5D=12699685&seen_snote_ids%5B%5D=9199031&seen_snote_ids%5B%5D=12638401&seen_snote_ids%5B%5D=12643079&seen_snote_ids%5B%5D=12626989&seen_snote_ids%5B%5D=12649754&seen_snote_ids%5B%5D=12426104&seen_snote_ids%5B%5D=12602622&seen_snote_ids%5B%5D=12666369&seen_snote_ids%5B%5D=12596530&seen_snote_ids%5B%5D=12641942&seen_snote_ids%5B%5D=12604251&seen_snote_ids%5B%5D=12597437&seen_snote_ids%5B%5D=12663298&seen_snote_ids%5B%5D=12677017&seen_snote_ids%5B%5D=12653491&seen_snote_ids%5B%5D=12677222&seen_snote_ids%5B%5D=12698017&seen_snote_ids%5B%5D=12658191&seen_snote_ids%5B%5D=12626163&seen_snote_ids%5B%5D=12627683&seen_snote_ids%5B%5D=12554040&seen_snote_ids%5B%5D=12627495&seen_snote_ids%5B%5D=12542947&seen_snote_ids%5B%5D=12638970&seen_snote_ids%5B%5D=12650928&seen_snote_ids%5B%5D=12658392&seen_snote_ids%5B%5D=12659878&seen_snote_ids%5B%5D=12612503&seen_snote_ids%5B%5D=12666856&seen_snote_ids%5B%5D=12694462&seen_snote_ids%5B%5D=12658333&seen_snote_ids%5B%5D=12631006&seen_snote_ids%5B%5D=12618207&seen_snote_ids%5B%5D=11935675&seen_snote_ids%5B%5D=12606275&seen_snote_ids%5B%5D=12630696&seen_snote_ids%5B%5D=12643216&seen_snote_ids%5B%5D=12593022&seen_snote_ids%5B%5D=12625921&seen_snote_ids%5B%5D=12602244&seen_snote_ids%5B%5D=12695755&seen_snote_ids%5B%5D=12609261&seen_snote_ids%5B%5D=12666981&seen_snote_ids%5B%5D=12700237&seen_snote_ids%5B%5D=11904387&seen_snote_ids%5B%5D=12599567&seen_snote_ids%5B%5D=12576413&seen_snote_ids%5B%5D=12565193&seen_snote_ids%5B%5D=12599881&seen_snote_ids%5B%5D=12511241&seen_snote_ids%5B%5D=12576991&seen_snote_ids%5B%5D=12514019&seen_snote_ids%5B%5D=12657719&seen_snote_ids%5B%5D=12533063&seen_snote_ids%5B%5D=12518230&seen_snote_ids%5B%5D=12563828&seen_snote_ids%5B%5D=12494682&seen_snote_ids%5B%5D=12649202&seen_snote_ids%5B%5D=12687940&seen_snote_ids%5B%5D=12492719&seen_snote_ids%5B%5D=12486128&seen_snote_ids%5B%5D=12619419&seen_snote_ids%5B%5D=12677114&seen_snote_ids%5B%5D=12687988&seen_snote_ids%5B%5D=12398046&seen_snote_ids%5B%5D=12639473&seen_snote_ids%5B%5D=12603532&seen_snote_ids%5B%5D=12632823&seen_snote_ids%5B%5D=12694209&seen_snote_ids%5B%5D=12610549&seen_snote_ids%5B%5D=12664416&seen_snote_ids%5B%5D=12600293&seen_snote_ids%5B%5D=12613945&seen_snote_ids%5B%5D=12642344&seen_snote_ids%5B%5D=12689271&seen_snote_ids%5B%5D=12694358&seen_snote_ids%5B%5D=12666521&seen_snote_ids%5B%5D=12473076&seen_snote_ids%5B%5D=12686633&seen_snote_ids%5B%5D=12517536&seen_snote_ids%5B%5D=12674926&seen_snote_ids%5B%5D=12684992&seen_snote_ids%5B%5D=12668691&seen_snote_ids%5B%5D=12606078&seen_snote_ids%5B%5D=12634192&seen_snote_ids%5B%5D=12623365&seen_snote_ids%5B%5D=12423597&seen_snote_ids%5B%5D=12616932&seen_snote_ids%5B%5D=12598725&seen_snote_ids%5B%5D=12624968&seen_snote_ids%5B%5D=12623323&seen_snote_ids%5B%5D=12580963&seen_snote_ids%5B%5D=12620773&seen_snote_ids%5B%5D=12471552&seen_snote_ids%5B%5D=12666934&seen_snote_ids%5B%5D=12148754&seen_snote_ids%5B%5D=12678642&seen_snote_ids%5B%5D=12569617&seen_snote_ids%5B%5D=12560935&seen_snote_ids%5B%5D=12680300&seen_snote_ids%5B%5D=12545244&seen_snote_ids%5B%5D=8286846&seen_snote_ids%5B%5D=12523208&seen_snote_ids%5B%5D=12684296&seen_snote_ids%5B%5D=12662715&seen_snote_ids%5B%5D=12507741&seen_snote_ids%5B%5D=12677811&seen_snote_ids%5B%5D=12504126&seen_snote_ids%5B%5D=12501985&seen_snote_ids%5B%5D=12546054&seen_snote_ids%5B%5D=12678929&seen_snote_ids%5B%5D=12673823&seen_snote_ids%5B%5D=12688021'
url = baseUrl + data
results = getDate(url)
excel_write(results)
输出结果
5.png
