Python爬虫运用之数据分析

今天，我爬取了豆瓣，的《逃避可耻但很有用》的短评，可能队数据分析还不太了解，所以爬取什么都不太懂，也就爬取了评星，日期，还有评论。以下是代码。

# -*- coding: utf-8 -*-

import requests

frombs4 import BeautifulSoup

import re

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0','Cookie':'bid=xQ0BMVjSem8; __utma=30149280.835324949.1486297712.1487313940.1487321509.4; __utmz=30149280.1487321509.4.4.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/search; ll="118130"; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1487321934%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3D39_qCZOrp9cDojY4vvLUS5DYN0eU4kifU3CGzStU3V7%26wd%3D%26eqid%3Dc0ddddb90001116a0000000258a69c0f%22%5D; _pk_id.100001.8cb4=ebaf729a854c56d8.1487245175.3.1487321957.1487314158.; _vwo_uuid_v2=1A8F9B4EEC4F6DBD33D4A2263CFEEB5D|a56d1425e9df513dba4da1881d8ab8fb; __utmc=30149280; ps=y; push_noty_num=0; push_doumail_num=0; __utmv=30149280.15792; ap=1; __utmb=30149280.5.10.1487321509; _pk_ses.100001.8cb4=*; __utmt=1; dbcl2="157929414:qvDHVrTju6Y"; ck=BMh9'}

class Comments():

def __init__(self,url):

self.url = url.split('?')[0]#分隔页面

self.pages = []

def getPages(self,first)

:trueurl = self.url + first

try:

html = requests.get(trueurl,headers=headers).text

nexth = BeautifulSoup(html,'html').find('div',id='paginator').find_all('a')[-1]

nextpage = nexth['href']self.pages.append(nextpage)

print len(self.pages)

if len(self.pages) > 50:

return u'前',len(self.pages),u'页抓取结束'

return self.getPages(nextpage)#递归地抓取页数

except AttributeError :

print u'页面提取结束！'

def getInfo(self):

f = file("comments.txt",'a+')

for page in self.pages:

pageulr = self.url + page

content = requests.get(pageulr,headers=headers).text

try:

all_info = BeautifulSoup(content,'html').find('div',id="comments").find_all(class_="comment-item")

for info in all_info:

pattrern = re.compile('(.*?)(.*?)<',re.S)

infolist = re.findall(pattrern,str(info))

for il in infolist:

star = il[0].strip()

time = il[1].strip()

comments =il[2].strip()

data = 'star='+star + '\ttime=' + time + '\ncomments=' + comments + '\n\n'

f.write(data)

except AttributeError:

print u'评论提取结束'

f.close()

print u'文件写入完毕!'

url ='https://movie.douban.com/subject/26816519/comments?status=P'

com = Comments(url)

com.getPages('?status=P')

com.getInfo()