爬取 豆瓣电影我不是药师 评论分析

#--*--coding:utf-8--*--

import requests

import json

import os

import sys

import random

import time

proxies = {

  "http": "proxy.xx.com:911",

  "https": "proxy.xx.com:911",

}

#url=r"https://movie.douban.com/subject/26752088/comments?status=P"

"""

https://movie.douban.com/subject/26752088/comments?start=0&limit=20&sort=new_score&status=P&percent_type=

https://movie.douban.com/subject/26752088/comments?start=40&limit=20&sort=new_score&status=P&percent_type=

https://movie.douban.com/subject/26752088/comments?start=60&limit=20&sort=new_score&status=P&percent_type=

"""

u1=r"https://movie.douban.com/subject/26752088/comments?start="

u2=r"&limit=20&sort=new_score&status=P&percent_type="

for i in range(20,1320,20):

    url=u1+str(i)+u2

    print(url)

    r=requests.get(url,proxies=proxies)

    print(r.status_code)

    html=str(r.content, encoding = "utf_8_sig")

    print(html)

    f = open('c:/users/ffan2/desktop/wbsys_pl.txt','a',encoding='utf_8_sig')

    f.write(html)

# coding: utf-8# In[1]:# -*- coding: utf-8 -*-import requestsimport jsonimport osimport sysimport randomimport timeimport reimport numpy as npimport pandas as pdfrom pandas import DataFrameimport matplotlib.pyplot as plt#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\jd_data.csv',encoding='UTF_8'))# In[3]:html=open('c:\\users\\ffan2\\desktop\\wbsys.txt','r',encoding = "UTF_8").read()# In[158]:votes_0=re.findall(r'.*?',html)votes_1=re.sub(r'.*?','',str(votes_0))

votes=re.sub(r'',r'',str(votes_1))votes  # get all vote numberslen(votes)# In[185]:comments_0=re.findall(r'

.*',html)

comments=re.sub(r'

.*?','',str(comments_0))#comments# In[165]:comments=re.sub(r'','',str(comments))len(comments)# In[166]:comment_time_0=re.findall(r'\'',r'',str(comment_time_2))#comment_time# In[167]:user_name_0=re.findall(r'','',str(user_name_0))

user_name=re.sub('<','',str(user_name_1))

#user_name

# In[154]:

a={'user_name':user_name,'comment_time':comment_time,'votes':votes,'comments':comments}

print(len(a['comments']))

print(len(a['votes']))

print(len(a['comment_time']))

print(len(a['user_name']))

# In[173]:

table=pd.DataFrame(a[comments])

#table.to_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding = "utf_8_sig")

#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding='UTF_8_sig'))

#print(table.shape)

#使用jieba 分词 分析rank~

word_str=''.join(comments)

word_rank=jieba.analyse.extract_tags(word_str, topK=20, withWeight=True, allowPOS=())

print(word_rank)

word_rank = pd.DataFrame(word_rank,columns=['word','rank'])

word_rank.set_index('rank')

rank word

0.159256电影

0.063695徐峥

0.056682题材

0.052965煽情

0.046412现实

0.042310王传君

0.040851药神

0.033984一部

0.032249达拉斯

0.028569这部

0.026092片子

0.026046希望

0.025504导演

0.025464真实

0.025396故事

0.024536剧情

0.024384中国

0.024108社会

0.023750国产片

0.023353演员