在目前为止爬取的新闻网站类型中,今日头条的爬取规则是抓取的最严的,本文的爬取代码不是我写出来的,有参与其中
#!coding=utf-8
# author： soliton.wang
# Email : soliton.wang@gmail.com
# Install_moduels: requests pyexecjs pymysql pandas
# toutiao.sing.js : https://shimo.im/docs/hjCPJGtRdyTDkdd8
import requests
import re
import json
import random
import time
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import hashlib
import execjs
import pandas as pd
from pymysql import *
###禁止提醒SSL警告
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# mysql connect
# mysqldb = connect(
# host='IP',
# port=3306,
# database='database',
# user='username',
# password='password',
# charset='utf8')
# cs = mysqldb.cursor()
# spider_web_name
name = '头条' # 插入数据库时使用
class toutiao(object):
def __init__(self, url):
self.url = url
self.s = requests.session()
headers = {'Accept': '*/*',
'Accept-Language': 'zh-CN',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
'Connection': 'Keep-Alive',
}
self.s.headers.update(headers)
self.channel = re.search('ch/(.*?)/', url).group(1)
def closes(self):
self.s.close()
def getdata(self): # 获取数据
req = self.s.get(url=self.url, verify=False)
# print (self.s.headers)
# print(req.text)
headers = {'referer': self.url}
max_behot_time = '0'
signature = '.1.hXgAApDNVcKHe5jmqy.9f4U'
eas = 'A1E56B6786B47FE'
ecp = '5B7674A7FF2E9E1'
self.s.headers.update(headers)
title = []
source = []
source_url = []
comments_count = []
tag = []
chinese_tag = []
label = []
abstract = []
behot_time = []
nowtime = []
duration = []
for i in range(0, 30): ##获取页数
Honey = json.loads(self.get_js())
# eas = self.getHoney(int(max_behot_time))[0]
# ecp = self.getHoney(int(max_behot_time))[1]
eas = Honey['as']
ecp = Honey['cp']
signature = Honey['_signature']
url = 'https://www.toutiao.com/api/pc/feed/?category={}&utm_source=toutiao&widen=1&max_behot_time={}&max_behot_time_tmp={}&tadrequire=true&as={}&cp={}&_signature={}'.format(
self.channel, max_behot_time, max_behot_time, eas, ecp, signature)
req = self.s.get(url=url, verify=False)
time.sleep(random.random() * 2 + 2)
print(req.text)
if req.text != '':
j = json.loads(req.text)
for k in range(0, 7):
now = time.time()
if j['data'][k]['tag'] != 'ad':
title.append(j['data'][k]['title']) ##标题
source.append(j['data'][k]['source']) ##作者
source_url.append('https://www.toutiao.com' + j['data'][k]['source_url']) ##文章链接
try:
comments_count.append(j['data'][k]['comments_count']) ###评论
except:
comments_count.append(0)
tag.append(j['data'][k]['tag']) ###频道名
try:
chinese_tag.append(j['data'][k]['chinese_tag']) ##频道中文名
except:
chinese_tag.append('')
try:
label.append(j['data'][k]['label']) ## 标签
except:
label.append('')
try:
abstract.append(j['data'][k]['abstract']) ###文章摘要
except:
abstract.append('')
behot = int(j['data'][k]['behot_time'])
behot_time.append(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(behot))) ####发布时间
nowtime.append(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now))) ##抓取时间
duration.append(now - behot) ##发布时长
time.sleep(2)
# max_behot_time=str(j['next']['max_behot_time'])
print('------------' + str(j['next']['max_behot_time']))
data = {'title': title, 'source': source, 'source_url': source_url, 'comments_count': comments_count,
'tag': tag, 'label': label,
'chinese_tag': chinese_tag, 'abstract': abstract, 'behot_time': behot_time,
'nowtime': nowtime, 'duration': duration
}
df = pd.DataFrame(data=data)
sql_values = df.values
print(sql_values)
# for sql_value in sql_values:
# query = "insert ignore into toutiao(title,source,source_url,comments_count, tag, label, chinese_tag,abstract,behot_time,nowtime,duration,class) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
# values = (sql_value[0], sql_value[1], sql_value[2], sql_value[3], sql_value[4], sql_value[5], sql_value[6],sql_value[7], sql_value[8], sql_value[9], sql_value[10], name)
# cs.execute(query, values)
# mysqldb.commit()
else:
print('spider_toutiao_data_null')
def getHoney(self, t): #####根据JS脚本破解as ,cp
# t = int(time.time()) #获取当前时间
# t=1534389637
# print(t)
e = str('%X' % t) ##格式化时间
# print(e)
m1 = hashlib.md5() ##MD5加密
m1.update(str(t).encode(encoding='utf-8')) ##转化格式
i = str(m1.hexdigest()).upper() ####转化大写
# print(i)
n = i[0:5] ##获取前5位
a = i[-5:] ##获取后5位
s = ''
r = ''
for x in range(0, 5):
s += n[x] + e[x]
r += e[x + 3] + a[x]
eas = 'A1' + s + e[-3:]
ecp = e[0:3] + r + 'E1'
return eas, ecp
def get_js(self):
f = open(r"toutiao_sing.js", 'r', encoding='UTF-8')
line = f.readline()
htmlstr = ''
while line:
htmlstr = htmlstr + line
line = f.readline()
ctx = execjs.compile(htmlstr)
return ctx.call('get_as_cp_signature')
if __name__ == '__main__':
url = 'https://www.toutiao.com/ch/news_world/'
t = toutiao(url)
t.getdata()
t.closes()
# cs.close()
# mysqldb.close()
Tips：url 填写你要获取的页面新闻目录,toutiao_sing.js文件自己下载,下载链接在代码开头,安装好需要的库后,可以直接执行获取新闻数据 toutiao_sing.js文件和代码文件放在同一级目录,如不在同一级目录就修改文件路径
