今天开始爬取今日头条的第二个入口搜索，搜索有四个tab分别为综合，视频，图集，用户。先来分析一下综合的搜索接口
https://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E7%A4%BE%E5%8C%BA%E6%96%B0%E9%9B%B6%E5%94%AE&autoload=true&count=20&cur_tab=1&from=search_tab
只需要传递keyword 搜索关键字即可，无需进行其他处理，直接解析接口返回数据，该接口返回的数据有点杂数据结构不统一，有文章，用户，微头条，问答，搜索推荐，其他关键字搜索推荐，在做数据解析的时候要针对这集中数据结构分别做处理。老样子直接上代码：
def get_search_article(self, keyword, offset=0):
keyword = urllib.request.quote(keyword)
req_url = "https://www.toutiao.com/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Connection': 'keep-alive',
'authority': 'www.toutiao.com',
'referer': "https://www.toutiao.com/search/?keyword={}".format(keyword),
'method': 'GET',
'path': "/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword),
'scheme': 'https'
}
self.s.headers.update(headers)
req = self.s.get(req_url, proxies=get_proxy_ip())
time.sleep(random.random() * 2 + 3)
data = json.loads(req.text)
items = data['data']
if data['has_more'] == 1:
self.page = self.page + 1
offset = 20 * self.page
self.parse_data(items)
time.sleep(2)
self.get_search_article(keyword, offset)
else:
self.parse_data(items)
toutiaodb.save(self.search_item_list)
def parse_data(self, items):
for item in items:
try:
type = item['cell_type']
except:
type = 0
if type == 37: #微头条
pass
elif type == 50:
pass
elif type == 66:
pass
elif type == 26: #内容推荐
pass
elif type == 20: #搜索推荐
pass
elif type == 38: #用户
pass
else:
titem = toutiaoitem()
titem.user_id = item['user_id']
try:
titem.source = item['source']
except:
titem.source = item['name']
titem.title = item['title']
titem.source_url = item['article_url']
titem.media_url = item['media_url']
titem.item_id = item['item_id']
titem.abstract = item['abstract']
titem.comments_count = item['comments_count']
titem.behot_time = item['behot_time']
titem.image_url = item['image_url']
titem.image_list = item['image_list']
titem.tag = item['tag']
if 'play_effective_count' in item:
titem.article_genre = 'vedio'
titem.read_count = item['play_effective_count']
else:
titem.article_genre = 'article'
self.search_item_list.append(titem)
