这周五，小伙伴出差来北京，大家想一起聚聚。提议在望京附近吃烤肉。用网页查哪家店不错也许不错，但是咱不是刚学了爬虫么。于是我用爬虫写了一小段代码来实现从美团信息的爬取
在从美团爬取之前，我先确定了本次需要爬取的信息：
店名，地址，评分，均价（毕竟还没有到财富自由...）和推荐菜
打开美团网页，确定网页。发现选取美团的不同category网址是不同的，但是翻页时，会发现pn+number 实现翻页
image.png
还有一种方法是使用i.meituan模拟手机端，这里cateId=54就是烧烤，55是川菜。。。剩下的你懂的。只需要做的就是加载所有页面。
2.PNG
回到第一种方法。直接来代码吧。
比较头疼的是遇到验证码。每次遇到验证码就需要停下代码，然后验证之后，获得cookies。我还在学习如何解决验证码的事情。。。
import requests
import lxml.etree as le
import json
import re
from pymongo import MongoClient
def parse1(page=1):
'''
1. 首先分析抓取什么内容
2. 通过url发现多页的规律
3. 如何定位2级页面
4. 使用爬虫将一级网页爬取后，获得相关内容
5. 将1~5封装为一个函数
下面基本已经成型，然后需要做的就是做一个函数，把下面的代码封装在函数里面
'''
url = "https://bj.meituan.com/meishi/c54b673/pn{page}/".format(page = page)
headers ={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36',
'Cookie': 'client-id=28d33416-a0ee-45a8-a0d0-469a9a693b6a; uuid=00b656f9-0a16-4ceb-9e6a-81e6bf75e71e; _lxsdk_cuid=1740f0bd059c8-0712241f957fe2-3c634203-144000-1740f0bd059c8; _hc.v=89b7e524-cd82-2fde-dbaf-12461dd77d3a.1597980269; IJSESSIONID=t6h6xxphbftpxae9521qjnj3; iuuid=C629760EE89131469C5A0CC8A366A46F2BA47DF0E20290569B7E62B1425945EB; ci=1; cityname=%E5%8C%97%E4%BA%AC; webp=1; ci3=1; __utma=74597006.1892913350.1597986656.1597986656.1597986656.1; __utmc=74597006; __utmz=74597006.1597986656.1.1.utmcsr=blog.csdn.net|utmccn=(referral)|utmcmd=referral|utmcct=/xing851483876/article/details/81842329; _lx_utm=utm_source%3Dblog.csdn.net%26utm_medium%3Dreferral%26utm_content%3D%252Fxing851483876%252Farticle%252Fdetails%252F81842329; _lxsdk=C629760EE89131469C5A0CC8A366A46F2BA47DF0E20290569B7E62B1425945EB; meishi_ci=1; cityid=1; latlng=40.008608,116.478681,1597987739686; i_extend=C_b0E169406054609187083475856891434993064435_e-6403314363349724909_v5186722642576530108_a%e6%9c%9b%e4%ba%ac+%e7%83%a7%e7%83%a4Gimthomepagecategory11H__a100005__b1; lat=40.002208; lng=116.463343; _lxsdk_s=17410011f3f-808-064-4ad%7C%7C3',
'Host': 'bj.meituan.com'
}
meituan1 = requests.get(url,headers = headers)
ret = re.findall(r"<script>window._appState = (.*?);</script>",meituan1.content.decode('utf-8')) #匹配太多
data = json.loads(ret[0])['poiLists']['poiInfos']
return data
def parse2(url):
#url = "https://www.meituan.com/meishi/195931264/"
headers1 = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36',
'Cookie': 'client-id=28d33416-a0ee-45a8-a0d0-469a9a693b6a; uuid=00b656f9-0a16-4ceb-9e6a-81e6bf75e71e; _lxsdk_cuid=1740f0bd059c8-0712241f957fe2-3c634203-144000-1740f0bd059c8; _hc.v=89b7e524-cd82-2fde-dbaf-12461dd77d3a.1597980269; IJSESSIONID=t6h6xxphbftpxae9521qjnj3; iuuid=C629760EE89131469C5A0CC8A366A46F2BA47DF0E20290569B7E62B1425945EB; ci=1; cityname=%E5%8C%97%E4%BA%AC; webp=1; ci3=1; __utma=74597006.1892913350.1597986656.1597986656.1597986656.1; __utmc=74597006; __utmz=74597006.1597986656.1.1.utmcsr=blog.csdn.net|utmccn=(referral)|utmcmd=referral|utmcct=/xing851483876/article/details/81842329; _lx_utm=utm_source%3Dblog.csdn.net%26utm_medium%3Dreferral%26utm_content%3D%252Fxing851483876%252Farticle%252Fdetails%252F81842329; _lxsdk=C629760EE89131469C5A0CC8A366A46F2BA47DF0E20290569B7E62B1425945EB; meishi_ci=1; cityid=1; latlng=40.008608,116.478681,1597987739686; i_extend=C_b0E169406054609187083475856891434993064435_e-6403314363349724909_v5186722642576530108_a%e6%9c%9b%e4%ba%ac+%e7%83%a7%e7%83%a4Gimthomepagecategory11H__a100005__b1; lat=40.002208; lng=116.463343; _lxsdk_s=17410011f3f-808-064-4ad%7C%7C3',
'Host': 'www.meituan.com'
}
meituan2 = requests.get(url, headers=headers1)
ret = re.findall(r'<script>window._appState = (.*?);</script>', meituan2.content.decode('utf-8'))
data = json.loads(ret[0])
#address = data['detailInfo']['address']
menu={}
for i in range(len(data['recommended'])):
parse2_recommend_name = data['recommended'][i]['name']
parse2_recommend_price = data['recommended'][i]['price']
menu.setdefault(parse2_recommend_name,parse2_recommend_price)
return {'menu' : menu}
def spider_meituan(c,start_page =1, end_page=2):
for page in range(start_page,end_page+1):
results = parse1(page=page)
for i in range(len(results)):
url = "https://www.meituan.com/meishi/%d" %results[i]['poiId']
parse2_data = parse2(url=url)
results[i]['menu'] = parse2_data['menu']
c.insert_one(results[i])
if __name__=='__main__':
client = MongoClient()
db = client.get_database('meituan')
ck= db.get_collection('shaokao')
spider_meituan(c=ck,start_page=1, end_page=5)
