讲解：
- 使用Python中PyQuery库 爬去猫眼电影 并存入MongoDB数据库 、txt文档、
- 涉及到Python异步
涉及Python相关库：
from urllib.robotparser import RobotFileParser
import requests
from pyquery import PyQuery
import pymongo
import copy
import asyncio
下面展示一段神奇的代码:
from urllib.robotparser import RobotFileParser
import requests
from pyquery import PyQuery
import pymongo
import copy
import asyncio
offert = 0
offert_tail = 100
async def get_one_page(url):
rb = RobotFileParser()
rb.set_url(url)
rb.read()
if rb.can_fetch("*", url) == True:
reponse = requests.get(url)
if reponse.status_code == 200:
return reponse.text
async def one_page(html):
doc = PyQuery(html)
aa = doc('dd').items()
item = {}
for i in aa:
dd = []
indexi = i.children('i').text()
namei = i.find('.name').text()
start = i.find('.star').text()
releasetime = i.find('.releasetime').text()
score = i.find('.score').text()
item['indexi'] = indexi
item['namei'] = namei
item['start'] = start
item['releasetime'] = releasetime
item['score'] = score
write_r(str(item))
try:
copyy = copy.deepcopy(item)
dd.append(copyy)
except:
print("添加异常")
finally:
mongo(dd)
async def main():
url = "https://maoyan.com/board/4?offset="
for i in range(offert, offert_tail, 10):
urls = url + str(i)
aa = await get_one_page(urls)
await one_page(aa)
async def alaws():
task = loop.create_task(main())
await asyncio.sleep(0.2)
try:
await task
except asyncio.CancelledError:
print("取消任务抛出异常")
finally:
print("获取结果：",task.result())
def mongo(i):
conn = pymongo.MongoClient()
db = conn.dev
collection = db.dev_test
try:
collection.insert_many(i)
except BaseException as e:
print("error:",e)
def write_r(neirong):
with open('./text.txt','a',encoding='utf-8') as f:
f.write(neirong + '\n')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
try:
print("开启事件")
loop.run_until_complete(alaws())
finally:
print("关闭事件")
loop.close()
如果关于PyQuery 还不了解的 请看上一章：
https://www.jianshu.com/p/ee588abe500d
