import threading
import time
from lxmlimport etree
import requests
from queueimport Queue
import re
"""
其实 函数参数中的冒号是参数的类型建议符，告诉程序员希望传入的实参的类型 。 函数后面跟着的箭头是函数返回值的类型建议符
，用来说明该函数返回的值是什么类型 。 更官方的解释：此为type hints，是Python 3.5新加的"""
headers = {
'user-agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55",
'refer':'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'
}
cookie={'cookie':'tt_webid=7077014656070960676; _S_DPR=1.5; _S_IPAD=0; MONITOR_WEB_ID=7077014656070960676; FRM=new; PIXIEL_RATIO=1.5; WIN_WH=1280_648; ttcid=c9720ff1a7c146a790a1c113c8f8e91f39; tt_scid=0LEZtCnx3EI6WHPLeDb9vGDK-P-YwQDee.cVi7b6xfkeANJ9dxJUAUIHgJiGKGF08d18; ttwid=1%7C5pSenh2ZSIAoAPAnZmcHQ2xN3TykA8ikp6iCucnETC0%7C1648709296%7Ce85f11b922972468dc1718583be6152a42d5113842d03386a442d9afccd8b8aa; _S_WIN_WH=1280_648'}
class Producer(threading.Thread):
def __init__(self, page_queue,img_queue, *args, **kwargs):
super(Producer, self).__init__(*args, **kwargs)
self.page_queue= page_queue
self.img_queue = img_queue
def run(self):
while True:
if self.page_queue.empty():
break
else:
url =self.page_queue.get()
self.parse_page(url)
def parse_page(self, url):
response = requests.get(url=url, headers=headers,cookies=cookie)
text = response.text
html = etree.HTML(text)
img_list = html.xpath('//div[@class="cs-view cs-view-block cs-card-content"]')
for liin img_list:
href = li.xpath('./a/@href')
if href:
m = re.findall(r'.*(\d{19}).*', href[0])
a ='https://www.toutiao.com/a' + m[0] +'/?channel=&source=search_tab'
self.parser2(a)
def parser2(self,a):
response = requests.get(url=a, headers=headers,cookies=cookie )
m = etree.HTML(response.text)
s = m.xpath('//article[@class="syl-article-base tt-article-content syl-page-article syl-device-pc"]')[0]
x = s.xpath('./div[@class="pgc-img"]')
for iin x:
ll = i.xpath('./img/@src')
if ll:
self.img_queue.put(ll[0])
class Consumer(threading.Thread):
def __init__(self, page_queue, img_queue, *args, **kwargs):
super(Consumer, self).__init__(*args, **kwargs)
self.page_queue = page_queue
self.img_queue = img_queue
print('af',self.img_queue.get())
def run(self):
while True:
if self.page_queue.empty()and self.img_queue.empty():
break
else:
imgurl=self.img_queue.get()
name = imgurl[-11:-8]
path =r"C:\Users\29258\Desktop\er\\" + name +'.jpg'
response = requests.get(url=imgurl, headers=headers,)
with open(path, 'wb')as fp:
fp.write(response.content)
print(name,'下载好了')
def main():
page_queue = Queue(50)# 存储页码链接
img_queue = Queue(10)# 存储解析出来的图片链接
# 想要爬取前10页的数据
for pagein range(50):
url =f'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'
print(url)
page_queue.put(url)# 将10页的页码链接加入到了page_queue
for xin range(3):
t = Producer(page_queue, img_queue)
t.start()
# 开启三个消费者
for xin range(5):
t = Consumer(page_queue, img_queue)
t.start()
time.sleep(3)
if __name__=='__main__':
main()
# 开启三个生产者
# for x in range(3):
# t = Producer(page_queue, img_queue)
# t.start()
# # 开启三个消费者
# for x in range(3):
# t = Consumer(page_queue, img_queue)
# t.start()
原创
