1.访问动态网页的方法
首先在middlerware中间键中process_request函数中重新写该方法
from selenium import webdriver
from scrapy.http import HtmlResponse
def process_request(self,request,spider):
path = r'example_path'
option = webdriver.ChromeOption()
option.add_argument('--headless')
deriver = webdriver.Chrome(excutable_path=path,option=option)
driver.get("example.com")
url = driver.current_url
body = driver.page_source
res = HtmlResponse(url=url,body=body,encoding='utf-8',request=request)
# 让界面滚动的方法,同时需要在setting中middleware下载中加入
# "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None
js = "var q = document.documentElement.scrollTop = {}"
for i in range(10):
driver.excute_script(js.formate(i*200)
return res
2.ip代理的问题
首先需要在setting配置文件中先定义一个ip池
IPPOOL = [
{ 'ip':''},
{ 'ip':''},
{ 'ip':''},
{ 'ip':''},
{ 'ip':''},
{ 'ip':''},
{ 'ip':''},
]
在middleware中的process_request函数中重新写
# 需要重新定义ip
def __init__(self,ip=''):
self.ip = ip
# 重新写process_request方法
def process_request(self,request,spider)
ip = random.choice(IPPOOL)['ip']
request.meta['proxy'] = 'https://' + ip
# 需要在setting中配置
DOWNLOADER_MIDDLEWARES = {
# 'daili.middleware.ExampleDownloaderMiddleware': 543,
'daili.middlewares.ExampleDownloaderMiddleware':222,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':333,
}
