import re
importr equests
from requests.exceptions import RequestException
import json
from multiprocessing import Pool
//抓取页面
def get_one_page(url):
try:
response = requests.get(url)
ifresponse.status_code ==200:
returnresponse.text
return None
except RequestException:
return None
//页面数据进行解析
def parse_one_page(html):
pattern = re.compile('.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">
+'.*?>(.*?).*?star">(.*?).*?releasetime">(.*?)'
+'.*?integer">(.*?).*?fraction">(.*?).*?',re.S)
items = re.findall(pattern,html)
for item in items:
yield{
'index':item[0],
'image':item[1],
'title':item[2],
'actor':item[3].strip()[3:],
'time':item[4].strip()[5:],
'score':item[5]+item[6],
}
//保存本地
def write_to_file(content):
withopen("result.txt","a",encoding="utf-8")asf:
f.write(json.dumps(content,ensure_ascii=False) +'\n')
f.close()
def main(offset):
url ="http://maoyan.com/board/4?offset="+str(offset)
html =get_one_page(url)
for item in parse_one_page(html):
write_to_file(item)
//多进程
if__name__ =='__main__':
pool = Pool()
pool.map(main,[i*10foriinrange(10)])
pool.close()
