map使用
map函数包含了序列操作，参数传递和结果保存的的一系列操作
导入
from multiprocessing.dummy import Pool
pool = Pool(4) #表示4线程
results = pool.map(爬取函数，网址列表)
附上简单并行爬虫的例子
__author__ = 'ding'
'''
多线程使用
'''
from multiprocessing.dummy import Pool as ThreadPool
import requests
import time
def getsource(url):
html = requests.get(url)
urls = []
for i in range(1, 21):
newpage = 'http://tieba.baidu.com/p/3522395718?pn=' + str(i)
urls.append(newpage)
time1 = time.time()
for i in urls:
print(i)
getsource(i)
time2 = time.time()
print('单线程耗时：' + str(time2 - time1))
pool = ThreadPool(4)
time3 = time.time()
results = pool.map(getsource, urls)
pool.close()
pool.join()
time4 = time.time()
print("并行耗时：" + str(time4 - time3))
out put
单线程耗时：109.83350014686584
并行耗时：13.332000017166138
网络较慢所以耗时较长，但相比较而言并行效率比单线程要高上很多
