源码见
功能分析
网上爬取豆瓣电影排行很多，但是由于做一个h5画廊需求大量电影素材，而列表与详情页的图片清晰度满足不了要求，所以决定爬取豆瓣原图，在查看是发现需要登陆，然而登陆后原图链接查看并没有任何cookie信息在请求头，遂想直接构造链接爬取，爬取过程出现302，重新分析请求头，发现有referer,嗯，直接加了豆瓣首页作为referer,mdfuck居然没有用，后来发现每个原图需要带着该图前中等缩略图链接作为referer，可以看出豆瓣在服务器端作了比较严格的过滤处理，虽然在nginx上也做过图片防盗链，自认为没有做如此猥琐，别人猥琐，咱也不能谦虚是吧，一个个构造请求头，利用python爬取原图。
所需模块
网页内容抓取用urllib2
页面解析用pyquery(一个可以用jquey方式解析html的模块)
下面是代码部分：（好久没写python,表示有点手生ฅʕ•̫͡•ʔฅ，图片大概200m，页面爬取很快，最后是io下载，所以速度还是取决网速）
#coding:utf-8
import urllib2
import re
import sys
import time
from pyquery import PyQuery as pq
#http://movie.douban.com/top250?start=0&filter=&type=
class Douban:
def __init__(self):
reload(sys)
sys.setdefaultencoding('utf-8')
self.start = 0
self.param = '&filter=&type='
self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36'}
self.movieList = []
self.filePath = './img/'
self.imgpath = 'https://img3.doubanio.com/view/photo/raw/public/'
self.refer = 'https://movie.douban.com/photos/photo/'
def getPage(self):
print '---getpagestart---'
try:
URL = 'http://movie.douban.com/top250?start=' + str(self.start)
request = urllib2.Request(url = URL, headers=self.headers)
response = urllib2.urlopen(request)
page = response.read().decode('utf-8')
pageNum = (self.start + 25)/25
print 'scrabing ' + str(pageNum) + 'page...'
self.start += 25;
return page
except urllib2.URLError, e:
if hasattr(e, 'reason'):
print 'failed reason', e.reason
def htmlparse(self):
print '---getMoviestart---'
while self.start < 250:
page = self.getPage()
html = pq(page);
list = html(".grid_view>li")
info = {};
for item in list:
item = pq(item)
info['name'] = item(".hd>a").text()
info['des'] = item(".bd p:first").text()
info['img'] = item(".pic img").attr('src');
group = re.findall(r'\/(p(\d+)\.jpg)', info['img'])
info['img'] = self.imgpath + group[0][0]
info['refer'] = self.refer + group[0][1] + '/'
#print info
self.movieList.append([info['name'], info['des'], info['img'],info['refer']])
def hook(self):
mfile = open(self.filePath + 'movielist.txt', 'w')
try:
for index, movie in enumerate(self.movieList):
print movie[0].encode('gbk','ignore')
#print movie
self.downImg(movie[2], movie[3], self.filePath + 'movie' + str(index+1) + '.jpg')
mfile.write(str(index+1) + '、' + movie[0] + '\\n'+ movie[1] + '\\n')
print 'wirte done'
finally:
mfile.close()
def downImg(self, URL, refer, imgpath):
head = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36'};
head['Referer'] = refer
request = urllib2.Request(url = URL, headers = head)
try:
f = open(imgpath, 'wb')
res = urllib2.urlopen(request).read()
f.write(res)
f.close()
except urllib2.URLError, e:
if hasattr(e, 'reason'):
print 'failed reason', e.reason
def main(self):
print '---mainstart---'
self.htmlparse()
print len(self.movieList)
self.hook()
DB = Douban()
DB.main()```
>运行效果
