image.png
伪装成浏览器，请求页面，并下载网页
import urllib.request
URL = "https://www.hao123.com/manhua/detail/176"
header ={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/51.0.2704.63 Safari/537.36'}
opener = urllib.request.build_opener()
opener.add_handler= [header]
data = opener.open(URL).read()
Hfile = open("/Users/vincentwen/Downloads/file.html","wb")
Hfile.write(data)
Hfile.close()
image.png
爬取漫画网站的首页的
import re
import urllib.request
import urllib.error
#读取需要爬取的网址
Readdata= urllib.request.urlopen("http://www.pufei.net/").read()
#对读取的结果进行编码
data = Readdata.decode("utf-8","ignore")
#定义正则表达式,匹配manhua目录下的所有网址
pat= ' href="http://www.pufei.net/manhua/(.*?)/"'
#匹配网页中所有的符合条件的url链接地址
allurl= re.compile(pat).findall(data)
for i in range(0,len(allurl)):
try:
print("第"+str(i)+"抓取")
thisurl= allurl[i]
file= "/Users/vincentwen/Downloads/"+str(i)+".html"
urllib.request.urlretrieve(thisurl, file)
print("------抓取成功-----")
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
image.png
