10.4.1 网页内容读取与域名分析
Python 提供了 urllib 库支持网页内容读取。
读取并显示网页内容。
>>> import urllib.request
>>> fp = urllib.request.urlopen(r'http://www.python.org')
>>> print(fp.read(100))
b'<!doctype html>\n<!--[if lt IE 7]> <html class="no-js ie6 lt-ie7 lt-ie8 lt-ie9"> <![endif]-->\n<!-'
>>> print(fp.read(100).decode())
-[if IE 7]> <html class="no-js ie7 lt-ie8 lt-ie9"> <![endif]-->
<!--[if IE 8]> <h
>>> fp.close()
用 GET 方法读取并显示指定 URL 内容。
>>> import urllib.parse
>>> params = urllib.parse.urlencode({'spam':1, 'eggs':2, 'bacon':0})
>>> url = "http://www.musi-cal.com/cgi-bin/query?%s"%params
>>> with urllib.request.urlopen(url) as f:
print(f.read().decode('utf-8'))
用 POST 方法提交参数并显示指定 URL 内容。
>>> data = urllib.parse.urlencode({'spam':1, 'eggs':2, 'bacon':0})
>>> data = data.encode('ascii')
>>> with urllib.request.urlopen("http://requestb.in/xrb182xr", data) as f:
print(f.read().decode('utf-8'))
urllib.parse 提供了域名解析功能，支持拆分与合并 URL 及相对地址到绝对地址的转换。
>>> from urllib.parse import urlparse
>>> o = urlparse('http://www.cwi.nl:80/sodifj/Python.html')
>>> o.port
80
>>> o.hostname
'www.cwi.nl'
>>> urlparse('http://www.cwi.nl:80/sodifj/Python.html')
ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/sodifj/Python.html', params='', query='', fragment='')
>>> from urllib.parse import urljoin
>>> urljoin('http://www.cwi.nl:80/sodifj/Python.html', 'FAQ.html')
'http://www.cwi.nl:80/sodifj/FAQ.html'
>>> from urllib.parse import urlsplit
>>> url = r'https://docs.python.org/3/library/urllib.parse.html'
>>> r1 = urlsplit(url)
>>> r1.hostname
'docs.python.org'
>>> r1.geturl()
'https://docs.python.org/3/library/urllib.parse.html'
>>> r1.netloc
'docs.python.org'
>>> r1.scheme
'https'
抓取指定网页的所有链接，并且可以深度抓取。
import sys
import multiprocessing
import re
import os
import urllib.request as lib
def craw_links(url, depth, keywords, processed):
''' '''
content = []
if url.startswith('http:') or url.startswith('https://'):
if url not in processed:
# mark this url as processed
processed.append(url)
else:
#
return
print('Crawing ' + url + '...')
fp = lib.urlopen(url) # 创建一个表示远程url的类文件对象，然后像本地文件一样操作这个类文件对象来获取远程数据
contents = fp.read()
contents_decoded = contents.decode('UTF-8')
fp.close()
pattern = '|'.join(keywords)
# if this page contains cetrain keywords, save it to a file
flag = False
if pattern:
searched = re.search(pattern, contents_decoded)
else:
# if the keywords to filter is not given, save current page
flag = True
# print(flag, searched)
if flag or searched:
with open('craw\\' + url.replace(':', '_').replace('/', '_'), 'wb') as fp:
fp.write(contents)
# find all the links in the current page
links = re.findall('href="(.*?)"', contents_decoded)
# craw all links in the current page
for link in links:
# consider the relative path
if not link.startswith(('http://', 'https://')):
try:
index = url.rindex('/') # 查找到最后一个/的位置，从该位置的下一个字符切片直到末尾
if link.startswith(('../')):
index = url[0:index].rindex('/')
link = link[3:]
link = url[0:index+1] + link
# print('link1:', link)
except Exception as e:
print(e)
pass
if depth > 0 and link.endswith(('.html', '.htm')):
# print('link2:', link)
craw_links(link, depth + 1, keywords, processed)
if __name__ == '__main__':
processed = []
keywords = ['KeyWord1', 'KeyWord2']
if not os.path.exists('craw') or not os.path.isdir('craw'):
os.mkdir('craw')
craw_links(r'https://docs.python.org/3/library/index.html', 1, keywords, processed)
print(processed)
