post 的请求参数的转换
- 第一步 导入
import urllib.parse
实例： data = urllib.parse.urlencode().decode(data) # 编码 + 转为二进制
json解析的使用
- 首先同样的还是要导入
import json
content_dict = json.loads(content) # json 解析
urllib下的文件存储
urllib.request.urlretrieve(cover_url, 'images1/%s.png'%title)
urllib.request.urlcleanup()
##注意## 要在当前目录创建相应的文件存放地址
正则的正确使用
获取岗位数量
# 使用正则：re
# 正则处理的是：字符串
# re.match()
# re.search()
# re.findall()
# (.*): 贪婪
# + 贪婪
# ? 非贪婪
# str: "<div>abc</div> <div>def</div>"
# re: <div>(.*)</div> => ["abc</div> <div>def"]
# re: <div>(.*?)</div> => ["abc", "def"]
#
- 第一步 导入
import re
- 第二步 创建好正则表达式
imgReg1 = '<img alt=".*" src="(.*?)" class="s-access-image cfMarker" height="260" width="200">'
- 第三步 匹配正则
img1_com = re.compile(imgReg1)
img2_com = re.compile(imgReg2, re.S)
img1_list = img1_com.findall(html)
img2_list = img2_com.findall(html)
文件操作
with open('ali.txt', 'a', encoding='utf-8') as fp:
for job in datas:
degree = job.get('degree') # 学历
departmentName = job.get('departmentName') # 部门
description = job.get('description') # 岗位要求
firstCategory = job.get('firstCategory') # 类型
workExperience = job.get('workExperience') # 要求
job_str = str((degree, departmentName, description, firstCategory, workExperience)) + "\n"
fp.write(job_str)
fp.flush()
handeler 和 open
import urllib
from urllib import request
# urlopen: 特殊的打开器opener
# urllib.request.urlopen(url)
# 处理cookie或代理 需要用到自定义打开器
# 处理器对象hander
http = urllib.request.HTTPHandler() # http处理器
# http = urllib.request.HTTPHandler(debuglevel=1) # debuglevel=1 调试级别，可以在控制台输出日志
# print(http)
# 创建打开器对象opener
opener = urllib.request.build_opener(http) # 要传入handler对象
# 设置opener为全局打开器
# 后面的urlopen也会使用opener去打开url
urllib.request.install_opener(opener)
# 打开url
response = opener.open("http://www.baidu.com")
print(response)
print(response.read().decode())
# urlopen()
# response = request.urlopen('http://www.baidu.com')
# print(response.read().decode())
代理IP的使用
import random
import urllib
from urllib import request
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36"
}
# 使用代理
# proxy = {'http': "61.135.155.82:443"}
# 使用ccproxy
proxy = {'http': 'http://user1:123456@10.20.154.59:808'}
# proxy = {'http': 'http://10.20.154.59:808'}
# IP代理池
proxy_list = [
{'http': "61.135.155.82:443"},
{'http': "61.183.233.6:54896"},
{'https': "218.249.45.162:35586"},
{'https': "14.118.135.10:808"},
# ...
]
# ua池（user-agent池 ）
user_agent_list=[
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko",
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Mobile Safari/537.36"
]
# 设置代理
# proxy = random.choice(proxy_list) # 从代理池中随机获取一个代理ip
# print(proxy)
proxy_handler = request.ProxyHandler(proxies=proxy)
opener = request.build_opener(proxy_handler)
url = "http://www.ifeng.com/"
req = request.Request(url, headers=headers)
req.add_header("User-Agent", random.choice(user_agent_list)) # 从ua池中随机获取一个ua
res = opener.open(req)
print(res.read().decode())
