python爬虫防止被封ip代理测试

# -*- coding: utf-8 -*-
import urllib2
from bs4 import BeautifulSoup
url = "http://www.ip181.com/"
# url="http://www.baidu.com/"
try:
    proxy_support = urllib2.ProxyHandler({'http': '117.43.1.83'})
    # 参数是一个字典{'类型':'代理ip:端口号'}
    opener = urllib2.build_opener(proxy_support)
    # 定制opener
    opener.add_handler = [
        ('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36')]
    # add_handler给加上伪装
    urllib2.install_opener(opener)
    response = urllib2.urlopen(url)
    html_cont = response.read()
    soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
    jw = soup.find('div',class_="col-md-12", style="font-size:16px;" )
    print jw
except urllib2.URLError,e:
    print e.reason

# -*- coding: utf-8 -*-
# 代理池处理
# import urllib2
# import random

# ip_list=['119.6.136.122','114.106.77.14']
# #使用一组ip调用random函数来随机使用其中一个ip
# url = "http://www.ip181.com/"
# proxy_support = urllib2.ProxyHandler({'http':random.choice(ip_list)})
# #参数是一个字典{'类型':'代理ip:端口号'}
# opener = urllib2.build_opener(proxy_support)
# #定制opener
# opener.add_handler=[('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36')]
# #add_handler给加上伪装
# urllib2.install_opener(opener)
# response = urllib2.urlopen(url)

# print response.read().decode('gbk')