#!python3
# -*- encoding:utf-8 -*-
# 作者：大帅
# 程序目的：国内代理爬取收集存储备用

# 导入需要的模块
import requests,lxml,re,pprint,datetime,time,sys
from bs4 import BeautifulSoup

starttime = datetime.datetime.now()

pageNumber = 0

ipDictFinal = {}

# 爬取代理ip的目标网址并设置头信息
def getHtml(pagenumber=1):

    url = 'http://www.xicidaili.com/nn/'

    headers = {'User-agent':'Mozilla/5.0'}

    proxies = {'https':'http://115.221.112.28:20718'}

# 发送请求并进行编码处理

    r = requests.get(url=url+str(pagenumber),proxies = proxies,headers=headers,timeout=30)

    r.encoding = r.apparent_encoding

    html = r.text

    return html

# 利用bs4模块对网址进行格式化处理，方便提取标签信息
def ipGet(html):

    x = 0
    
    soup = BeautifulSoup(html,"html.parser")

    ipSpam = soup.find_all('tr',class_='odd')

    newsoup = BeautifulSoup(str(ipSpam),"html.parser")

    ip = newsoup.find_all('td')

# 建立存储ip的词典和临时存储的列表

    ipDict = {}

    iplist = []

# 对bs4定位后的标签信息编写正则提取信息

    for i in range(len(ip)):
        patternIp = re.compile(r'\d+\.\d+\.\d+\.\d+|^\d+$|^HTTP(S)?')
        rejex = patternIp.search(str(ip[i].string))
        if rejex == None :
            continue
        else:
# 对提取出来的信息进行第一步存储
            iplist.append(rejex.group())

# 由于词典key不支持列表，采用列表转元组的方法
    while x < (len(iplist)-2):
        iptuple = tuple(iplist)
        ipDict[iptuple[x]] = [iptuple[x+1],iptuple[x+2]]
        x += 3
    return ipDict

# 对元组进行分组赋值给词典的key和value
'''
    for i in range(len(iptuple)-1):
        if True :
            ipDict[iptuple[i][1]]=[iptuple[i][2],iptuple[i][3]]
        else:
            continue
'''
    

def ipSave(ipDict):
    pprint.pformat(ipDict)
    ipfile = open(r'f:\workspace\ipsave.py','w')
    ipfile.write('ipDict='+pprint.pformat(ipDict)+'\n')

if __name__ =="__main__":
    while True:
        pageNumber += 1
        time.sleep(3)
        ipDict1=ipGet(getHtml(pageNumber))
        ipDictFinal = ipDict1.copy()
        ipDictFinal.update(ipDictFinal)

        if pageNumber == 3:
            ipSave(ipDictFinal)
            print('已完成爬取，共爬取%d个代理' % len(ipDictFinal))
            break
        else:
            nowtime = datetime.datetime.now()
            killtime = (nowtime - starttime).seconds
            killtimestr = str(killtime)
            killtimeint = int(killtimestr)
            
            sys.stdout.write('程序正在执行中：已执行%d秒\n' % killtimeint)
            sys.stdout.flush()
            continue

# 漂亮打印验证
#pprint.pprint(ipDict)