爬虫代码总结

包包配置

import requests
import ast
import json

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/44.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate',
    'Connection': 'keep-alive'}

当网页可以找到相应的json数据的时候：

url = ""
# 在网页header页面拉到最下面然后找到postdata部分
postdata= {
}
import requests
# 请求
def get_html(url):
    try:
        r = requests.get(url, headers={'User-Agent':'Mozilla/5.0'}, timeout=30)
        r.raise_for_status() # throw HTTPError if the status code is not 200
        r.encoding = r.apparent_encoding # handling encoding issue
        return r.text
    except:
        return "Error: something is Wrong!"
result = requests.post(url,data=postdata, headers=headers).text

# 对返回的string进行按照 自己的 需求进行清洗
result = result.replace('\n','')[4:]
result = result.replace('null,', '')

# 将list  dic 类型的string转成 list dic
clean_result = ast.literal_eval(result)

print json.dumps(clean_result).decode("unicode-escape")

当需要用到解析网页的时候：

一般来说会用到Beautiful Soup: https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/
BeautifulSoup库是用来解析、遍历、维护“标签树”的功能库（t.name,t.string,t.attrs）
另一种方法是RE: https://docs.python.org/3/library/re.html

def getMobileInfo(base_url, page):
    result = []

    for i in range(0,page,1):
        # 拼接url
        url = ''.join([base_url, str(i), '.html'])
        html = requests.get(url, headers=headers).text
        beautyHtml = BeautifulSoup(html, 'lxml')#解析式的一种，错误最少，最全的
        ＃一般来说都会寻找相应的
        div_list = beautyHtml.find_all('div', attrs={
            'class': 'feeds-item'})

        if div_list:
            for item in div_list:
                # 取得tag属性的值
                href = item.div.h3.a['href']
                # 取得tag的值
                name = item.div.h3.a.text.encode('utf-8')
                reg = '201\d年\d+月'
                regResul = re.findall(reg, name, re.S)
                if len(regResul) != 0:
                    print '___Name:', name
                    print '___ADD', href
                    result.append(href)
    return result
## get all links in the page
link_list = [link.get('href') for link in soup.find_all('a')]
for link in link_list:
    if link is not None and link[:4] == 'http': # Note: lazy evaluation
        external_links.append(link)
[link for link in link_list if l is not None and l.startswith('http')]

网页的结构和re相关文档

各种tag等的含义：https://www.w3schools.com/tags/
大部分的浏览器会将HTML解析成DOM:https://www.w3.org/DOM/
正则表达式：https://regexone.com/ ，https://docs.python.org/3/howto/regex.html
正则表达式测试：http://regexr.com/
代理池使用:/wenshu_scrach/将文件夹和smilerequest.py引入即可（先cd到ip文件夹然后run起来）

定制好的函数

list抓取

def getListData(outputfile, *args):
    url = ''.join(args)
    print url
    html = requests.get(url, headers=headers).text
    beautyHtml = BeautifulSoup(html, 'lxml')

    # 第三步：最重要的定制规则
    # 搜索tag
    tag = beautyHtml.find('div', attrs={'class': 'bottom'})
    tags = beautyHtml.find_all('div')
    tags = beautyHtml.find_all(id='html')
    sub_tags = tag.find_all('div')

    # 获取tag的值
    tag_value = tag.div.string  # type: bs4.element.NavigableString 也是一种unicode, 可以用str() 进行转换
    tag_value = tag.head.text  # type: bs4.element.NavigableString 也是一种unicode, 可以用str() 进行转换

    # 获取property的值
    property_value = tag['class'].string


if __name__ == '__main':
    # 第一步：确定好输出文件
    inputfliePath = '/Users/i309929/Desktop/input.txt'
    outputfile = open(inputfliePath, 'w')

    # 第二步：定制url
    url = 'http://www.aqistudy.cn/historydata/index.php'
    getListData(outputfile, url)

    outputfile.close()

表格抓取

def getExcelFromWebTable(outputSheet, *keyword):
    url = ''.join(keyword)
    print url
    html = requests.get(url, headers=headers).text
    beautyHtml = BeautifulSoup(html, 'lxml')

    tables = beautyHtml.find_all('table')
    if tables:
        print "一共找到 " + str(len(tables)) + " 表格"
        print "默认选取第一个表格......."
        first_table = tables[0]
        trs = first_table.find_all('tr')

        if trs:
            row_count = len(trs)
            # print "行数：" + str(row_count)
            for i in xrange(0, row_count):
                global  start_row
                tr = trs[i]
                tds = tr.find_all('td')
                if tds==None:
                    tds = tr.find_all('th')
                col_count = len(tds)
                # print "列数" + str(col_count)
                for j in xrange(0, col_count):
                    td = tds[j]
                    text = td.text
                    outputSheet.write(start_row, j + 1, label=text)
                start_row = start_row + 1
        else:
            print "表格tr中没有数据"
    else :
        print url + " 中没有表格格式数据"

if __name__ == '__main__':
    '''
    ------------------------------------------------------------------------------------------------
    从text文件中中每一行读取关键字变量进行遍历
    '''
    # inputFilePath = '/Users/i309929/Desktop/cities.txt'
    # queryKeywords = open(inputFilePath, 'r')
    #
    # outputFile = ExcelWrite.Workbook(encoding='utf-8')
    # outputSheet = outputFile.add_sheet("output_sheet", cell_overwrite_ok=True)
    #
    # for keyword in queryKeywords:
    #     global start_row
    #     start_row = start_row + 1
    #     outputSheet.write(start_row, 0, label=keyword)
    #     keyword = string.rstrip(keyword)
    #     print '-------------keyword: ' + str(keyword) + ' ----: '
    #     baseURL = 'http://www.aqistudy.cn/historydata/monthdata.php?city='
    #     getExcelFromWebTable(outputSheet, baseURL, str(keyword))
    #     sleep(1)
    #
    # queryKeywords.close()
    # outputFile.save('/Users/i309929/Desktop/output.xls')

    '''
    ------------------------------------------------------------------------------------------------
    直接进行遍历
    '''
    outputFile = ExcelWrite.Workbook(encoding='utf-8')
    outputSheet = outputFile.add_sheet("output_sheet", cell_overwrite_ok=True)

    getExcelFromWebTable(outputSheet, 'http://tianqihoubao.com/weather/top/beijing.html')

    outputFile.save('/Users/i309929/Desktop/output.xls')