爬取百度贴吧的帖子

要解析下面的 HTML, 提取到 div 中的文本:

<div id="post_content_91765531755" class="d_post_content j_d_post_content  clearfix">            楼主现在iOS10么</div>

定位到 @id 以 post_content 开头并且 @class为 d_post_content j_d_post_content clearfix 的 div。

>>> import lxml
>> html = requests.get('http://tieba.baidu.com/p/4609646212')
>>> content = etree.HTML(html.text)
>>> content = content.xpath('//div[starts-with(@id, "post_content") and contains(@class,"d_post_content j_d_post_content  clearfix")]')

starts-with(@attr, "xxxx") 函数, 以 xxxx 开头的 attr 属性。
contains(@attr, "xxxx") 函数, 精确含有值为 xxxx 的属性。
and, 两个函数都为真时, 则返回过滤后的元素。

爬取百度贴吧里面的帖子, 爬取字段为「回帖日期」、「回帖人」、「回帖内容」:

# -*- coding:utf-8 -*-

from lxml import etree
from multiprocessing.dummy import Pool as ThreadPool
import requests
import json

def spider(url):
    # test_url = 'http://tieba.baidu.com/p/4609646212'
    html = requests.get(url)
    selector = etree.HTML(html.text)
    # 获取每个内容块
    content_field = selector.xpath('//div[@class="l_post j_l_post l_post_bright  "]')
    reply = {}
    for each_content in content_field:
        reply_info = json.loads(each_content.xpath('@data-field')[0])
        author = reply_info['author']['user_name']
        reply_time = reply_info['content']['date']
        content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[starts-with(@id, "post_content") \
                                        and contains(@class,"d_post_content j_d_post_content  clearfix")]')
        #content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[@class="d_post_content j_d_post_content  clearfix"]')
        print(author)
        print(reply_time)
        print(content[0].xpath('string(.)').replace(' ', ''))
        print('----------------------------------------------------')
        reply['reply_author'] = author
        reply['reply_content_time'] = reply_time
        reply['reply_content'] = content[0].xpath('string(.)').replace(' ', '')
        savetofile(reply)


def savetofile(dict):
    f.writelines(u'回帖时间:' + str(dict['reply_content_time']) + "\n")
    f.writelines(u'回贴人:'   + dict['reply_author'] + "\n")
    f.writelines(u'回帖内容:' + dict['reply_content'] + "\n")
    f.writelines("\n\n")


if __name__ == '__main__':
    pool = ThreadPool(4) # 使用 4 核 cpu
    page = []
    base_url = 'http://tieba.baidu.com/p/4609646212?pn='
    f = open("result.txt", "a", encoding='utf-8') # 将结果写入文件

    [page.append(base_url + str(i)) for i in range(1, 21)]

    result = pool.map(spider, page)
    pool.close()
    pool.join()
    f.close()

注意，元素的定位一定要精确, 不然会发生报错。

爬取百度贴吧的帖子

推荐阅读更多精彩内容