在参照豆瓣影评的基础上写的抓取澎湃新闻列表代码

# coding = utf-8
import requests
from lxml import etree
import time
import random
import csv
import codecs

data = []

def getPP(page):
    url = 'https://www.thepaper.cn/load_chosen.jsp?nodeids=25949&topCids=2123127,2122823,2120622,2123384,&pageidx=%d'%(page)
    response = requests.get(url)
    response.encoding = 'utf-8'
    response = etree.HTML(response.content,parser=etree.HTMLParser(encoding='utf-8'))#添加默认编码

    for i in range(1,21):
        title1 = response.xpath('//body/div[%d]/h2/a'%(i))#获取title
        url1 = response.xpath('//body/div[%d]/h2/a'%(i))#获取url
        summary1 = response.xpath('//body/div[%d]/p'%(i))#获取summary
        tag1 = response.xpath('//body/div[%d]/div[2]/a'%(i))#获取文章分类
        time1 = response.xpath('//body/div[%d]/div[2]/span[1]'%(i))#获取发布时间
        rtimes1 = response.xpath('//body/div[%d]/div[2]/span[2]'%(i))#获取回复数

        title_element = title1[0].text.encode('utf-8')

        url_element = url1[0].attrib['href']
        url_element = 'https://www.thepaper.cn/' + url_element#完善链接

        summary_element = summary1[0].text.encode('utf-8')
        summary_element = summary_element.replace(' ','')#去空格
    
        tag_element = tag1[0].text.encode('utf-8')
        time_element = time1[0].text.encode('utf-8')

        if len(rtimes1) == 0:
            rtimes_element = 0
        else:
            rtimes_element = rtimes1[0].text#有些回复数为空，rtimes[0]不存在，会报错，用if函数避免这个问题

        data.append([title_element,url_element,summary_element,tag_element,time_element,rtimes_element])


for i in range(1,3):#抓2页
    getPP(i)
    time.sleep(random.uniform(3,5))


with open ('getPP.csv','wb') as f:
    f.write(codecs.BOM_UTF8)
    writer = csv.writer(f)
    writer.writerow(['title','url','summary','tag','publishtime','rtimes'])
    for k in data:
        writer.writerow(k)