python-基础爬虫（爬取ajax接口后整理数据到xlsx中）

一、爬取的网页截图

image.png

二、通过爬虫生成的xlsx截图

hahow课程网页爬取ajax的接口处理数据后整理到xlsx中.png

三、代码

import requests
import pandas as pd

# 设置url地址
url = "xxx"

# 设置headers模拟用户端访问接口
headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"}

# 接收响应信息
response = requests.get(url, headers=headers)

# 处理数据
course_list = []
if response.status_code == 200:
    # 成功读取网页
    data = response.json()["data"]
    # print(data)
    for info in data:
        course_data = [
            info["title"],
            info["price"],
            info["averageRating"],
            info["numSoldTickets"]
        ]
        # 处理成这种是因为pandas需要列表
        course_list.append(course_data)
    # 写入xlsx表格中
    pf = pd.DataFrame(course_list, columns=["课程标题","课程价格","课程评分","课程售卖数量"])
    pf.to_excel("new_course.xlsx", index = False, engine="openpyxl")
    print("Save!")
else:
    print("网页读取失败")