功能描述

• 目标：获取淘宝搜索页面的信息，提取其中的关键信息。
• 理解：
  淘宝的搜索接口
  翻页的处理
• 技术路线：
  requests-re
  （使用正则表达式来提取信息）

import requests
import re

def getHTMLText(url):
    kv = {'cookie' : 'miid=413241011674319298; cna=bzhfE71XryECATo6NDSjkj6C; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; tracknick=%5Cu9759%5Cu5019%5Cu7075%5Cu5F52520370; tg=0; enc=yJqBSdMue2TKATKCxQ5nNmbUHL82jBeGpw%2BhqDIRlcL%2FVHFFQQ9sFshL4l8YJoS%2F5g%2F%2BV2gV3NoT9YYn30kbnQ%3D%3D; _m_h5_tk=b1f75d048d6aeed57d814f7a50d66641_1586700106396; _m_h5_tk_enc=0bdf063ab77a2936ccb2e059d6db64c0; t=8e80b386bca784fb223e0f83f73cebda; v=0; cookie2=1847beaf312bd87963ff8272d0b8d0b7; _tb_token_=5e7353571ad1e; alitrackid=www.taobao.com; _samesite_flag_=true; sgcookie=EKJ4L5czU7sbync8UgcsK; unb=2355217709; uc3=lg2=U%2BGCWk%2F75gdr5Q%3D%3D&id2=UUtO%2FVqSVpj5eg%3D%3D&nk2=3Wy5AkoyvJZ%2BX5%2FSa5Y%3D&vt3=F8dBxdGLZSg9H8AFy1U%3D; csg=5b481f1c; lgc=%5Cu9759%5Cu5019%5Cu7075%5Cu5F52520370; cookie17=UUtO%2FVqSVpj5eg%3D%3D; dnk=%5Cu9759%5Cu5019%5Cu7075%5Cu5F52520370; skt=f9711d0519a0fdcd; existShop=MTU4NzE3NzM4OA%3D%3D; uc4=id4=0%40U2l0uKuhTOtMhU1bieJbpIsIPhXN&nk4=0%403%2BcBjITjn0CCuBQVdwk4JLVExOAMNgh%2FfA%3D%3D; _cc_=U%2BGCWk%2F7og%3D%3D; _l_g_=Ug%3D%3D; sg=092; _nk_=%5Cu9759%5Cu5019%5Cu7075%5Cu5F52520370; cookie1=V33DIdASji0W00xDbJVy0fvJaNbu6aAZ2v0LJi537Ss%3D; JSESSIONID=531AFDD871D866503438DC04A71EB67E; lastalitrackid=login.taobao.com; tfstk=cQ_CByvhtvDCt6bE7TNwYOO1swKPawr66k9NOGQhIF1rwAfeBsjmuKvLcgDsWnd1.; uc1=cookie16=VFC%2FuZ9az08KUQ56dCrZDlbNdA%3D%3D&cookie21=URm48syIYB3rzvI4Dim4&cookie15=VT5L2FSpMGV7TQ%3D%3D&existShop=false&pas=0&cookie14=UoTUPc3rdfGOww%3D%3D; mt=ci=45_1; isg=BA8PUHFyjZyKVomRPVnYAHqfnqMZNGNWFeQ1eyEcQn6L8C7yKASepoeq8iDO7DvO; l=eBPjk4lnQH8FVhdQBO5wlJ4FzbbTCIRb8uPr-CjsIIHca1oCtebG9NQcQeZDSdtjgtCXyeKPFFZ7gRB4r34dTxDDBexrCyConxvO.',
          'user-agent' : 'Mozilla/5.0'}

    try:
        r = requests.get(url, headers = kv, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


#关键代码区：用正则表达式实现所需信息的提取
def parsePage(ilt, html):
    try:
    #这些正则表达式都没有加转义字符‘\’
        plt = re.findall(r'"view_price":"[\d.]*"', html) #找到所有的价格信息
        tlt = re.findall(r'"raw_title":".*?"', html)
        flt = re.findall(r'"view_fee":"[\d.]*"', html)
        slt = re.findall(r'"view_sales":".*?"', html)
    #这一步完成，提取出来的信息都是键值对形式？？？（如："view_price":"127.00"），所以要做进一步的处理，只取出值
        for i in range(len(plt)):
    #  eval()函数
            price = eval(plt[i].split(':')[1])
            title = eval(tlt[i].split(':')[1])
            fee = eval(flt[i].split(':')[1])
            sales = eval(slt[i].split(':')[1])
   
    #将列表元素（[price, fee, sales, title]），存入ilt列表中
            ilt.append([price, fee, sales, title])
    except:
        print("")


#打印爬取到的信息，完成输出格式的定义
def printGoodsList(ilt):
    tplt = "{:4}\t{:8}\t{:8}\t{:10}\t{:16}"
    print(tplt.format("序号", "价格", "邮费", "已购人数", "商品名称"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count, g[0], g[1], g[2], g[3]))


def main():
    goods = '子时当归同款茶壶'   #淘宝检索关键字
    depth = 2  #需求页数
    start_url = 'https://s.taobao.com/search?q=' + goods
    infoList = []  #定义一个列表，保存爬取回来的数据
    for i in range(depth):
        try:
            url = start_url + '&s=' + str(44 * i)
            html = getHTMLText(url)
            parsePage(infoList, html)
        except:
            continue
    printGoodsList(infoList)
main()

• 总结：

1. 当爬不出数据，并且程序并无报错时，查看一下网页的提交数据（headers）是不是包含cookie和user-agent。
2. 正则表达式的使用一定要规范。
3. 翻页的实现主要是靠：观察网页url几页之间参数的不同，合理假设大胆判断。