直接上代码
#!/usr/bin/python3
# -*- coding: UTF-8 -*-
import urllib.request
import urllib.error
import re
import csv
import time
global_row = 0
'''
需求：
爬取京东商品数据，以‘java’关键字为例。要求使用最基础的urllib和re库。
需要保存书名，价格，评论数，出版社等信息。
实现：
找出页面规律如下
一、每页显示60个商品，但分为两部分。
1. 每页前30个商品，通过search.jd.com/Search?keyword=java接口获取
每页后30个商品，通过search.jd.com/s_new.php?keyword=java接口获取，这个接口是个XHR请求，通过Chrome的开发者选项可以看出。模拟人向下滚动页面
2. 每个接口有两个关键的参数'page'和's'
参数'page'好理解，就是页面数。对第一个接口变化规律为1,3,5...，对第二个接口变化规律为2,4,6...；
参数's'我猜应该是start，也就是起始商品的索引。实际测试时，这个参数变化不是很规律，这里强制设置每页30个，
这样对第一个接口变化规律1,61,121...，对第二个接口变化规律为31,91,151...
3. 对于获取不到商店名称的情况，再构造chat1.jd.com/api/checkChat请求，获取商店名称。（但还是存在获取不到的情况，页面也不能显示）
'''
def crawl_page(n, csv_writer):
#上半部分网页
top_url = 'http://search.jd.com/Search?keyword=java&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&page=%d&s=%d&click=0' % (2*n- 1, 1 + 60*(n - 1))
content = get_page_content(top_url, True)
parse_content(content, n, csv_writer)
#下半部分网页（需要发送XHR请求的）
current_time = '%.5f' % time.time()
url = 'http://search.jd.com/s_new.php?keyword=java&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&page=%d&s=%d&scrolling=y&log_id=%s' %(2*n, 31 + 60*(n - 1), current_time)
content = get_page_content(url, False)
parse_content(content, n, csv_writer)
def get_page_content(url, is_top):
req = urllib.request.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3192.0 Safari/537.36')
if not is_top:
req.add_header('Referer', 'http://search.jd.com/Search?keyword=java&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&click=0')
req.add_header('X-Requested-With', 'XMLHttpRequest')
response = urllib.request.urlopen(req)
return response.read().decode('utf8')
def get_seller(shop_url):
req = urllib.request.Request(shop_url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3192.0 Safari/537.36')
req.add_header('Referer', 'http://search.jd.com/Search?keyword=java&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&click=0')
response = urllib.request.urlopen(req)
content = response.read().decode('utf8')
print('content', content)
seller = re.search(r'"seller":"(.*?)"', content)
if seller:
#获得的值为类似于"\u58a8\u9a6c\u56fe\u4e66\u65d7\u8230\u5e97"，需要转码
return seller.group(1).encode('latin-1').decode('unicode_escape')
return ''
def parse_content(content, n, csv_writer):
li_list = re.findall(r'<li data-sku="\d*" class="gl-item">.*?</li>', content, re.DOTALL)
for li in li_list:
#书名
name_match = re.search(r'<div class="p-name p-name-type-2">.*?<em>(.*?)</em>.*?</div>', li, re.DOTALL)
name = ''
if name_match:
name = re.sub(r'<.*?>', '', name_match.group(1))
else:
print('page %d, name is empty' % n)
#价格
price_match = re.search(r'<div class="p-price">.*?<i>(.*?)</i>.*?</div>', li, re.DOTALL)
price = ''
if price_match:
price = price_match.group(1)
else:
print('page %d, price is empty' % n)
#评论数
commit_match = re.search(r'<div class="p-commit">.*?<a .*?>(.*?)</a>.*?</div>', li, re.DOTALL)
commit = ''
if commit_match:
commit = commit_match.group(1)
else:
print('page %d, price is empty' % n)
#出版社（商店）
shop_match = re.search(r'<div class="p-shop" [^>]*>\s*<span [^>]*><a [^>]*>(.*?)</a>.*?</div>', li, re.DOTALL)
shop = ''
if shop_match:
shop = shop_match.group(1)
else:
uid = re.match(r'<li data-sku="(\d*)" class="gl-item">.*?</li>', li, re.DOTALL).group(1)
seller_url = 'https://chat1.jd.com/api/checkChat?pid=' + uid + '&returnCharset=utf-8'
shop = get_seller(seller_url)
if not shop:
print('page %d, shop is empty' % n)
global global_row
global_row = global_row + 1
csv_writer.writerow([str(global_row), name, price, commit, shop])
def main():
with open('output.csv', 'w', encoding='gbk') as f:
csv_writer = csv.writer(f)
csv_writer.writerow(['序号', '书名', '价格', '评论数', '出版社'])
for i in range(1, 6):
crawl_page(i, csv_writer)
if __name__ == '__main__':
main()
