xpath，selenium爬取腾讯视频下的乒乓网乒乓球教学视频官方频道播放量
页面：
http://v.qq.com/vplus/pingpang/videos
-
爬取的播放量是这样的
-
下方标签一看
点击以后没有激活XHR，也没有改变页面的url，但是激活了javascript，我在源代码里找到了api。
js
XHR没有
这个API太复杂了，jQuery都不知道怎么构造
这个api请求如下：
难以构造，于是结论：上selenium吧
- 代码
from selenium import webdriver
import time
import datetime
import pandas as pd
set_end_time = '2019-03-31' #含边界
set_start_time = '2018-01-01' #含边界
url = 'http://v.qq.com/vplus/pingpang/videos'
my_driver = webdriver.Chrome(executable_path='./chromedriver')
my_driver.get(url)
now = datetime.datetime.now().strftime('%Y-%m-%d %H时%M分')
def qian2time(delay):
if '小时' in delay:
delay = -int(delay.split('小时')[0])
return (datetime.datetime.now() + datetime.timedelta(hours=delay)).strftime('%Y-%m-%d')
elif '分钟' in delay:
delay = -int(delay.split('分钟')[0])
return (datetime.datetime.now() + datetime.timedelta(minutes=delay)).strftime('%Y-%m-%d')
elif '秒' in delay:
delay = -int(delay.split('秒')[0])
return (datetime.datetime.now() + datetime.timedelta(seconds=delay)).strftime('%Y-%m-%d')
else:
return delay
def removewan(view):
if "万" in view:
return str(int(float(view.split('万')[0])*10000))
else:
return view
next_page_xpath = './/a[@class = "page_btn page_num current"]/following-sibling::a[1]'
duration_xpath = './/em[@class="mask_txt"]'
url_xpath = '//*[@id="videolst_cont"]/li/a'
title_xpath = '//*[@id="videolst_cont"]/li/a'
view_xpath = '//*[@id="videolst_cont"]/li/div/span/span'
upload_xpath = './/span[@class="figure_info_time"]'
durations,urls,titles,views,upload_times=[],[],[],[],[]
while True:
durations_each24 = [i.text for i in my_driver.find_elements_by_xpath(duration_xpath)]
urls_each24 = [i.get_attribute('href') for i in my_driver.find_elements_by_xpath(url_xpath)]
titles_each24 = [i.get_attribute('title') for i in my_driver.find_elements_by_xpath(title_xpath)]
views_each24 = [removewan(i.text) for i in my_driver.find_elements_by_xpath(view_xpath)]
upload_times_each24 = [qian2time(i.text) for i in my_driver.find_elements_by_xpath(upload_xpath)]
durations += durations_each24
urls += urls_each24
titles += titles_each24
views += views_each24
upload_times += upload_times_each24
if upload_times_each24[-1]< set_start_time:
break
else:
my_driver.find_element_by_xpath(next_page_xpath).click()
time.sleep(5)
excel_frames = pd.DataFrame({
'上传时间':upload_times,
'观看数':views,
'题目':titles,
'时长':durations,
'直链':urls
})
excel_frames = excel_frames[excel_frames['上传时间']<=set_end_time]
excel_frames = excel_frames[excel_frames['上传时间']>=set_start_time]
excel_frames.sort_values("上传时间",inplace=True)
excel_frames.to_csv(f'./{set_start_time}~{set_end_time}数据 生成时间{now}.csv' ,index = False)
my_driver.close()
print('完成')
运行结果：
踩过的坑：
- 分页问题
- selenium的xpath必须定位到元素，即不能有
/@value或者/text()
