受人之托，需要youtube视频下面的评论数据，由于youtube对于爬虫有限制，因此第一次尝试selenium模拟人工点击爬虫，但爬取到的评论数远远低于youtube上面显示的评论数。后面经过调查发现，Google有自己的youtube data api，方便你对评论数据的需要。
网址：https://developers.google.com/youtube/v3/quickstart/python
第一步：
点击刚才给的网址，注册一个Google cloud platform，操作就按下面图片进行就好。
第二步：
根据主页面网址：https://developers.google.com/youtube/v3/
找到下面的页面：
image.png
第三步：
测试一下
执行execute,查看右边出现的结果，没有报错就说明成功了。
第四步：
下面我用Python代码抓取评论
import os
import numpy as np
import google_auth_oauthlib.flow
import googleapiclient.discovery
import googleapiclient.errors
from googleapiclient.errors import HttpError
import pandas as pd
import json
import socket
import socks
import requests
## 科学上网，你也需要科学使用代理，不然科学不了外网，也许你不会需要。
## headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}
## socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", 1080)
## socket.socket = socks.socksocket
##
scopes = ["https://www.googleapis.com/auth/youtube.force-ssl"]
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
# 这个文件需要自己注册完，自己下载
client_secrets_file = "client_secret_961831598513-urdgliumr9j4ab4g68jtocc30dimqb9g.apps.googleusercontent.com.json"
# Get credentials and create an API client
flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(
client_secrets_file, scopes)
credentials = flow.run_console()
youtube = googleapiclient.discovery.build(
api_service_name, api_version, credentials=credentials)
videoId = '5YGc4zOqozo'
request = youtube.commentThreads().list(
part="snippet,replies",
videoId=videoId,
maxResults = 100
)
response = request.execute()
# print(response)
totalResults = 0
totalResults = int(response['pageInfo']['totalResults'])
count = 0
nextPageToken = ''
comments = []
first = True
further = True
while further:
halt = False
if first == False:
print('..')
try:
response = youtube.commentThreads().list(
part="snippet,replies",
videoId=videoId,
maxResults = 100,
textFormat='plainText',
pageToken=nextPageToken
).execute()
totalResults = int(response['pageInfo']['totalResults'])
except HttpError as e:
print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
halt = True
if halt == False:
count += totalResults
for item in response["items"]:
# 这只是一部分数据，你需要啥自己选就行，可以先打印下你能拿到那些数据信息，按需爬取。
comment = item["snippet"]["topLevelComment"]
author = comment["snippet"]["authorDisplayName"]
text = comment["snippet"]["textDisplay"]
likeCount = comment["snippet"]['likeCount']
publishtime = comment['snippet']['publishedAt']
comments.append([author, publishtime, likeCount, text])
if totalResults < 100:
further = False
first = False
else:
further = True
first = False
try:
nextPageToken = response["nextPageToken"]
except KeyError as e:
print("An KeyError error occurred: %s" % (e))
further = False
print('get data count: ', str(count))
### write to csv file
data = np.array(comments)
df = pd.DataFrame(data, columns=['author', 'publishtime', 'likeCount', 'comment'])
df.to_csv('google_comments.csv', index=0, encoding='utf-8')
### write to json file
result = []
for name, time, vote, comment in comments:
temp = {}
temp['author'] = name
temp['publishtime'] = time
temp['likeCount'] = vote
temp['comment'] = comment
result.append(temp)
print('result: ', len(result))
json_str = json.dumps(result, indent=4)
with open('google_comments.json', 'w', encoding='utf-8') as f:
f.write(json_str)
f.close()
if __name__ == "__main__":
main()
