selenium库

1. 安装

首先安装库，pip就行

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple selenium

然后安装浏览器驱动，博主我使用的是Chrome，首先查看自己的Chrome浏览器版本，地址栏输入

chrome://version/

2. 爬取斗鱼直播页面信息

最近学的东西颇杂，做笔记太烦了，直接写项目

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By

import json


def browser():
    """设置无头浏览器，使爬取时不用打开浏览器"""
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    browser = webdriver.Chrome(options=chrome_options)
    return browser


def get_data_list(browser, url):
    """
    获取页面所有直播间信息，返回列表
    :return:
    """
    browser.get(url)
    wait = WebDriverWait(browser, 10)
    wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'ListFooter')))
    data_list = browser.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li')
    return len(data_list)


def generator_data(browser):
    """子生成器，取出文本内容"""
    while True:
        i = yield
        if i is None:
            break
        print('正在爬取第{}条数据'.format(i))
        title = catch(browser, i, 'div/a/div[2]/div/h3')
        category = catch(browser, i, 'div/a/div[2]/div[1]/span')
        owner = catch(browser, i, 'div/a[1]/div[2]/div[2]/h2')
        hots = catch(browser, i, 'div/a/div[2]/div[2]/span')
        data = {
            'title': title,
            'category': category,
            'owner': owner,
            'hots': hots
        }
        # print(data)
        return data


def write(browser):
    """委派生成器，从子生成器取出内容，然后写入文件"""
    while True:
        # next(get_data_list())
        data = yield from generator_data(browser)
        json_data = json.dumps(data, ensure_ascii=False) + ',\n'
        with open('douyuSpider.json', 'a+', encoding='utf-8') as f:
            print('正在写入', json_data)
            f.write(json_data)


def catch(browser, n, path):
    """抓取内容"""
    data = browser.find_element_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]/{}'.format(n, path))
    return data.text


if __name__ == '__main__':
    url = 'https://www.douyu.com/directory/all'
    browser = browser()
    num = get_data_list(browser, url)
    w = write(browser)
    next(w)  # 激活生成器
    for i in range(num):
        w.send(i+1)

这里博主我是为了顺便复习生成器的知识，就没按正常套路来写代码