2020-05-22：selenium 爬取微博热搜榜单内容，并分类存储到文本文件

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

hots_content=''

with open("d:\weibo.txt", 'w')as f:

driver = webdriver.Chrome()

driver.get('https://m.weibo.cn/')

# 点击“大家都在搜”

driver.find_element_by_class_name('m-search').click()

time.sleep(1)

# 定位热搜榜所在的大标签,使用了显式等待

hotlist_all = WebDriverWait(driver,5,0.5).until(EC.visibility_of_element_located((By.CLASS_NAME, "m-col-2")))

#hotlist_all2 = driver.find_element_by_class_name('m-col-2')

# 找出热搜榜大标签里的所有元素

hoteles = hotlist_all.find_elements_by_class_name('m-item-box')

# 找到“微博热搜榜”并点击

hotlist = hoteles[-1]

hotlist.click()

# 显式等待，定位实时热点，每分钟刷新页面部分

all_hots = WebDriverWait(driver, 5, 0.5).until(EC.visibility_of_all_elements_located((By.CLASS_NAME, "card4")))

#all_hots = driver.find_elements_by_class_name('card4')

for hotin all_hots:

# 获取热搜标题

hot_summary = hot.find_element_by_class_name('m-text-cut').text

# 获取每条热搜点击率

hot_click_amount = hot.find_element_by_xpath(

'//*[@id="app"]/div[1]/div[1]/div[2]/div/div/div[3]/div/div/div/div/span[2]/span[2]').text

icon = hot.find_elements_by_class_name("m-link-icon")

if icon:

img = icon[0].find_element_by_tag_name('img')

src = img.get_attribute('src')

if "hot" in src:

hottag ='热'

hots =f'{hot_summary, hot_click_amount, hottag}'

print(hots)

hots_content += hots +'\n'

elif "recom" in src:

hottag ='荐'

hots =f'{hot_summary, hot_click_amount, hottag}'

print(hots)

hots_content += hots +'\n'

elif "new" in src:

hottag ='新'

hots =f'{hot_summary, hot_click_amount, hottag}'

print(hots)

hots_content += hots +'\n'

elif "fei" in src:

hottag ='沸'

hots =f'{hot_summary, hot_click_amount, hottag}'

print(hots)

hots_content += hots +'\n'

driver.quit()

f.write(hots_content)

最后打印结果：

('两会2020', '1205575', '热')

('建议将HPV疫苗纳入国家免疫规划', '1205575', '沸')

('中国不存在隐性军费问题', '1205575', '新')

('清华学霸放弃保研成为女特种兵', '1205575', '沸')

('建议取消生育三孩以上处罚', '1205575', '沸')

('计划今明两年制定修改法律17部', '1205575', '新')

('建议离婚过错方少分或不分财产', '1205575', '沸')

('郑恺苗苗结婚', '1205575', '沸')

('张宝艳建议保护离异家庭儿童亲情权', '1205575', '新')

('民法典草案累计收到42.5万人102万条意见', '1205575', '新')

('张驰温婉', '1205575', '热')

('孙耀威给老婆转5201314元', '1205575', '热')

('张艺兴骑马舞', '1205575', '新')

('程晓玥', '1205575', '热')

('GAI方言RAP', '1205575', '新')

('俄女排名教将申请吉尼斯纪录', '1205575', '新')

('老师改编量子力学版消愁', '1205575', '新')

('建议将欺诈发行罪最高刑提至无期', '1205575', '新')

('KPL常规赛收官大战', '1205575', '新')

最后编辑于：2020.05.22 00:19:46