Crawler Demo 04

from bs4 import BeautifulSoup
from urllib.request import urlopen
import re
import random


base_url = "https://baike.baidu.com"
his = ["/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/5162711"]

for i in range(20):
    url = base_url + his[-1]
    html = urlopen(url).read().decode('utf-8')
    soup = BeautifulSoup(html, features='lxml')
    print(i, soup.find('h1').get_text(), '    url: ', base_url + his[-1])
    sub_urls = soup.find_all("a", {"target": "_blank", "href": re.compile("/item/(%.{2})+$")})
    if len(sub_urls) != 0:
        his.append(random.sample(sub_urls, 1)[0]['href'])
    else:
        his.pop()


# 0 网络爬虫     url:  https://baike.baidu.com/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/5162711
# 1 分布式搜索引擎     url:  https://baike.baidu.com/item/%E5%88%86%E5%B8%83%E5%BC%8F%E6%90%9C%E7%B4%A2%E5%BC%95%E6%93%8E
# 2 网络爬虫     url:  https://baike.baidu.com/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/5162711
# 3 超文本     url:  https://baike.baidu.com/item/%E8%B6%85%E6%96%87%E6%9C%AC
# 4 文本     url:  https://baike.baidu.com/item/%E6%96%87%E6%9C%AC
# 5 苏联     url:  https://baike.baidu.com/item/%E5%89%8D%E8%8B%8F%E8%81%94
# 6 匈牙利十月事件     url:  https://baike.baidu.com/item/%E5%8C%88%E7%89%99%E5%88%A9%E5%8D%81%E6%9C%88%E4%BA%8B%E4%BB%B6
# 7 多瑙河     url:  https://baike.baidu.com/item/%E5%A4%9A%E7%91%99%E6%B2%B3
# 8 奥地利     url:  https://baike.baidu.com/item/%E5%A5%A5%E5%9C%B0%E5%88%A9
# 9 多瑙河     url:  https://baike.baidu.com/item/%E5%A4%9A%E7%91%99%E6%B2%B3
# 10 韦斯     url:  https://baike.baidu.com/item/%E9%9F%A6%E6%96%AF
# 11 多瑙河     url:  https://baike.baidu.com/item/%E5%A4%9A%E7%91%99%E6%B2%B3
# 12 最大水头     url:  https://baike.baidu.com/item/%E6%9C%80%E5%A4%A7%E6%B0%B4%E5%A4%B4
# 13 多瑙河     url:  https://baike.baidu.com/item/%E5%A4%9A%E7%91%99%E6%B2%B3
# 14 温带海洋性气候     url:  https://baike.baidu.com/item/%E6%B8%A9%E5%B8%A6%E6%B5%B7%E6%B4%8B%E6%80%A7%E6%B0%94%E5%80%99
# 15 多瑙河     url:  https://baike.baidu.com/item/%E5%A4%9A%E7%91%99%E6%B2%B3
# 16 汛期     url:  https://baike.baidu.com/item/%E6%B1%9B%E6%9C%9F
# 17 黑龙江     url:  https://baike.baidu.com/item/%E9%BB%91%E9%BE%99%E6%B1%9F
# 18 额木尔河     url:  https://baike.baidu.com/item/%E9%A2%9D%E6%9C%A8%E5%B0%94%E6%B2%B3
# 19 大兴安岭     url:  https://baike.baidu.com/item/%E5%A4%A7%E5%85%B4%E5%AE%89%E5%B2%AD