获取小说各章节网址
import requests
from bs4 import BeautifulSoup
# 获取章节页面
def get_url(url):
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36'}
response = requests.get(url, headers = headers)
response = response.text
soup = BeautifulSoup(response, 'lxml')
text = soup.dl.children
return text
# 获取章节内容
def get_html(text):
for child in text:
try:
html = "https://www.biquge.com.cn" + child.a.get('href')
print(html)
except AttributeError:
pass
url = "https://www.biquge.com.cn/book/15517/"
text = get_url(url)
get_html(text)
