request_html常用方法
from requests_html import HTMLSession
session = HTMLSession()
r = session.get('https://news.cnblogs.com/n/recommend')
news = r.html.find('h2.news_entry > a')
for new in news:
print(new.text)
print(new.absolute_links)
加载js，下载漫画图片
%%time
for i in range(1, 16):
r = session.get('http://www.gugu5.com/n/14178/556176.html?p=%s'%i)
r.html.render()
src = r.html.find('#qTcms_pic', first=True).attrs['src']
print(src)
display(Image(url=src))
http://html.python-requests.org/
小例子
from requests_html import HTMLSession
from IPython.display import display, Image
session = HTMLSession()
%%time
for i in range(1, 15):
r = session.get('http://www.gugu5.com/n/14178/531259.html?p=%s'%i)
r.html.render()
src = r.html.find('#qTcms_pic', first=True).attrs['src']
display(Image(url=src))
print('第%s页'%i)
爬取猫眼top100电影
import requests
from bs4 import BeautifulSoup
import csv
def get_page(url):
headers = {
'Host': 'maoyan.com',
'Referer': 'http://maoyan.com/board',
'User-Agent': 'Mozilla/5.0'
}
r = requests.get(url, headers=headers)
return r.text
def parse(content):
movies = []
soup = BeautifulSoup(content, 'lxml')
dl = soup.find('dl', attrs={'class': 'board-wrapper'})
for dd in dl.find_all('dd'):
rank = dd.find('i').text
title = dd.find('div', attrs={'class': "movie-item-info"}).find('a').text
stars = dd.find('p', attrs={'class': 'star'}).text.strip().split('：')[-1].strip()
releasetime = dd.find('p', attrs={'class': 'releasetime'}).text.strip().split('：')[-1].strip()
score = dd.find('p', attrs={'class': 'score'}).text
movie = [rank, title, stars, releasetime, score]
movies.append(movie)
return movies
def write(rows, file):
with open(file, 'a', newline='') as f:
writer = csv.writer(f)
writer.writerows(rows)
def main():
url = 'http://maoyan.com/board/4?offset='
for i in range(10):
print(url + str(i*10))
content = get_page(url + str(i*10))
movies = parse(content)
# print(movies)
write(movies, '1.csv')
if __name__ == '__main__':
main()
