# 豆瓣TOP250
# 0. 封面
# 1.序号
# 2.电影名
# 3.详细链接
# 4.电影信息
# 5.评级 9.7
# 6.多少人评价
# 7.引述
# https://movie.douban.com/top250
from xpinyin import Pinyin
import requests
from lxml import html
import pandas as pd

def spider():
  movie_info_list_TOP250 = []

  for i in range(0,250,25):

    url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
    print(url)
    print('您要查询的目标站点是', url)
    print('爬虫进行中,请稍后......')
    # 请求头信息,目的是伪装浏览器进行爬虫
    headers = {
        "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'
        }  # 伪装成浏览器
    # 提取网页源代码
    response = requests.get(url, headers=headers)
    html_data = response.text
    # print(html_data)
    # 提取我们想要的内容
    selector = html.fromstring(html_data)
    ol_list = selector.xpath('//ol[@class="grid_view"]/li')
    # print('ol',len(ol_list))

    movie_info_list = []
    for li in ol_list:
        # 获取电影海报
        movie_img = li.xpath('div[1]/div[1]/a/img/@src')[0]
        movie_img = '没查到数据' if len(movie_img) == 0 else movie_img
        # print(movie_img)

        # 获取排名序号
        movie_number = li.xpath('div[1]/div[1]/em[1]/text()')[0]
        movie_number = '没查到数据' if len(movie_number) == 0 else movie_number
        # print(movie_number)

        # 获取电影名字
        movie_name = li.xpath('div[1]/div[2]/div[1]/a/span[1]/text()')[0]
        movie_name = '没查到数据' if len(movie_name) == 0 else movie_name
        # print(movie_name)

        # 获取电影详细链接
        movie_link = li.xpath('div[1]/div[2]/div[1]/a/@href')[0]
        movie_link = '没查到数据' if len(movie_link) == 0 else movie_link
        # print(movie_link)

        # 获取电影信息
        movie_info = li.xpath('div[1]/div[2]/div[2]/p[1]/text()')[0]+li.xpath('div[1]/div[2]/div[2]/p[1]/text()')[1]
        movie_info = '没查到数据' if len(movie_info) == 0 else movie_info
        movie_info =str(movie_info.replace('\n',''))
        movie_info =str(movie_info.replace('\xa0',''))
        movie_info = str(movie_info.replace(' ', ''))
        # print(movie_info)

        # 获取电影评级
        movie_rating_number = li.xpath('div[1]/div[2]/div[2]/div[1]/span[2]/text()')[0]
        movie_rating_number = '没查到数据' if len(movie_rating_number) == 0 else movie_rating_number
        # print(movie_rating_number)

        # 获取评价人数
        movie_voting_number = li.xpath('div[1]/div[2]/div[2]/div[1]/span[4]/text()')[0]
        movie_voting_number = '没查到数据' if len(movie_voting_number) == 0 else movie_voting_number
        movie_voting_number =int(movie_voting_number.replace('人评价',''))
        # print(movie_voting_number)

        # 获取电影引述
        movie_quote = li.xpath('div[1]/div[2]/div[2]/p[2]/span[1]/text()')
        movie_quote = '没查到数据' if len(movie_quote) == 0 else movie_quote
        # print(movie_quote)

        movie_info_list.append({
            "movie_img":movie_img,
            "movie_number":movie_number,
            "movie_name":movie_name,
            "movie_link":movie_link,
            "movie_info":movie_info,
            "movie_rating_number":movie_rating_number,
            "movie_voting_number":movie_voting_number,
            "movie_quote":movie_quote
        })


    # 遍历
    for movie in movie_info_list:
        # print(movie)
        movie_info_list_TOP250.append(movie)

        with open('./douban_img/{}.jpg'.format(movie['movie_name']), 'wb') as f:
            f.write(requests.get(movie['movie_img']).content)

  print(movie_info_list_TOP250)

  pd.DataFrame(movie_info_list_TOP250).to_csv('movie_info_list_TOP250.csv')

spider()