2020-07-21

import requests, bs4, re
from pyquery import PyQuery as pq
import  time
import os
import datetime
headers ={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.9 Safari/537.36'}
No=1  #文档编号

# 封装一个获取新闻内容网址和新闻标题的函数
def index():
    url = 'https://it.sohu.com/?spm=smpc.news-home.header.9.1595315935541HkUEZX9'
    r = requests.get(url, headers=headers,).text
    doc = pq(r)
    a = doc("#main-container > div.z-main-1.clear > div.z-main-1_mid").items('a')
    print("a",a)
    for x in a:
        time.sleep(1)
        print("x", x)
        href = x.attr("href")
        if href.startswith("http")==True:
            pass
        else:
            href = "https:"+x.attr("href")
        print("href",href)
        title = x.text()
        print(title)
        get_news(href, title)

# 封装一个获取内嵌链接地址新闻内容的函数
def get_news(href,title):
    global No  # 声明函数中使用的No是全局变量
    response = requests.get(href,headers=headers).text
    # 数据初始化
    doc =pq(response)
    # 通过类选择器news_txt提取新闻内容
    news = doc(".article").items()
    # 遍历数据
    for x in news:
        # 取出数据中的文本数据，获取到新闻信息
        new = x.text()
        print(new)
    #
        # 获取今日日期，并转换为字符串的形式。以此日期命名建立文件路径
        date = datetime.datetime.now().strftime('%Y-%m-%d') + "搜狐科技新闻" + "//"
        # 判断是否存在此文件夹
        folder = os.path.exists(date)
        # 如果不存在就新建该文件夹
        if not folder:
            os.makedirs(date)
        title=validateTitle(title)
        title=str(No)+"_"+title
        # 打开date文件夹.以"a"追加的方式,编码为"utf-8"保存为txt文件.
        with open(date + "{}.txt".format(title), "a", encoding="utf-8") as f:
            # 将获取到的新闻数据写入
            f.write(new)
            # 关闭读写
            f.close()
        No+=1
#保存文件时候, 去除名字中的非法字符
def validateTitle(title):
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(rstr, "_", title)  # 替换为下划线
    return new_title
if __name__ == '__main__':
    index()