简书7日热门文章数据分析+更新推送(持续更新···)

简书7日热门文章数据分析+更新推送(持续更新···)

工作流程

  1. 爬取简书7日热门文章;
  2. 存入MongoDB数据库;
  3. 文章简单的数据分析;
  4. 再次爬取简书7日热门文章;
  5. 判断新爬取的文章是否与已爬取的内容重复;
  6. 如文章不重复,发送邮件提醒;

URL结构

参考代码

爬虫部分代码

import requests
from lxml import etree

homepage_url = 'http://www.jianshu.com'
base_url = 'http://www.jianshu.com/trending/weekly'


def get_html(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    if resp.status_code == 200:
        return resp.text
    else:
        return None


# 解析首页数据
def parse_first_html(html, db):
    root = etree.HTML(html)
    articles = root.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "content", " " ))]')
    seen_snote_ids = root.xpath('//ul[@class="note-list"]/li/@data-note-id')
    # print(seen_snote_ids)
    for article in articles:
        author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
        author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
        title = ''.join(article.xpath('a[@class="title"]/text()'))
        article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
        abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
        read_and_comments = article.xpath('div[@class="meta"]/a/text()')
        like_and_pay = article.xpath('div[@class="meta"]/span/text()')
        # 数据处理
        read_and_comments = ''.join(read_and_comments).strip().split()
        like_and_pay = ''.join(like_and_pay).strip().split()
        # read_count = meta.xpath('a/text()')
        # print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
        # print(''.join(like_and_pay).strip().split())
        data ={
            'author': author,
            'author_url': author_url,
            'title': title,
            'article_url': article_url,
            'abstract': abstract,
            'read_and_comments': read_and_comments,
            'like_and_pay': like_and_pay
        }
        if not is_data_exist(db, data):
            print('保存新数据')
            save_data(db, data)
        else:
            print('数据已存在,更新数据')
            update_data(db, data)
    return seen_snote_ids


def parse_other_html(html, db):
    root = etree.HTML(html)
    articles = root.xpath('//li/div[@class="content"]')
    seen_snote_ids = root.xpath('//li/@data-note-id')
    # print(seen_snote_ids)
    for article in articles:
        author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
        author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
        title = ''.join(article.xpath('a[@class="title"]/text()'))
        article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
        abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
        read_and_comments = article.xpath('div[@class="meta"]/a/text()')
        like_and_pay = article.xpath('div[@class="meta"]/span/text()')
        # 数据处理
        read_and_comments = ''.join(read_and_comments).strip().split()
        like_and_pay = ''.join(like_and_pay).strip().split()
        # read_count = meta.xpath('a/text()')
        # print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
        data ={
            'author': author,
            'author_url': author_url,
            'title': title,
            'article_url': article_url,
            'abstract': abstract,
            'read_and_comments': read_and_comments,
            'like_and_pay': like_and_pay
        }
        if not is_data_exist(db, data):
            print('保存新数据')
            save_data(db, data)
        else:
            print('数据已存在,更新数据')
            update_data(db, data)
    return seen_snote_ids


def get_other_html(url, page, seen_snote_ids):
    url_param = url
    page_param = str(page)
    seen_snote_ids_param = '?seens_snote_ids%5B%5D=' + '?seens_snote_ids%5B%5D='.join(seen_snote_ids)
    url = url + '?page=' + page_param + seen_snote_ids_param
    # print(url)
    return get_html(url)


# 初始化数据库
def init_sql():
    from pymongo import MongoClient
    client = MongoClient('localhost', 27017)
    # 创建数据库sdifen
    db = client.JIANSHU
    return db


# 保存数据到数据库中
def save_data(db, data):
    db.WEEKLY.insert(data)


# 更新数据:删除原数据,重新保存该数据???
def update_data(db, data):
    pass

# 判断数据库中是否存在数据
def is_data_exist(db, data):
    return db.WEEKLY.find({'article_url': data.get('article_url')}).count()

def main():
    db = init_sql()
    resp = get_html(base_url)
    seen_snote_ids = parse_first_html(resp, db) 
    for i in range(2, 15):
        resp = get_other_html(base_url, i, seen_snote_ids)
        seen_snote_ids.extend(parse_other_html(resp, db))


if __name__ == '__main__':
    main()

数据分析部分代码

import pandas as pd
import numpy
import matplotlib.pyplot as plt
# 读取MongoDB数据
def read_data():
    import pymongo
    from pymongo import MongoClient
    client = MongoClient()
    db = client.JIANSHU
    collection = db.WEEKLY
    data = pd.DataFrame(list(collection.find()))
    return data


# 去重,文章链接一样的数据视为重复数据
data = read_data().drop_duplicates(['article_url'])
userClientCol = ['作者', '文章数']
# 构造DataFrame,注意:需数组转置
userClientDataFrame = pd.DataFrame(numpy.array([list(set(data.get('author'))), [list(data.get('author')).count(level) for level in list(set(data.get('author')))]]).T, columns=userClientCol)
# 根据作者文章数量进行排序,排序方式为降序
userClientDataFrame = userClientDataFrame.sort_values('文章数', ascending=False)
# 选取文章数量前20名的作者
userClientDataFrame = userClientDataFrame.head(15)
plt.figure(figsize=(20,10),dpi=100)
labels = list(userClientDataFrame['作者'])
plt.bar(range(len(labels)),userClientDataFrame['文章数'],tick_label=labels)
plt.title('作者')
plt.show()
发布文章作者前20名

说明:Mac默认的Python或者Anaconda环境下,中文会出现乱码,解决方法如下:

  • 下载微软雅黑.ttf文件
  • 重命名为DejaVuSans.ttf
  • 替换掉/anaconda/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf(Anaconda环境)
最后编辑于
©著作权归作者所有,转载或内容合作请联系作者
平台声明:文章内容(如有图片或视频亦包括在内)由作者上传并发布,文章内容仅代表作者本人观点,简书系信息发布平台,仅提供信息存储服务。

推荐阅读更多精彩内容