通过Node 爬取数据 (豆瓣书籍信息)

1.使用ORM框架 定义数据模型 （本次使用的是：sequelize）

  模型：/**
 * @function 书籍
 */
const sequelize = require('./db')
const { DataTypes } = require('sequelize')
const Book = sequelize.define("Book", {
    name: {
        type: DataTypes.STRING,
        allowNull: false
    },
    imgUrl:{
        type:DataTypes.STRING,
        allowNull:false
    },
    publishDate:{
        type:DataTypes.DATE,
        allowNull:false
    },
    author:{
        type:DataTypes.STRING
    }
}, {
    paranoid: true
})

module.exports = Book;

下面是爬虫具体代码 (数据请求使用的是 axios htm 解析的使用的cheerio)

const axios = require('axios').default;
const cheerio = require('cheerio')
const book = require('../models/Book')
//获取html页面
/**
 * 爬取 豆瓣读书的 数据信息
 */
const getHtml = async () => {
    let resultHtml = await (await axios('https://book.douban.com/latest')).data;
    return resultHtml
}

const getLinks = async () => {
    const html = await getHtml();
    const $ = cheerio.load(html);
    const Links = $('#content .grid-12-12  li a.cover')
    let _links = Links.map((i, element) => {
        let href = element.attribs['href'];
        return href;
    }).get()
    console.log(_links, 1)
    return _links;
}
//获取每一个书籍的详情页
const getItemContentValue = async (url) => {
    let rep = await (await axios(url)).data;
    const $ = cheerio.load(rep);
    let name = $('h1').text().trim();
    const imgUrl = $('#mainpic a.nbg img').attr('src');
    const publishDate = new Date();
    const author = '测试';
    return {
        name,
        imgUrl,
        publishDate,
        author
    }
}

const getListLinsBooKContent = async () => {
    let list = await getLinks();
    console.log(list, 1)
    let data = await Promise.all(list.map(it => {
        return getItemContentValue(it)
    }))
    return data;
}


let demo = async () => {
    const list = await getListLinsBooKContent();
    let data = await book.bulkCreate(await getListLinsBooKContent())  //获取的数据放到数据库
    console.log(data,'11111')
}
demo()