准备工作-使用到的模块
//全局安装自动重启工具nodemon
cnpm install -g nodemon
cnpm i --save koa koa-router mysql cheerio superagent-charset superagent
----app.js----
const Koa = require('koa'),
app = new Koa(),
index = require('./routes/index');
app.use(index.routes(), index.allowedMethods());
app.listen(3000);
//路由信息
----/routes/index----
const router = require('koa-router')(),
mysql = require('../db/mysql'),
superagent = require('../caiji/superagent');
router.get('/caiji/:page', async (ctx, next) => {
let page = ctx.params;
let html = await superagent.get(`https://cnodejs.org/?tab=all&page=${page.page}`);
if (html.length > 0) {
try {
let count = 0;
for (let i in html) {
await mysql.query(`insert into nodeData(title,userName,time)
values('${html[i].title}','${html[i].userName}','${html[i].time}')`);
count += 1
}
ctx.body = {
code: 1,
message: `该页采集完成,共采集【${count}】条`
}
} catch (error) {
ctx.body = {
code: 0,
message: `采集失败:${error}`
}
}
}
})
//"数据库配置"
----/db/config----
module.exports = {
DATABASE:'test',
USERNAME:'root',
PASSWORD:'zhy123456',
PORT:'3306',
HOST:'localhost'
}
----/db/mysql----
const mysql = require('mysql');
const config = require('./config');
let pool = mysql.createPool({
host:config.HOST,
user:config.USERNAME,
password:config.PASSWORD,
database:config.DATABASE
})
class Mysql{
constructor(){
}
query(sql){
console.log(sql)
return new Promise((resolve,resject)=>{
pool.query(sql,(err,res,fields)=>{
if (err) {
throw err;
}
resolve(res)
})
})
}
}
module.exports = new Mysql()
//采集模块
----/db/caiji----
const cheerio = require('cheerio'),
superagent = require('superagent'),
charset = require('superagent-charset');
charset(superagent);
module.exports = {
get(url) {
return new Promise((resolve, reject) => {
superagent.get(url)
.charset('utf-8')
.end((err, res) => {
if (err) {
resolve([])
}
if (res) {
let $ = cheerio.load(res.text, {
decodeEntities: false
})
let arr = [];
for (let i in $('.cell')) {
let title = $('.cell').eq(i).find('.topic_title').eq(0).html(),
userName = $('.cell').eq(i).find('.user_avatar').eq(0).find('img').eq(0).attr('title'),
time = $('.cell').eq(i).find('.last_active_time').eq(0).html(),
views = $('.cell').eq(i).find('.count_of_visits').eq(0).text();
if (title) {
arr.push({
title: title.trim(),
userName,
time,
views: views.trim()
})
}
}
resolve(arr)
} else {
resolve([])
}
})
})
}
}
//启动
nodemon app.js
//浏览器地址
[http://127.0.0.1:3000/caiji/1](http://127.0.0.1:3000/caiji/1)
第一页采集完成
