爬取简书推荐作者信息并入库
以下是代码（我是初学者，写的不好请见谅）
涉及到知识点包括：
一、python基础知识
二、html基础知识
三、http基础知识
四、sql基础知识
五、requests库
六、urllib2库
七、BeautifulSoup库
八、MySQLdb库
结论：{"平均文章数": 148, "平均粉丝数": 14732, "平均关注数": 110, "平均写字数": 257475, "平均喜欢数": 4871}

# coding:utf-8
import requests
import MySQLdb
import json
import time
from bs4 import BeautifulSoup


# 连接数据库方法
def connect_db():
    db = MySQLdb.connect(host='localhost', user='root', passwd='ar281835e67_', db='python', charset='utf8')
    cursor = db.cursor()
    return db, cursor


# 创建数据库表方法
def create_table(db, cur):
    sql = """create table author4(
        id int auto_increment not null,
        username varchar(55) not null,
        summary varchar(500),
        carenum int,
        fansnum int,
        articlenum int,
        writenum int,
        likenum int,
        primary key(id))"""
    try:
        cur.execute(sql)
        db.commit()
        print '创建表成功'
    except:
        db.rollback()
        print '创建表失败'


# 往数据库表插入数据方法
def insert_table(db, cur, username, summary, carenum, fansnum, articlenum, writenum, likenum,num):
    param = [username, summary, carenum, fansnum, articlenum, writenum, likenum]
    sql = "insert into author4(username,summary,carenum,fansnum,articlenum,writenum,likenum) Values(%s,%s,%s,%s,%s,%s,%s)"
    try:
        cur.execute(sql, param)
        db.commit()
        print '成功插入'+str(num)+'条数据'
    except:
        db.rollback()
        print '插入数据失败'


#  创建需要爬取的url管理器，传递count参数表示希望爬取多少个推荐用户
def url_collect(count):
    inturl = 'https://www.jianshu.com/users/recommended?seen_ids=&count=5&only_unfollowed=true'
    firstUrl = inturl.split('5')[0]
    lastUrl = inturl.split('5')[1]
    url = firstUrl + str(count) + lastUrl
    try:
        response = requests.get(url,timeout=10)
        res = json.loads(response.content)
        userList = res['users']
        slugList = []
        for user in userList:
            slugList.append(user['slug'])
        print '成功获取用户slug数据'
        userUrl = 'https://www.jianshu.com/u/'
        userUrlList = []
        for slug in slugList:
            userUrlList.append(userUrl + str(slug))
        print '成功获取到需爬取的URL集合'
        return userUrlList
    except:
        print '请求推荐作者接口数据拉取失败'


#  定义获取用户字典信息存入列表的方法
def get_userDict(firstNum,lastNum,userUrllist,userList):
    for url in userUrllist[firstNum:lastNum]:
            userInfo = {}
            response = requests.get(url)
            res = response.content
            soup = BeautifulSoup(res, 'html.parser')
            username = soup.find('a', attrs={'class': 'name'}).text
            summary = soup.find('div', attrs={'class': 'js-intro'}).text
            otherInfo = soup.select('.meta-block p')
            carenum = otherInfo[0].text
            fansnum = otherInfo[1].text
            articelnum = otherInfo[2].text
            writenum = otherInfo[3].text
            likenum = otherInfo[4].text
            userInfo['username'] = username
            userInfo['summary'] = summary
            userInfo['carenum'] = carenum
            userInfo['fansnum'] = fansnum
            userInfo['articelnum'] = articelnum
            userInfo['writenum'] = writenum
            userInfo['likenum'] = likenum
            userList.append(userInfo)
            firstNum=firstNum+1
            print '成功获取到'+str(firstNum)+'个用户字典并存入列表中'
            time.sleep(5)
    return userList
         

# 通过用户url爬取用户信息的方法,传入count表示希望爬取多少个推荐用户
def get_userInfo(count):
    userUrllist = url_collect(count)
    userNum=len(userUrllist)
    userList = []
    num=0
    try:
        print '开始请求推荐作者主页数据'
        for url in userUrllist[num:userNum]:
            userInfo = {}
            response = requests.get(url)
            res = response.content
            soup = BeautifulSoup(res, 'html.parser')
            username = soup.find('a', attrs={'class': 'name'}).text
            summary = soup.find('div', attrs={'class': 'js-intro'}).text
            otherInfo = soup.select('.meta-block p')
            carenum = otherInfo[0].text
            fansnum = otherInfo[1].text
            articelnum = otherInfo[2].text
            writenum = otherInfo[3].text
            likenum = otherInfo[4].text
            userInfo['username'] = username
            userInfo['summary'] = summary
            userInfo['carenum'] = carenum
            userInfo['fansnum'] = fansnum
            userInfo['articelnum'] = articelnum
            userInfo['writenum'] = writenum
            userInfo['likenum'] = likenum
            userList.append(userInfo)
            num=num+1
            print '成功获取到'+str(num)+'个用户字典并存入列表中'
            time.sleep(5)
        print '成功获取到所有的用户信息'
        return userList
    except:
        print '请求推荐作者主页数据拉取失败,正在重新尝试'
        print '正在尝试获取第%s个用户字典' %(num+1)
        return get_userDict(num+1,userNum,userUrllist,userList)

# 定义最终的爬虫方法，把爬来的数据进行入库
def spider(db, cur, count):
    userList = get_userInfo(count)  # uersList是一个这样结构的数据 [{user1},{user2},{user3}]
    num=0
    for user in userList:
        username = user['username']
        summary = user['summary']
        carenum = user['carenum']
        fansnum = user['fansnum']
        articelnum = user['articelnum']
        writenum = user['writenum']
        likenum = user['likenum']
        num=num+1
        insert_table(db, cur, username, summary, carenum, fansnum, articelnum, writenum, likenum,num)
    print '爬虫工作结束'
dbInfo = connect_db()
db_src = dbInfo[0]
cur_src = dbInfo[1]
create_table(db_src,cur_src)
spider(db_src, cur_src, 10)

以下是代码执行过程，都打印出来了

image.png