实战计划0430-石头的练习作业

作业要求

实现效果

实现代码

__author__ = 'daijielei'

'''
    获取赶集网的二手物品分类信息，提取出各个分类的链接，并保存在一个数组中以备提取用
'''
from bs4 import BeautifulSoup
import requests

hosturl = 'http://bj.ganji.com'

def getGanjiAlltype(url='http://bj.ganji.com/wu/#all_category'):
    webdata = requests.get(url)
    soup = BeautifulSoup(webdata.text,'lxml')

    for item in soup.select('div.content div.clearfix dl.fenlei a'):
        print(hosturl + str(item.get('href')))
        #print('---------------------------------------------')


#getGanjiAlltype()

links = '''
    http://bj.ganji.com/jiaju/
    http://bj.ganji.com/rirongbaihuo/
    http://bj.ganji.com/chuangdian/
    http://bj.ganji.com/guizi/
    http://bj.ganji.com/zhuoyi/
    http://bj.ganji.com/shafachaji/
    http://bj.ganji.com/zixingchemaimai/
    http://bj.ganji.com/diandongche/
    http://bj.ganji.com/motuoche/
    http://bj.ganji.com/shouji/
    http://bj.ganji.com/shoujihaoma/
    http://bj.ganji.com/iphone/
    http://bj.ganji.com/nokia/
    http://bj.ganji.com/htc/
    http://bj.ganji.com/sanxingshouji/
    http://bj.ganji.com/motorola/
    http://bj.ganji.com/shouji/_%E5%B0%8F%E7%B1%B3/
    http://bj.ganji.com/shouji/_%E9%AD%85%E6%97%8F/
    http://bj.ganji.com/tongxuntaocan/
    http://bj.ganji.com/qqhao/
    http://bj.ganji.com/bangong/
    http://bj.ganji.com/nongyongpin/
    http://bj.ganji.com/bangongjiaju/
    http://bj.ganji.com/jiguangyitiji/
    http://bj.ganji.com/dayinji/z1/
    http://bj.ganji.com/shipinjiagongshebei/
    http://bj.ganji.com/shengchanjiamengshebei/
    http://bj.ganji.com/jichuang/
    http://bj.ganji.com/tuolaji/
    http://bj.ganji.com/jiadian/
    http://bj.ganji.com/dianshi/
    http://bj.ganji.com/bingxiang/
    http://bj.ganji.com/kongtiao/
    http://bj.ganji.com/reshuiqi/
    http://bj.ganji.com/xiyiji/
    http://bj.ganji.com/diancilu/
    http://bj.ganji.com/weibolu/
    http://bj.ganji.com/yueqiyinxiang/
    http://bj.ganji.com/ershoubijibendiannao/
    http://bj.ganji.com/pingbandiannao/z1/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E8%8B%B9%E6%9E%9C/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E8%81%94%E6%83%B3/
    http://bj.ganji.com/ershoubijibendiannao/z1/_Thinkpad/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E7%B4%A2%E5%B0%BC/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E6%88%B4%E5%B0%94/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E5%8D%8E%E7%A1%95/
    http://bj.ganji.com/ershoubijibendiannao/z1/_%E6%83%A0%E6%99%AE/
    http://bj.ganji.com/ruanjiantushu/
    http://bj.ganji.com/yueqi/
    http://bj.ganji.com/yinxiang/
    http://bj.ganji.com/yundongqicai/
    http://bj.ganji.com/yingyouyunfu/
    http://bj.ganji.com/tongche/
    http://bj.ganji.com/tongzhuang/
    http://bj.ganji.com/yingerche/
    http://bj.ganji.com/yingerchuang/z1/
    http://bj.ganji.com/niaobushi/
    http://bj.ganji.com/wanju/
    http://bj.ganji.com/naifen/
    http://bj.ganji.com/diannao/
    http://bj.ganji.com/taishidiannaozhengji/
    http://bj.ganji.com/xianka/
    http://bj.ganji.com/cpu/
    http://bj.ganji.com/yingpan/
    http://bj.ganji.com/luyouqi/
    http://bj.ganji.com/3gwangka/
    http://bj.ganji.com/xianzhilipin/
    http://bj.ganji.com/shoucangpin/
    http://bj.ganji.com/qitalipinzhuanrang/
    http://bj.ganji.com/baojianpin/
    http://bj.ganji.com/xiaofeika/
    http://bj.ganji.com/fushixiaobaxuemao/
    http://bj.ganji.com/meironghuazhuang/
    http://bj.ganji.com/fushi/
    http://bj.ganji.com/xiangbao/
    http://bj.ganji.com/xuemao/
    http://bj.ganji.com/shoubiao/
    http://bj.ganji.com/shipin/
    http://bj.ganji.com/huazhuangpin/
    http://bj.ganji.com/hufupin/
    http://bj.ganji.com/shuma/
    http://bj.ganji.com/shumaxiangji/
    http://bj.ganji.com/shumashexiangji/
    http://bj.ganji.com/youxiji/
    http://bj.ganji.com/suishenting/
    http://bj.ganji.com/yidongcunchu/
    http://bj.ganji.com/laonianyongpin/
    http://bj.ganji.com/zibubaojian/z2/
    http://bj.ganji.com/anmobaojian/z1/
    http://bj.ganji.com/bawanwujian/
    http://bj.ganji.com/xuniwupin/
    http://bj.ganji.com/qitawupin/
    http://bj.ganji.com/ershoufree/
    http://bj.ganji.com/wupinjiaohuan/
    http://bj.ganji.com/zhuanqu_anjia/all/
    http://bj.ganji.com/zhuanqu_jiaren/all/
    http://bj.ganji.com/zhuanqu_shenghuo/all/
'''

__author__ = 'daijielei'

'''
    爬虫，抓取赶集网下的相关信息

'''
from bs4 import BeautifulSoup
import requests
import pymongo
import time
import random
import IP_Proxy
import datetime


__Test__ = ''   # 'test'   #'该开关用来做测试用的控制'，打开时，不会写入数据库和做数据库的判断
__Prosey__ = '' # 'use'  #该开关用来测试控制是否使用IP代理
'''
-----------------------------------------------------------------------------------------------------
|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
以下为数据库相关代码
更多的mongoDB可以查看如下：http://www.runoob.com/mongodb/mongodb-indexing.html
                        https://blog.codecentric.de/files/2012/12/MongoDB-CheatSheet-v1_0.pdf
1、mongoDB可以通过建立索引来提高查询效率
2、可以使用条件操作符来进行> >= < <= !=的查询
3、可以使用$type来查询指定同一个名称下指定类型的数据
4、可以使用limit来限制查询的结果的数量，使用skip来跳过指定数目的结果
5、可以进行排序、索引、聚合等



'''
client = pymongo.MongoClient('localhost',27017)
ganjiDB = client['ganjiDB']


'''
    ---------            gangjiGetlistChecksheet该表用来保存已经爬过的列表，防止重新爬               --------
    ---------            gangjiGetInfoChecksheet该表用来保存已经爬过的商品信息，防止重新爬               --------
'''
gangjiGetlistChecksheet = ganjiDB['gangjiGetlistChecksheet']#保存爬过的list,如果已经爬过就不再爬了
gangjiGetInfoChecksheet = ganjiDB['gangjiGetInfoChecksheet']#保存爬过的信息,如果已经爬过就不再爬了
#将爬过的列表的URL进行保存
def insertgangjiGetlistChecksheet(data):
    if(__Test__ == 'test'):return
    gangjiGetlistChecksheet.insert_one(data)
#将爬过的商品的URL进行保存
def insertgangjiGetInfoChecksheet(data):
    if(__Test__ == 'test'):return
    gangjiGetInfoChecksheet.insert_one(data)
#判断该列表URL是否已经爬过
def ifexistgangjiGetlistChecksheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if(gangjiGetlistChecksheet.find({'url':url}).count()>0):
        ifexist = True
    return ifexist
#判断该商品URL是否已经爬过
def ifexistgangjiGetInfoChecksheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if(gangjiGetInfoChecksheet.find({'url':url}).count()>0):
        ifexist = True
    return ifexist
'''
    ---------            ganjiUrlSheet该表用来保存已经爬过的商品URL，以用判断用               --------
'''
ganjiUrlSheet = ganjiDB['ganjiUrlSheet']#保存爬下来的赶集商品url内容
#对商品的url表进行插入
def insertganjiUrlSheet(data={}):
    if(__Test__ == 'test'):return
    ganjiUrlSheet.insert_one(data)
#对商品的url表进行显示
def showganjiUrlSheet():
    for item in ganjiUrlSheet.find():
        print(item)
#对商品的url表进行删除
def deletAllganjiUrlSheet():
    ganjiUrlSheet.delete_many({})
    print(ganjiUrlSheet.count())
#对商品的url表是否存在进行判断
def ifexistganjiUrlSheet(url):
    if(__Test__ == 'test'):return False
    ifexist = False
    if ganjiUrlSheet.find({'url':url}).count() > 0:
        ifexist = True
    return ifexist



'''
    ---------            ganjiInfoSheet该表用来保存已经爬过的商品信息，以做判断用               --------
'''
ganjiInfoSheet = ganjiDB['ganjiInfoSheet']
#对商品信息插入对应的表
def insertganjiInfoSheet(data={}):
    if(__Test__ == 'test'):return
    ganjiInfoSheet.insert_one(data)
#显示全部商品信息
def showganjiInfoSheet():
    for item in ganjiInfoSheet.find():
        print(item)


'''
    ---------            对多个表进行操作               --------
'''
#显示当前表中已有的数据
def showAllDBcount():
    print('赶集url表中以存数据：'+str(ganjiUrlSheet.count()))
    print('赶集商品表中以存数据：'+str(ganjiInfoSheet.count()))

#showAllDBcount()

#清除全部表中的全部数据
def clearAllganjisheet():
    ganjiUrlSheet.delete_many({})
    gangjiGetlistChecksheet.delete_many({})
    gangjiGetInfoChecksheet.delete_many({})
    ganjiInfoSheet.delete_many({})
    print(ganjiUrlSheet.count())
    print(gangjiGetlistChecksheet.count())
    print(gangjiGetInfoChecksheet.count())
    print(ganjiInfoSheet.count())

'''
-----------------------------------------------------------------------------------------------------
|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
以下为爬虫部分的代码

gangji_getlist  用来抓取列表里的商品URL等信息

gangji_getInfo  用来抓取商品的相关信息

'''

headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
    'Connection':'keep-alive',
}
#代理服务器可以从中选择：http://cn-proxy.com/
#获取随机的代理服务器，以缓解被服务器禁用的风险
ip_proxy = ['1','2']#IP_Proxy.getProxy() 不用Proxy
if __Prosey__ == 'use':
    ip_proxy = IP_Proxy.getProxy()
def getRandomProxy():
    proxies = {
        'http':random.choice(ip_proxy),
    }
    return  proxies

def delProxy(proxy):
    proxy = proxy['http']
    index = list(ip_proxy).index(proxy)
    del ip_proxy[index]
    print('清除无效的ip_proxy,剩余数量：'+str(len(ip_proxy)))


#connect()用来连接URL获取页面的html数据
def connect(url):
    try:
        proxies = getRandomProxy()
        if __Prosey__ == 'use':
            webdata = requests.get(url,headers=headers,proxies=proxies) #使用proxies,来缓解被阻拦的风险
        else:
            webdata = requests.get(url)
        return webdata
    except (requests.exceptions.ConnectionError,TimeoutError):
        delProxy(proxies)
        print('ConnectionError')
    return None

#connectTry()因为该页面链接经常失效，故通过一个重连机制来不断重试
def connectTry(url):
    count = 0
    webdata = None
    while not webdata:  #如果没有获取到数据，继续重试
        time.sleep(1)
        webdata = connect(url)
        count = count + 1
        print('第'+str(count)+'次重试')
        if(count > 100):    #超过最大重试次数，直接退出
            return None
    return webdata



'''                              gangji_getlist  用来抓取列表里的商品URL等信息                            '''
def gangji_getlist(url='http://bj.ganji.com/motuoche/',page=0):
    typeName = url.strip('/').split('/')[-1]
    finalUrl = url + 'o' + str(page)

    if(ifexistgangjiGetlistChecksheet(finalUrl)):#判断是否爬过
        print('已经爬过该列表')
        return

    webdata = connectTry(finalUrl)
    #webdata = requests.get(finalUrl)
    soup = BeautifulSoup(webdata.text,'lxml')


    for item in soup.select('dl.list-bigpic.clearfix dt a'):#将列表中的数据内容按数据块方式拆解，以方便解析
        if('x.htm' in item.get('href')):#只解析赶集标准的商品url，不标准的如转转等不进行解析
            data = {
                'url':item.get('href'),
                'type':typeName,
                'catchTime':"-",

            }
            print(data)
            gangji_getInfo(data['url'])   #函数本身会判断是否已经爬过，这边就没必要进行判断了
            #if(not ifexistganjiUrlSheet(item.get('href'))):#如果该项已经存在了，则直接跳过该项目
            #    gangji_getInfo(data['url'])
            #    insertganjiUrlSheet(data)
            #else:
            #    print('已经存在')
        print('---------------------------------------------')

    insertgangjiGetlistChecksheet({'url':finalUrl})#将爬过的url插入表中，以方便下次重新开始爬起时跳过已爬过内容




'''                              gangji_getInfo  用来抓取商品的相关信息                            '''
def gangji_getInfo(url = 'http://bj.ganji.com/motuoche/1537729832x.htm'):
    if(ifexistgangjiGetInfoChecksheet(url)):#如果该项已经存在了，则直接跳过该项目
        print('已经爬过该列表')
        return

    webdata = connectTry(url)
    #webdata = requests.get(url)
    soup = BeautifulSoup(webdata.text,'lxml')

    data = {#要保存下来的数据内容和相关结构
        'title':soup.select('h1.title-name')[0].get_text() if soup.select('h1.title-name') else "",
        'type':[a.get_text() for a in soup.select('.crumbs.routes.clearfix a')] if soup.select('.crumbs.routes.clearfix a') else "",
        'price':None,
        'place':None,
        'phone':None,
        'date':'无日期',
        'saleday':None,
        'chense':'无数据',
        'catchDate':time.strftime("%m-%d", time.localtime()),
    }
    if soup.select('i.pr-5'):
        if(len(soup.select('i.pr-5')[0].get_text()) > 4):
            data['date'] = str(soup.select('i.pr-5')[0].get_text()).strip().split()[0]


    for item in soup.select('div.second-sum-cont ul.second-det-infor.clearfix li'):#获取成色
        label = item.label.get_text() if item.label else ""#获取label，以方便区分该块是什么
        if('新旧程度：' in label):
            #print(item.get_text())
            data['chense'] = item.get_text().strip().replace('新旧程度：\n','').replace(' ','')

    for item in soup.select('ul.det-infor li'):#获取某个块的细分项目，在根据项目的标签提取信息
        label = item.label.get_text() if item.label else ""#获取label，以方便区分该块是什么
        if('价' in label):
            data['price'] = item.select('i.f22')[0].get_text() if item.select('i.f22') else ''
        #对交易地点的数据做相关处理
        if('交易地点' in label):
            if(item.select('a')):
                data['place'] = [a.get_text() if a else  "" for a in item.select('a')]

    print(data)
    insertganjiInfoSheet(data)
    insertgangjiGetInfoChecksheet({'url':url})








if __name__ == '__main__':
    print('main')
    #clearAllganjisheet()
    #showAllDBcount()
    gangji_getlist('http://bj.ganji.com/iphone/')
    #gangji_getInfo('http://bj.ganji.com/diannao/1991466771x.htm')
    #gangji_getInfo()