python实战计划2_1

整个编程过程没什么问题。。基本上20分钟搞定
就是直接存数据进mongodb会将价格存成string类型，没法用$gte来比对
用了int()强制转换。

import pymongo
import requests
from bs4 import BeautifulSoup



def get_information(url):#这个函数完成了爬取其中一个页面商品信息的任务
    # url = 'http://bj.xiaozhu.com/fangzi/1508951935.html'
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text,'lxml')


    title = soup.select('div.pho_info > h4')[0].text
    address = soup.select('div.pho_info > p')[0].get('title')#[0]代表取数组的第1个元素，因为soup.select返回的是一个数组，然后获取title属性
    price = soup.select('div.day_l > span')[0].text
    pic = soup.select('#curBigImage')[0].get('src')
    fangdong_name = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a.lorder_name')[0].text
    fangdong_gender = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')[0].get('class')[0]

    gender="男"
    if fangdong_gender=='member_ico':
        gender="男"
    elif fangdong_gender=='member_icol':
        gender="女"


    data={
        'title':title,
        'address':address,
        'price':price,
        'img':pic,
        'host_name':fangdong_name,
        'host_gender':gender
    }

    return data

def get_info():
    for i in range(1,4):
        oriurl="http://bj.xiaozhu.com/search-duanzufang-p"+str(i)+"-0/"
        # oriurl="http://bj.xiaozhu.com/"
        wb_data_ori = requests.get(oriurl)
        soup_ori = BeautifulSoup(wb_data_ori.text,'lxml')
        links=soup_ori.select('#page_list > ul > li > a')
        infos=[]
        for link in links :
            link=link.get('href')
            infos.append(get_information(link))
        return infos




client=pymongo.MongoClient('localhost',27017)
walden=client['walden']
sheet_duanzufang=walden['sheet_duanzufang']

infos=get_info()
for info in infos:
    print(info)
    sheet_duanzufang.insert_one(info)


for item in sheet_duanzufang.find():
    if int(item['price'])>=500:
        print(item)

结果也很简单

{'host_gender': '男', 'img': 'http://image.xiaozhustatic1.com/00,800,533/3,0,37,4642,1800,1200,7b756ae5.jpg', 'title': '\n东大桥地铁口，毗邻三里屯工体朝阳门世贸天阶\n', 'price': '598', 'host_name': 'himhimhuang', '_id': ObjectId('574307097ca76743808cace7'), 'address': '北京市朝阳区呼家楼街道'}
{'host_gender': '男', 'img': 'http://image.xiaozhustatic1.com/00,800,533/3,0,79,3464,1800,1200,937686ec.jpg', 'title': '\n长安街沿线 鸟瞰国贸 紧邻地铁温馨舒适大三居\n', 'price': '680', 'host_name': '好心情123', '_id': ObjectId('574307097ca76743808cace8'), 'address': '北京市朝阳区八里庄西里1号'}
{'host_gender': '男', 'img': 'http://image.xiaozhustatic1.com/00,800,533/3,0,24,2692,1800,1200,4a3c697f.jpg', 'title': '\n到鸟巢去散步，安慧东里敞亮大三居\n', 'price': '698', 'host_name': 'ocheese', '_id': ObjectId('574307097ca76743808caceb'), 'address': '北京市朝阳区新都市计划大厦'}