抓取结果

p{'lorder_gender': '女', 'prices': '438', 'lorder_name': 'gjy1234', 'addr': '上海市徐汇区高安路69弄', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/7,0,86,9803,1800,1200,0b2f41ad.jpg', 'title': '衡山路 精致一居 近地铁1/7/9/10号线'}
{'lorder_gender': '男', 'prices': '538', 'lorder_name': '王老闆', 'addr': '上海市虹口区北苏州路400号河滨大楼', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/4,0,68,8498,1798,1200,ca0c88ef.jpg', 'title': '看外滩的超大阳台房 沙逊1935的河滨大楼'}
{'lorder_gender': '男', 'prices': '998', 'lorder_name': '风子上海', 'addr': '上海市徐汇区余庆路', 'house_image': 'http://image.xiaozhustatic1.com/00,801,533/6,0,59,6286,1136,756,d7287d22.jpg', 'title': '【春上春宿】法租界名人故居/有松鼠飞鸟的院子'}
……

CODE

from bs4 import BeautifulSoup
import requests
import time


urls = ["http://sh.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,3)]
def get_detail_url(url):
    web_data = requests.get(url)
    time.sleep(1)
    soup = BeautifulSoup(web_data.text, 'lxml')
    detail_urls = soup.select('#page_list > ul > li >div.result_btm_con.lodgeunitname')
    for single_detail_url in detail_urls:
        data = single_detail_url.get('detailurl')
        get_page_details(data)#获取每页详细内容


def get_page_details(url, infos = None):
    web_data = requests.get(url)
    time.sleep(1)
    soup = BeautifulSoup(web_data.text, 'lxml')
    titles =soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em ')
    addrs = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5 ')
    prices = soup.select('div.day_l > span ')
    house_images = soup.select('#curBigImage')
    lorder_names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    lorder_genders = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span ')

    if infos == None:
        for title, addr, price, house_image, lorder_name, lorder_gender in zip(titles, addrs, prices, house_images, lorder_names, lorder_genders):
            info = {
            'title':title.get_text(),
            'addr':addr.get_text().strip(),
            'prices':price.get_text(),
            'house_image':house_image.get('src'),
            'lorder_name':lorder_name.get('title'),
            'lorder_gender':get_gender(lorder_gender.get('class'))
            }
            print(info)


def get_gender(class_name):
    if class_name ==['member_boy_ico']:
        return '男'
    elif class_name == ['member_girl_ico']:
        return '女'
    else:
        return None

for url in urls:
    get_detail_url(url)

心得体会：

• 这节课安装lxml库的时候被难住了，网上到处搜，最后发现解决办法：

Paste_Image.png

• 函数可以嵌套使用，在本节课的练习中使用了三个函数，用get_detail_url()函数嵌套get_page_details()。

• 图片的抓取是难点，本例中还没有熟练掌握。抓取结果疑似无法下载。