爬取的详情页链接

从详情页获得的数据

# 具体代码
from bs4 import BeautifulSoup
import requests
import time
detail_url = []


def get_detailurl(page):
    source_url = 'http://bj.58.com/pbdn/0/pn'+str(page)
    web_data = requests.get(source_url)
    time.sleep(1)
    soup = BeautifulSoup(web_data.text, 'lxml')
    edetail_urls = soup.select('td.t > a.t ')
    for urls in edetail_urls:
            detail_urls = urls.get('href')
            if detail_urls.rfind('http://bj.58.com') != -1:
                detail_url.append(detail_urls)
            print(detail_url)


def get_view(api):
    for a in detail_url:
        header = {'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language':'zh-CN,zh;q=0.8',
        'Cache-Control':'max-age=0',
        'Connection':'keep-alive',
        'Cookie': 'id58=8vXrx1dBfEKQ6mZc+CYong==; mcity=qhd; myfeet_tooltip=end; __utma=253535702.1066767632.1463909395.'
                 '1463909395.1463909395.1; __utmz=253535702.1463909395.1.1.utmcsr=link|utmccn=(not%20set)|utmcmd=(not%2'
                 '0set); als=0; bj=2016521172959; 58home=bj; ipcity=qhd%7C%u79E6%u7687%u5C9B%7C0; city=bj; bj58_id58s="'
                 'eUcyN3dTaXVXTi1QNDA4OA=="; bangbigtip2=1; sessionid=690e8177-c481-4cc4-b56c-0676790c6ecc; 58tj_uuid=d8'
                 '55a902-f081-4b23-9340-1b08713c0090; new_uv=3; final_history=26090732655920%2C26098857137450%2C26100784016'
                 '311%2C26098181862586%2C26091423827534; bj58_new_session=1; bj58_init_refer="http://bj.58.com/pbdn/1/"; b'
                 'j58_new_uv=4',
        'Host': 'jst1.58.com',
        'Referer': 'http://bj.58.com/pingbandiannao/26090732655920x.shtml?psid=150292251191878423914965113&entinfo=26090732655920_0&iuType=p_1&PGTID=0d305a36-0000-14bb-71c7-bebb30845ebf&ClickID=3',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
        }
        ID = a.split('/')[4].split('?')[0].strip('x.shtml')
        api_url = 'http://jst1.58.com/counter?infoid={}'.format(ID)
        web_data = requests.get(api_url, headers=header)
        views = BeautifulSoup(web_data.text, 'lxml').text.split('=')[-1]
        return views


def get_info(url):
    web_data = requests.get(url)
    soup = BeautifulSoup(web_data.text, 'lxml')
    item = soup.select('span.crb_i > a')[-1].text
    title = soup.select('div.col_sub.mainTitle > h1')[0].text
    date = soup.select('ul.mtit_con_left.fl > li.time')[0].text
    price = soup.select('div.su_con > span.price ')[0].text
    degree = list(soup.select('ul > li > div.su_con')[1].stripped_strings)
    area = list(soup.select('.c_25d')[0].stripped_strings) if soup.find_all('span', 'c_25d') else None
    view = get_view(url)
    data = {
            '类目': item,
            '名称': title,
            '日期': date,
            '价格': price,
            '新旧程度': degree,
            '区域': area,
            '浏览量': view
        }
    print(data)


def get_detail_info(start, end):
    for i in range(start, end):
        get_detailurl(i)
    for k in detail_url:
        get_info(k)

get_detail_info(1, 2)