作业一：动手做网页
动手做网页
总结：了解了网页的结构，先写结构代码，再填充样式
作业二：爬取本地网页
from bs4 import BeautifulSoup
path='F:/百度云/Plan-for-combating-master/week1/1_2/1_2answer_of_homework/index.html'
with open(path,'r') as web_data:
soup=BeautifulSoup(web_data.read(),'lxml')
titles=soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4 > a')
images=soup.select('body > div > div > div.col-md-9 > div > div > div > img')
prices=soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4.pull-right')
stars=soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings ')
reviews=soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings > p.pull-right')
#print(stars)
for title,image,price,star,review in zip (titles,images,prices,stars,reviews):
data={
'title':title.get_text(),
'image':image.get('src'),
'price':price.get_text()[1:],
'star':len(star.find_all('span','glyphicon glyphicon-star')),
'review':review.get_text()[:-8],
}
print(data)
结果：
爬取结果
总结：了解了BeautifulSoup的基本用法，CSS Select的基本用法，用之前学过的[1:]截取需要的信息，结果是可显示的，但是不知道用法对不对，
作业三：爬取租房信息
作业代码：
from bs4 import BeautifulSoup
import requests,time,random
url='http://bj.xiaozhu.com/search-duanzufang-p1-0/'
def get_urls (url):
wb_data = requests.get(url)
soup=BeautifulSoup(wb_data.text,'lxml')
urls=soup.select('#page_list > ul > li > a')
url_list = []
for url in urls:
url_list.append(url.get('href'))
return url_list
def get_member_sex (class_name):
if class_name ==['member_boy_ico']:
return u'男'
else:
return u'女'
def get_date (url):
wb_data = requests.get(url)
soup = BeautifulSoup(wb_data.text, 'lxml')
title=soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
address=soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
price=soup.select('#pricePart > div.day_l > span')
image=soup.select('#curBigImage')
member_pic=soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
member_name=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
member_sex=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')
title_data=title[0].get_text()
address_data=address[0].get_text()
price_data=price[0].get_text()
image_data=image[0].get('src')
member_pic_data=member_pic[0].get('src')
member_name_data=member_name[0].get_text()
member_sex_data=get_member_sex(member_sex[0].get('class'))
data={
'title':title_data,
'address':address_data,
'price':price_data,
'image':image_data,
'member_pic':member_pic_data,
'member_name': member_name_data,
'member_sex': member_sex_data,
}
print(data)
def spider (url):
for i in get_urls(url):
get_date(i)
#spider(url)
def spider2 (page=1):
for j in range(1,page+1):
url=('http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(j))
for i in get_urls(url):
get_date(i)
t=random.randrange(1,5)
print('爬完一页，休息{}秒---------------'.format(t))
time.sleep(t)
spider2(2)
爬取结果：
爬取结果
总结：熟悉了requests的用法，member_boy_ico，get class之后变成一个列表，不知为何，所以判断的时候只能以列表去对比。
作业四：爬取58
from bs4 import BeautifulSoup
import requests,time,random
def get_urls (url):
web_data=requests.get(url)
soup=BeautifulSoup(web_data.text,'lxml')
urls=soup.select('td.t a.t')
url_list = []
for url in urls:
url_list.append(url.get('href'))
return url_list
def get_views(url):
id = url.split('/')[-1].strip('x.shtml')
headers = {
'User-Agent':r'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
'Cookie':r'id58=c5/ns1ct99sKkWWeFSQCAg==; city=bj; 58home=bj; ipcity=yiwu%7C%u4E49%u4E4C%7C0; als=0; myfeet_tooltip=end; bj58_id58s="NTZBZ1Mrd3JmSDdENzQ4NA=="; sessionid=021b1d13-b32e-407d-a76f-924ec040579e; bangbigtip2=1; 58tj_uuid=0ed4f4ba-f709-4c42-8972-77708fcfc553; new_session=0; new_uv=1; utm_source=; spm=; init_refer=; final_history={}; bj58_new_session=0; bj58_init_refer=""; bj58_new_uv=1'.format(str(id)),
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host':'jst1.58.com',
'Referer':r'http://bj.58.com/pingbandiannao/{}x.shtml'.format(id)
}
api = 'http://jst1.58.com/counter?infoid={}'.format(id)
js = requests.get(api,headers = headers)
view = js.text.split('=')[-1]
return view
def get_info(url):
web_data=requests.get(url)
soup=BeautifulSoup(web_data.text,'lxml')
data = {
'title' : (soup.title.text).split('-')[0],
'price' : soup.select('.price.c_f50')[0].get_text() if soup.find('span','price c_f50') else '0',
'date': soup.select('li.time')[0].text,
'area': list(soup.select('.c_25d')[0].stripped_strings) if soup.find_all('span','c_25d') else '[]',
'fenlei':soup.select('.crb_i')[1].text,
'views': get_views(url),
}
print(data)
def spider (page=1):
for j in range(1,page+1):
url=('http://bj.58.com/pbdn/0/pn{}/'.format(j))
for i in get_urls(url):
get_info(i)
t=random.randrange(1,5)
print('爬完一页，休息{}秒---------------'.format(t))
time.sleep(t)
spider(2)
总结：根据老师和同学的代码，学习了split,回顾了stripped_strings，和requests的加载headers，前面讲过的基础知识还是要多看几遍，不然一下就忘了
