爬取某一年哈尔滨市的天气预报信息。
网址
需要爬取2016年整年的资料。
import urllib.request
from bs4 import BeautifulSoup
import pandas as pd
# URL list for crawler:
'''
http://lishi.tianqi.com/haerbin/201601.html
......
http://lishi.tianqi.com/haerbin/201612.html
'''
URList=['http://lishi.tianqi.com/haerbin/2016'+str(i).zfill(2)+'.html' for i in range(1,13)]
label=[]
con_in=[]
con_all=[]
for url in URList:
content = urllib.request.urlopen(url).read().decode('gbk') #获取HTML
soup = BeautifulSoup(content,'html.parser') #解析
tempall=soup.select('.tqtongji2') #定位到要爬取信息的类
uls=tempall[0].findAll('ul') #再进一步精确定位
if label==[]:
lis=uls[0].findAll('li')
for li in lis:
label.append(li.text)
for ul in uls[1:]:
lis=ul.findAll('li')
for li in lis:
con_in.append(li.text)
con_all.append(con_in)
con_in=[]
df = pd.DataFrame.from_records(con_all, columns=label)
df.to_csv('forecast_info_Harbin.csv')
