kaggle 线性回归预测房价

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

df_train = pd.read_csv('train.csv')
df_test = pd.read_csv('test.csv')

# 查看各列数据类型
df_train.dtypes.value_counts()

object     43
int64      35
float64     3
dtype: int64

# 把训练集和训练集合并一起处理，保证后期特征逻辑一致，注意不要随便drop行，drop行后顺序会变，后期拆分测试集会有问题
df_train_and_test = pd.concat([df_train.loc[:, 'MSSubClass': 'SaleCondition'], df_test.loc[:, 'MSSubClass': 'SaleCondition']], axis=0, ignore_index=True)

# 发现数据MSSubClass应为字符类型，进行转换
df_train_and_test['MSSubClass'] = df_train_and_test['MSSubClass'].astype('str')

# 查看训练集的相关性矩阵，如果值高说明相关性高，存在共线性，只保留共性特征中的1个
corrmat = df_train.corr()
f, ax = plt.subplots(figsize=(20, 10))
sns.heatmap(corrmat, vmax=.8, square=True)

<matplotlib.axes._subplots.AxesSubplot at 0x1fc3b1737b8>
output_5_1.png
# 查看系数矩阵和Y值最相关的10个特征
cols = corrmat.nlargest(n=10, columns='SalePrice').index

# 将10个特征相关系统矩阵按图形展示
cm = np.corrcoef(df_train[cols].values.T)
plt.figure(figsize=(20, 10))
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
output_7_0.png
# 删除共线性特征，保留与y值相关性高的特征
df_train_and_test = df_train_and_test.drop('GarageArea', axis=1) 
df_train_and_test = df_train_and_test.drop('1stFlrSF', axis=1) 
df_train_and_test = df_train_and_test.drop('TotRmsAbvGrd', axis=1) 

# 统计展示缺失数据情况
missing = df_train_and_test.isnull().sum()
missing = missing[missing>0]
missing.sort_values(ascending=False, inplace=True)
missing_percent = df_train_and_test[missing.index].isnull().sum()/df_train_and_test[missing.index].isnull().count()
missing_types = df_train_and_test[missing.index]
missing_data = pd.concat([missing, missing_percent, missing_types.dtypes], axis=1, keys=['Total', 'Percent', 'Types'])
missing_data.sort_values(by='Total', ascending=False, inplace=True)
missing_data

<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;">
<th></th>
<th>Total</th>
<th>Percent</th>
<th>Types</th>
</tr>
</thead>
<tbody>
<tr>
<th>PoolQC</th>
<td>2909</td>
<td>0.996574</td>
<td>object</td>
</tr>
<tr>
<th>MiscFeature</th>
<td>2814</td>
<td>0.964029</td>
<td>object</td>
</tr>
<tr>
<th>Alley</th>
<td>2721</td>
<td>0.932169</td>
<td>object</td>
</tr>
<tr>
<th>Fence</th>
<td>2348</td>
<td>0.804385</td>
<td>object</td>
</tr>
<tr>
<th>FireplaceQu</th>
<td>1420</td>
<td>0.486468</td>
<td>object</td>
</tr>
<tr>
<th>LotFrontage</th>
<td>486</td>
<td>0.166495</td>
<td>float64</td>
</tr>
<tr>
<th>GarageCond</th>
<td>159</td>
<td>0.054471</td>
<td>object</td>
</tr>
<tr>
<th>GarageQual</th>
<td>159</td>
<td>0.054471</td>
<td>object</td>
</tr>
<tr>
<th>GarageFinish</th>
<td>159</td>
<td>0.054471</td>
<td>object</td>
</tr>
<tr>
<th>GarageYrBlt</th>
<td>159</td>
<td>0.054471</td>
<td>float64</td>
</tr>
<tr>
<th>GarageType</th>
<td>157</td>
<td>0.053786</td>
<td>object</td>
</tr>
<tr>
<th>BsmtCond</th>
<td>82</td>
<td>0.028092</td>
<td>object</td>
</tr>
<tr>
<th>BsmtExposure</th>
<td>82</td>
<td>0.028092</td>
<td>object</td>
</tr>
<tr>
<th>BsmtQual</th>
<td>81</td>
<td>0.027749</td>
<td>object</td>
</tr>
<tr>
<th>BsmtFinType2</th>
<td>80</td>
<td>0.027407</td>
<td>object</td>
</tr>
<tr>
<th>BsmtFinType1</th>
<td>79</td>
<td>0.027064</td>
<td>object</td>
</tr>
<tr>
<th>MasVnrType</th>
<td>24</td>
<td>0.008222</td>
<td>object</td>
</tr>
<tr>
<th>MasVnrArea</th>
<td>23</td>
<td>0.007879</td>
<td>float64</td>
</tr>
<tr>
<th>MSZoning</th>
<td>4</td>
<td>0.001370</td>
<td>object</td>
</tr>
<tr>
<th>BsmtFullBath</th>
<td>2</td>
<td>0.000685</td>
<td>float64</td>
</tr>
<tr>
<th>Utilities</th>
<td>2</td>
<td>0.000685</td>
<td>object</td>
</tr>
<tr>
<th>BsmtHalfBath</th>
<td>2</td>
<td>0.000685</td>
<td>float64</td>
</tr>
<tr>
<th>Functional</th>
<td>2</td>
<td>0.000685</td>
<td>object</td>
</tr>
<tr>
<th>Exterior1st</th>
<td>1</td>
<td>0.000343</td>
<td>object</td>
</tr>
<tr>
<th>Exterior2nd</th>
<td>1</td>
<td>0.000343</td>
<td>object</td>
</tr>
<tr>
<th>SaleType</th>
<td>1</td>
<td>0.000343</td>
<td>object</td>
</tr>
<tr>
<th>BsmtFinSF1</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
<tr>
<th>BsmtFinSF2</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
<tr>
<th>BsmtUnfSF</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
<tr>
<th>Electrical</th>
<td>1</td>
<td>0.000343</td>
<td>object</td>
</tr>
<tr>
<th>KitchenQual</th>
<td>1</td>
<td>0.000343</td>
<td>object</td>
</tr>
<tr>
<th>GarageCars</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
<tr>
<th>TotalBsmtSF</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
</tbody>
</table>

# 图形展示缺失数据
missing.plot.bar(figsize=(20, 5))

<matplotlib.axes._subplots.AxesSubplot at 0x1fc3b141438>
output_10_1.png
## 同时是相关性列，也是缺失数据的有2个
missing_data.index.intersection(cols)

Index(['GarageCars', 'TotalBsmtSF'], dtype='object')

# 查看这2个特征的缺失情况
missing_data.loc[missing_data.index.intersection(cols)]

<div>
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;">
<th></th>
<th>Total</th>
<th>Percent</th>
<th>Types</th>
</tr>
</thead>
<tbody>
<tr>
<th>GarageCars</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
<tr>
<th>TotalBsmtSF</th>
<td>1</td>
<td>0.000343</td>
<td>float64</td>
</tr>
</tbody>
</table>
</div>

# drop掉其它缺失数据列
df_train_and_test = df_train_and_test.drop(missing_data[missing_data['Total']>1].index, axis=1)
df_train_and_test.isnull().sum().max()

1

# 对于剩余的数据按列平均值进行填充
df_train_and_test = df_train_and_test.fillna(df_train_and_test.mean())

# 剩余一个特殊为字符的后续转化成one_hot处理
df_train_and_test.isnull().sum()

MSSubClass       0
LotArea          0
Street           0
LotShape         0
LandContour      0
LotConfig        0
LandSlope        0
Neighborhood     0
Condition1       0
Condition2       0
BldgType         0
HouseStyle       0
OverallQual      0
OverallCond      0
YearBuilt        0
YearRemodAdd     0
RoofStyle        0
RoofMatl         0
Exterior1st      1
Exterior2nd      1
ExterQual        0
ExterCond        0
Foundation       0
BsmtFinSF1       0
BsmtFinSF2       0
BsmtUnfSF        0
TotalBsmtSF      0
Heating          0
HeatingQC        0
CentralAir       0
Electrical       1
2ndFlrSF         0
LowQualFinSF     0
GrLivArea        0
FullBath         0
HalfBath         0
BedroomAbvGr     0
KitchenAbvGr     0
KitchenQual      1
Fireplaces       0
GarageCars       0
PavedDrive       0
WoodDeckSF       0
OpenPorchSF      0
EnclosedPorch    0
3SsnPorch        0
ScreenPorch      0
PoolArea         0
MiscVal          0
MoSold           0
YrSold           0
SaleType         1
SaleCondition    0
dtype: int64

# 将字符值特征转换成one_hot类型
df_train_and_test = pd.get_dummies(df_train_and_test, drop_first=True, dummy_na=True)

df_train_and_test.isnull().sum().max()

0

# 查看所有特殊的相关系数矩阵，对于值超过0.8的只取其中1个
corrmat = df_train_and_test.corr()

droplist = []
n_col = len(corrmat.columns)
n_row = len(corrmat.index)
for i in range(n_col):
    for j in range(n_row):
        x = corrmat.iloc[j, i]
        idx = corrmat.index[j]
        cln = corrmat.columns[i]
        if np.abs(x) > 0.8 and idx != cln:
            print(idx, cln, x)
            droplist.append(idx)

HouseStyle_2Story 2ndFlrSF 0.8102595976
BldgType_2fmCon MSSubClass_190 0.97511848656
HouseStyle_1.5Unf MSSubClass_45 0.864323350941
HouseStyle_SLvl MSSubClass_80 0.958427756655
BldgType_Duplex MSSubClass_90 1.0
MSSubClass_190 BldgType_2fmCon 0.97511848656
MSSubClass_90 BldgType_Duplex 1.0
MSSubClass_45 HouseStyle_1.5Unf 0.864323350941
2ndFlrSF HouseStyle_2Story 0.8102595976
MSSubClass_80 HouseStyle_SLvl 0.958427756655
RoofStyle_Hip RoofStyle_Gable -0.939468653735
RoofStyle_Gable RoofStyle_Hip -0.939468653735
Exterior2nd_CmentBd Exterior1st_CemntBd 0.983410908223
Exterior2nd_HdBoard Exterior1st_HdBoard 0.887998938432
Exterior2nd_MetalSd Exterior1st_MetalSd 0.969711678996
Exterior2nd_VinylSd Exterior1st_VinylSd 0.978177709604
Exterior2nd_Wd Sdng Exterior1st_Wd Sdng 0.861610287457
Exterior2nd_nan Exterior1st_nan 1.0
Exterior1st_CemntBd Exterior2nd_CmentBd 0.983410908223
Exterior1st_HdBoard Exterior2nd_HdBoard 0.887998938432
Exterior1st_MetalSd Exterior2nd_MetalSd 0.969711678996
Exterior1st_VinylSd Exterior2nd_VinylSd 0.978177709604
Exterior1st_Wd Sdng Exterior2nd_Wd Sdng 0.861610287457
Exterior1st_nan Exterior2nd_nan 1.0
ExterQual_TA ExterQual_Gd -0.89966850725
ExterQual_Gd ExterQual_TA -0.89966850725
ExterCond_TA ExterCond_Gd -0.871903195903
ExterCond_Gd ExterCond_TA -0.871903195903
KitchenQual_TA KitchenQual_Gd -0.825028510061
KitchenQual_Gd KitchenQual_TA -0.825028510061
SaleCondition_Partial SaleType_New 0.986572970445
SaleType_New SaleCondition_Partial 0.986572970445

df_train_and_test = df_train_and_test.drop(droplist, axis=1)

# 把数据集拆分成训练集和测试集
train_number = df_train.shape[0]
X = df_train_and_test[:train_number]
y = df_train['SalePrice']
test = df_train_and_test[train_number:]

# 查看训练集的数据情况
sns.set()
cols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']
sns.pairplot(df_train[cols], size=2.5)
plt.show()
output_22_0.png
# 单独查看特征和y值情况
var = 'GrLivArea'
data = pd.concat([y, X[var]], axis=1)
data.plot.scatter(x=var, y='SalePrice')

<matplotlib.axes._subplots.AxesSubplot at 0x1fc3e6dda58>
output_23_1.png
# 查看右下2个异常点
data[data[var]>4500]

<div>
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;">
<th></th>
<th>SalePrice</th>
<th>GrLivArea</th>
</tr>
</thead>
<tbody>
<tr>
<th>523</th>
<td>184750</td>
<td>4676</td>
</tr>
<tr>
<th>1298</th>
<td>160000</td>
<td>5642</td>
</tr>
</tbody>
</table>
</div>

# drop这2个异常点
X = X.drop(data[data[var]>4500].index)
y = y.drop(data[data[var]>4500].index)

var = 'GrLivArea'
data = pd.concat([y, X[var]], axis=1)
data.plot.scatter(x=var, y='SalePrice')

<matplotlib.axes._subplots.AxesSubplot at 0x1fc3d329860>
output_25_1.png
# 查看y值的分布情况，图形显示正偏度
from scipy.stats import norm
import scipy.stats as stats
sns.distplot(y, fit=norm)
fig = plt.figure()
res = stats.probplot(y, plot=plt)

C:\Program Files\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.
  warnings.warn("The 'normed' kwarg is deprecated, and has been "
output_26_1.png
output_26_2.png
# 取对数处理正偏度情况
y = np.log(y)

from scipy.stats import norm
import scipy.stats as stats
sns.distplot(y, fit=norm)
fig = plt.figure()
res = stats.probplot(y, plot=plt)

C:\Program Files\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.
  warnings.warn("The 'normed' kwarg is deprecated, and has been "
output_27_1.png
output_27_2.png
# 拆分训练集和验证集
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=.33)

# 使用线性回归进行拟合，使用验证集进行评估
from sklearn import linear_model
lr = linear_model.LinearRegression()
model = lr.fit(X_train, y_train)
pred = model.predict(X_test)

from sklearn.metrics import mean_squared_error
print('RMSE is: \n', mean_squared_error(y_test, pred))

RMSE is: 
 0.0168625962345

# 预测测试集数据，生成结果
test_pred = model.predict(test)
test_pred = np.exp(test_pred)

output = pd.DataFrame({'Id': df_test['Id'], 'SalePrice': test_pred})
output.to_csv('output.csv', index=False)
©著作权归作者所有,转载或内容合作请联系作者
  • 人面猴
    序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
    沈念sama阅读 215,384评论 6赞 497
  • 死咒
    序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
    沈念sama阅读 91,845评论 3赞 391
  • 救了他两次的神仙让他今天三更去死
    文/潘晓璐 我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
    开封第一讲书人阅读 161,148评论 0赞 351
  • 道士缉凶录：失踪的卖姜人
    文/不坏的土叔 我叫张陵，是天一观的道长。 经常有香客问我，道长，这世上最难降的妖魔是什么？ 我笑而不...
    开封第一讲书人阅读 57,640评论 1赞 290
  • 港岛之恋（遗憾婚礼）
    正文 为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
    茶点故事阅读 66,731评论 6赞 388
  • 恶毒庶女顶嫁案：这布局不是一般人想出来的
    文/花漫 我一把揭开白布。 她就那样静静地躺着，像睡着了一般。 火红的嫁衣衬着肌肤如雪。 梳的纹丝不乱的头发上，一...
    开封第一讲书人阅读 50,712评论 1赞 294
  • 城市分裂传说
    那天，我揣着相机与录音，去河边找鬼。 笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。 我是一名探鬼主播，决...
    沈念sama阅读 39,703评论 3赞 415
  • 双鸳鸯连环套：你想象不到人心有多黑
    文/苍兰香墨 我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
    开封第一讲书人阅读 38,473评论 0赞 270
  • 万荣杀人案实录
    序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
    沈念sama阅读 44,915评论 1赞 307
  • 护林员之死
    正文 独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋 以下内容为张勋视角 年9月15日...
    茶点故事阅读 37,227评论 2赞 331
  • 白月光启示录
    正文 我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。 大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
    茶点故事阅读 39,384评论 1赞 345
  • 活死人
    序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
    沈念sama阅读 35,063评论 5赞 340
  • 日本核电站爆炸内幕
    正文 年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
    茶点故事阅读 40,706评论 3赞 324
  • 男人毒药：我在死后第九天来索命
    文/蒙蒙 一、第九天 我趴在偏房一处隐蔽的房顶上张望。 院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
    开封第一讲书人阅读 31,302评论 0赞 21
  • 一桩弑父案，背后竟有这般阴谋
    文/苍兰香墨 我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。 一阵脚步声响...
    开封第一讲书人阅读 32,531评论 1赞 268
  • 情欲美人皮
    我被黑心中介骗来泰国打工， 没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。 一个月前我还...
    沈念sama阅读 47,321评论 2赞 368
  • 代替公主和亲
    正文 我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。 传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
    茶点故事阅读 44,248评论 2赞 352

推荐阅读更多精彩内容