基本的创建及运算
Numpy
import numpy as np
a=np.array([[1,2,3],[4,5,6]])
print(a.ndim)#多少维
print(a.shape)#形状
print(a.size)#一共有多少个元素
print(a.dtype)#数据类型
print(a.nbytes)#占的字节数大小
2
(2, 3)
6
int32
24
在原生中numpy有内置的数据类型，所有元素的数据类型必须相同
主要的四个属性如上所示不传入默认的参数numpy会自动推断数据类型
np.int 可以加32 或 64 或 128同 np.float，np.complex
import numpy as np
np.zeros((3,4),dtype=np.int32)#不填默认为np.float
np.ones((3,4))#生成全为1的矩阵
print(np.empty((3,4)))#生成极小的值接近于很小。。。解释不清楚反正就是数极小
print(np.arange(10,20,3))#开始结束步长生成np数组
print(np.linspace(1,10,6))#生成线段，把从开始到结束平均分成制定的段数
a=np.arange(1,20,2)
a.reshape((2,5))#改变数组形状
[[1. 1. 1. 1.]
[1. 1. 1. 1.]
[1. 1. 1. 1.]]
[10 13 16 19]
[ 1. 2.8 4.6 6.4 8.2 10. ]
array([[0.77527422, 0.83932489, 0.64089705, 0.6470655 ],
[0.57605182, 0.13560442, 0.77435193, 0.52690343]])
一些创建数组API
import numpy as np
a=np.array([1,2,3,4])
b=np.array([10,20,30,40])
print(a+b,a**2,a*b)#加减，次方，相乘每一项都互相相乘,乘数字也是一样的
print(np.sin(a))#把每一项都求sin，cos，tan，cot同理
print(b<3)#返回一个同规模的矩阵，满足为True不满足就是False，大小等于都一样
#乘法
a1=a.reshape(2,2)#不会改变原数组
b1=b.reshape(2,2)
print(a1*b1)#普通对于元素对应相乘
print(np.dot(a1,b1))#矩阵相乘
np.dot(a1,b1)==a1.dot(b1)#相乘的了两种方法
[11 22 33 44] [ 1 4 9 16] [ 10 40 90 160]
[ 0.84147098 0.90929743 0.14112001 -0.7568025 ]
[False False False False]
[[ 10 40]
[ 90 160]]
[[ 70 100]
[150 220]]
array([[ True, True],
[ True, True]])
import numpy as np
a=np.random.random((2,4))#生成0-1之内的随机数组成的矩阵
print(a)
print("最大值",np.max(a),"\n最小值",np.min(a),"\n求和",np.sum(a))#求一个数组所有元素的 最大 最小 和
print("最大值",np.max(a,axis=1),"\n最小值",np.min(a,axis=0),"\n求和",np.sum(a,axis=0))#分别对 行，列，列
[[0.88326345 0.88160519 0.35671284 0.21065075]
[0.43092957 0.72153207 0.4825476 0.09734723]]
最大值 0.883263451828129
最小值 0.09734723037984405
求和 4.064588702909058
最大值 [0.88326345 0.72153207]
最小值 [0.43092957 0.72153207 0.35671284 0.09734723]
求和 [1.31419302 1.60313727 0.83926043 0.30799798]
参数axis=1是对每行进行求值，为0是对列求值
import numpy as np
a=np.arange(0,20).reshape(4,5)
print(np.argmin(a))#找到最小值的坐标
print(np.argmax(a))#找到最大值的坐标输出并非二维坐标
print(np.mean(a),a.mean())#输出数组的平均值
print(np.median(a))#中位数
print(np.cumsum(a))#相当于reduce,循环累加，输出一维数组
print(np.diff(a))#累差，每两个数之间的差组成的数组
print(np.nonzero(a))#输出两个数组，代表行和列，对应坐标的对象不为0
print(np.sort(a))
0
19
9.5 9.5
9.5
[ 0 1 3 6 10 15 21 28 36 45 55 66 78 91 105 120 136 153
171 190]
[[1 1 1 1]
[1 1 1 1]
[1 1 1 1]
[1 1 1 1]]
(array([0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3],
dtype=int64), array([1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4],
dtype=int64))
[[ 0 1 2 3 4]
[ 5 6 7 8 9]
[10 11 12 13 14]
[15 16 17 18 19]]
import numpy as np
a=np.arange(0,20).reshape(4,5)
print(a,np.transpose(a),a.T)#矩阵的反向 左上到右下轴反转
print((a.T).dot(a))#矩阵的乘法
print(np.clip(a,1,5))#参数是 想要截取的矩阵 最小值 最大值 会把比最大值大的全都变为最大值，比最小值小的都变成最小值
[[ 0 1 2 3 4]
[ 5 6 7 8 9]
[10 11 12 13 14]
[15 16 17 18 19]] [[ 0 5 10 15]
[ 1 6 11 16]
[ 2 7 12 17]
[ 3 8 13 18]
[ 4 9 14 19]] [[ 0 5 10 15]
[ 1 6 11 16]
[ 2 7 12 17]
[ 3 8 13 18]
[ 4 9 14 19]]
[[350 380 410 440 470]
[380 414 448 482 516]
[410 448 486 524 562]
[440 482 524 566 608]
[470 516 562 608 654]]
[[1 1 2 3 4]
[5 5 5 5 5]
[5 5 5 5 5]
[5 5 5 5 5]]
numpy的函数大部分都有axis参数可以对行=1列=0进行计算
注意numpy中的轴，默认for只迭代行,a.flat属性是一个包含每一个元素的迭代器，a.flatten()返回一个一维包含所有元素的列表
import numpy as np
a=np.array([1,1,1,1])
b=np.array([2,2,2,2])
print(np.vstack((a,b)),np.vstack((a,b)).shape)#vertical stack 上下合并
print(np.hstack((a,b)),np.hstack((a,b)).shape)#horizontal stack 水平合并
print(np.concatenate((a,a,b,b),axis=0))#进行多个矩阵的合并？？？
b=b[:,np.newaxis]
print(a[:,np.newaxis])#一维的矩阵没法直接反转，可以用这个方法转换，意思是在列上增加一个维度
print(a[np.newaxis,:])#同上，将一列的矩阵反过来
[[1 1 1 1]
[2 2 2 2]] (2, 4)
[1 1 1 1 2 2 2 2] (8,)
[1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2]
[[1]
[1]
[1]
[1]]
[[1 1 1 1]]
import numpy as np
a=np.arange(0,20).reshape(4,5)
print(np.split(a,2,axis=0))#按照指定的段数分割矩阵,vsplit,hsplit竖直分割和水平分割
print(np.array_split(a,2,axis=1))#可以进行不等分割，结果却也不等了
[array([[0, 1, 2, 3, 4],
[5, 6, 7, 8, 9]]), array([[10, 11, 12, 13, 14],
[15, 16, 17, 18, 19]])]
[array([[ 0, 1, 2],
[ 5, 6, 7],
[10, 11, 12],
[15, 16, 17]]), array([[ 3, 4],
[ 8, 9],
[13, 14],
[18, 19]])]
import numpy as np
a=np.array([1,1,1,1])
b=a
b is a
#重新开一个对象要用deep copy
b=a.copy()
np.nan
nan
对数组矩阵进行赋值只是增加对象引用而没有新建对象改变一个另一个也变
Pandas
大型字典，多个键值对和series组成
import pandas as pd
import numpy as np
pd.Series([1,"svfsag",np.nan])#定义一个Series同样具有dtype等属性
s=pd.date_range("20160518",periods=6)#生成日期，往后推6个日期
DF=pd.DataFrame(np.random.randn(6,4),index=s,columns=["a","b","c","d"])#创建一个DataFrame，传入数据 传入index行，columns列
#不输入行列默认为0123.....
DF
<div>
<style>
.dataframe thead tr:only-child th {
text-align: right;
}
.dataframe thead th {
text-align: left;
}
.dataframe tbody tr th {
vertical-align: top;
}
</style>
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;">
<th></th>
<th>a</th>
<th>b</th>
<th>c</th>
<th>d</th>
</tr>
</thead>
<tbody>
<tr>
<th>2016-05-18</th>
<td>0.838417</td>
<td>-1.401509</td>
<td>0.696444</td>
<td>0.286138</td>
</tr>
<tr>
<th>2016-05-19</th>
<td>-0.761240</td>
<td>0.332608</td>
<td>-0.856670</td>
<td>-1.351850</td>
</tr>
<tr>
<th>2016-05-20</th>
<td>-0.956997</td>
<td>-1.282749</td>
<td>0.164582</td>
<td>-0.387242</td>
</tr>
<tr>
<th>2016-05-21</th>
<td>-0.470080</td>
<td>-1.145040</td>
<td>-0.321351</td>
<td>0.064876</td>
</tr>
<tr>
<th>2016-05-22</th>
<td>0.126539</td>
<td>0.179270</td>
<td>-0.598734</td>
<td>1.343720</td>
</tr>
<tr>
<th>2016-05-23</th>
<td>0.100891</td>
<td>1.590030</td>
<td>0.869404</td>
<td>-0.184462</td>
</tr>
</tbody>
</table>
</div>
import pandas as pd
import numpy as np
df=pd.DataFrame({"A":1.,
"B":pd.Timestamp(20130102),#时间戳
"C":pd.Series(1,index=list(range(4)),dtype="float32"),
"D":np.array([3]*4,dtype="int32"),
"E":pd.Categorical(["test","train","sdasd","sghuu"]),#统计类别数，可调用codes 数目 和 categories 类别
"F":"foo"})
print(df)#用字典初始化一个DataFrame,只传入一个值默认全部为该值，字典键默认为行标
print(df.dtypes,df.index,df.columns,df.values)#类型，行列，值
print(df.describe())#对数字类型的数据描述，平均，标准差，方差
print(df.T)#矩阵转置
A B C D E F
0 1.0 1970-01-01 00:00:00.020130102 1.0 3 test foo
1 1.0 1970-01-01 00:00:00.020130102 1.0 3 train foo
2 1.0 1970-01-01 00:00:00.020130102 1.0 3 sdasd foo
3 1.0 1970-01-01 00:00:00.020130102 1.0 3 sghuu foo
A float64
B datetime64[ns]
C float32
D int32
E category
F object
dtype: object Int64Index([0, 1, 2, 3], dtype='int64') Index(['A', 'B', 'C', 'D', 'E', 'F'], dtype='object') [[1.0 Timestamp('1970-01-01 00:00:00.020130102') 1.0 3 'test' 'foo']
[1.0 Timestamp('1970-01-01 00:00:00.020130102') 1.0 3 'train' 'foo']
[1.0 Timestamp('1970-01-01 00:00:00.020130102') 1.0 3 'sdasd' 'foo']
[1.0 Timestamp('1970-01-01 00:00:00.020130102') 1.0 3 'sghuu' 'foo']]
A C D
count 4.0 4.0 4.0
mean 1.0 1.0 3.0
std 0.0 0.0 0.0
min 1.0 1.0 3.0
25% 1.0 1.0 3.0
50% 1.0 1.0 3.0
75% 1.0 1.0 3.0
max 1.0 1.0 3.0
0 1 \
A 1 1
B 1970-01-01 00:00:00.020130102 1970-01-01 00:00:00.020130102
C 1 1
D 3 3
E test train
F foo foo
2 3
A 1 1
B 1970-01-01 00:00:00.020130102 1970-01-01 00:00:00.020130102
C 1 1
D 3 3
E sdasd sghuu
F foo foo
对DataFrame排序
import pandas as pd
import numpy as np
df=pd.DataFrame({"A":1.,
"B":pd.date_range(20180502,periods=4),#时间戳
"C":pd.Series(1,index=list(range(4)),dtype="float32"),
"D":np.array([3]*4,dtype="int32"),
"E":pd.Categorical(["test","train","sdasd","sghuu"]),#统计类别数，可调用codes 数目 和 categories 类别
"F":"foo"})
print(df.sort_index(axis=1,ascending=False))#对矩阵的某列进行倒排序
print(df.sort_values(by="D"))#指定某一行对值进行排序
print(df["A"])#,df["19700101":"19700103"])#?????
print(df.loc[3],df.loc[:,["C","B"]],df.loc[3,["C","B"]])#从index，column按名称选取对象,index为3，column为C，B
print(df.iloc[0:2,3:7])#选取下标，可范围选择，传入列表[[1,3,5],3:7]可以选1，3，5的值
#print(df.ix[0:3,["A","C"]])#要过期第一次运行会报错，混合选取，建议用上面俩代替
print(df[df.C>0])
F E D C B A
0 foo test 3 1.0 1970-01-01 00:00:00.020180502 1.0
1 foo train 3 1.0 1970-01-02 00:00:00.020180502 1.0
2 foo sdasd 3 1.0 1970-01-03 00:00:00.020180502 1.0
3 foo sghuu 3 1.0 1970-01-04 00:00:00.020180502 1.0
A B C D E F
0 1.0 1970-01-01 00:00:00.020180502 1.0 3 test foo
1 1.0 1970-01-02 00:00:00.020180502 1.0 3 train foo
2 1.0 1970-01-03 00:00:00.020180502 1.0 3 sdasd foo
3 1.0 1970-01-04 00:00:00.020180502 1.0 3 sghuu foo
0 1.0
1 1.0
2 1.0
3 1.0
Name: A, dtype: float64
A 1
B 1970-01-04 00:00:00.020180502
C 1
D 3
E sghuu
F foo
Name: 3, dtype: object C B
0 1.0 1970-01-01 00:00:00.020180502
1 1.0 1970-01-02 00:00:00.020180502
2 1.0 1970-01-03 00:00:00.020180502
3 1.0 1970-01-04 00:00:00.020180502 C 1
B 1970-01-04 00:00:00.020180502
Name: 3, dtype: object
D E F
0 3 test foo
1 3 train foo
A C
0 1.0 1.0
1 1.0 1.0
2 1.0 1.0
3 1.0 1.0
A B C D E F
0 1.0 1970-01-01 00:00:00.020180502 1.0 3 test foo
1 1.0 1970-01-02 00:00:00.020180502 1.0 3 train foo
2 1.0 1970-01-03 00:00:00.020180502 1.0 3 sdasd foo
3 1.0 1970-01-04 00:00:00.020180502 1.0 3 sghuu foo
D:\python\Anaconda\lib\site-packages\ipykernel_launcher.py:16: DeprecationWarning:
.ix is deprecated. Please use
.loc for label based indexing or
.iloc for positional indexing
See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated
app.launch_new_instance()
选取之后即可进行赋值，可以新增行列df["X"]=np.nan或者新的列即可，注意index要相等？？？
import pandas as pd
import numpy as np
df=pd.DataFrame({"A":1.,
"B":pd.date_range(20180502,periods=4),#时间戳
"C":pd.Series(1,index=list(range(4)),dtype="float32"),
"D":np.array([3]*4,dtype="int32"),
"E":pd.Categorical(["test","train","sdasd","sghuu"]),#统计类别数，可调用codes 数目 和 categories 类别
"F":"foo"})
df.insert(0,'G',[np.nan,np.nan,np.nan,np.nan])#插入某一列
print(df)
df2=df.drop(0,axis=0,inplace=False)#删除某一行
df1=df.drop('A',axis=1,inplace=False) #删除某一列
print(df2)
print(df.isnull())#判断里面有没有nan
df['E'] = df['E'].cat.add_categories(0) # ！！Categorical中必须要添加已有的类别才可进行fill操作
print(np.all(df.isnull()))#any也行，判断是不是有nan或者是不是全是什么东西
print(df.fillna(value=0))
print(df.dropna(axis=0,how="any"))#有any和all两种模式任意一个或者全是才会删除
G A B C D E F
0 NaN 1.0 1970-01-01 00:00:00.020180502 1.0 3 test foo
1 NaN 1.0 1970-01-02 00:00:00.020180502 1.0 3 train foo
2 NaN 1.0 1970-01-03 00:00:00.020180502 1.0 3 sdasd foo
3 NaN 1.0 1970-01-04 00:00:00.020180502 1.0 3 sghuu foo
G A B C D E F
1 NaN 1.0 1970-01-02 00:00:00.020180502 1.0 3 train foo
2 NaN 1.0 1970-01-03 00:00:00.020180502 1.0 3 sdasd foo
3 NaN 1.0 1970-01-04 00:00:00.020180502 1.0 3 sghuu foo
G A B C D E F
0 True False False False False False False
1 True False False False False False False
2 True False False False False False False
3 True False False False False False False
False
G A B C D E F
0 0.0 1.0 1970-01-01 00:00:00.020180502 1.0 3 test foo
1 0.0 1.0 1970-01-02 00:00:00.020180502 1.0 3 train foo
2 0.0 1.0 1970-01-03 00:00:00.020180502 1.0 3 sdasd foo
3 0.0 1.0 1970-01-04 00:00:00.020180502 1.0 3 sghuu foo
Dataframe的合并
import pandas as pd
import numpy as np
df0=pd.DataFrame(np.ones((3,4))*0,columns=["a","b","c","d"])#生成三个DataFrame
df1=pd.DataFrame(np.ones((3,4))*1,columns=["a","b","c","d"])
df2=pd.DataFrame(np.ones((3,4))*2,columns=["a","b","c","d"])
df3=pd.DataFrame(np.ones((3,4))*2,columns=["b","c","d","e"])
print(pd.concat([df0,df1,df2],axis=0))#按行或者列合并几个df
pd.concat([df0,df3],axis=0,join_axes=[df0.index])#按照df0的index来合并多余的删掉，也可以指定join="inner"/"outer"参考数据inner/outer
s=pd.Series([1,2,3,4],index=["a","b","c","d"])#创建新的序列
res=df0.append(s,ignore_index=True)#合并两个DF，参数忽略列？？？？？？？？？？？？？？？
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
0 1.0 1.0 1.0 1.0
1 1.0 1.0 1.0 1.0
2 1.0 1.0 1.0 1.0
0 2.0 2.0 2.0 2.0
1 2.0 2.0 2.0 2.0
2 2.0 2.0 2.0 2.0
import pandas as pd
import numpy as np
df0=pd.DataFrame({"key":["K0","K1","K2","K4"],
"A":["A0","A1","A2","A3"],
"B":["B0","B1","B2","B3"]})
df1=pd.DataFrame({"key":["K0","K1","K2","K3"],
"C":["C0","C1","C2","C3"],
"D":["D0","D1","D2","D3"]})
print(pd.merge(df0,df1,on="key",join="outer"))#默认是inner join，不同的列会丢弃
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-7-2c1da58b8b2e> in <module>()
9 "C":["C0","C1","C2","C3"],
10 "D":["D0","D1","D2","D3"]})
---> 11 print(pd.merge(df0,df1,on="key",join="outer"))#默认是inner join，不同的列会丢弃
TypeError: merge() got an unexpected keyword argument 'join'
