from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").appName("shuangyu").getOrCreate()
#读取数据，并查看前5行
sc = spark.sparkContext
f = sc.textFile("/home/shuangyu/Downloads/ccFraud.csv")
f.take(5)
['"custID","gender","state","cardholder","balance","numTrans","numIntlTrans","creditLine","fraudRisk"',
'1,1,35,1,3000,4,14,2,0',
'2,2,2,1,0,9,0,18,0',
'3,2,2,1,0,27,9,16,0',
'4,1,15,1,0,12,0,5,0']
#准备创建DataFrame
import pyspark.sql.types as typ
#获取第一行，（第一行为列名）
header = f.first()
#数据去掉头部，并转换成int
f = f.filter(lambda line:line != header).map(lambda line:[int(e) for e in line.split(",")])
#创建schema
cols = [*[typ.StructField(h[1:-1],typ.IntegerType(),True) for h in header.split(",")]]
schema = typ.StructType(cols)
#创建dataframe
df = spark.createDataFrame(f,schema)
df.cache()
df.printSchema()
root
|-- custID: integer (nullable = true)
|-- gender: integer (nullable = true)
|-- state: integer (nullable = true)
|-- cardholder: integer (nullable = true)
|-- balance: integer (nullable = true)
|-- numTrans: integer (nullable = true)
|-- numIntlTrans: integer (nullable = true)
|-- creditLine: integer (nullable = true)
|-- fraudRisk: integer (nullable = true)
#查看数据中性别比例
df.groupBy("gender").count().show()
+------+-------+
|gender| count|
+------+-------+
| 1|6178231|
| 2|3821769|
+------+-------+
#获取指定字段的一些统计信息
df.describe("balance","numTrans","numIntlTrans").show()
+-------+-----------------+------------------+-----------------+
|summary| balance| numTrans| numIntlTrans|
+-------+-----------------+------------------+-----------------+
| count| 10000000| 10000000| 10000000|
| mean| 4109.9199193| 28.9351871| 4.0471899|
| stddev|3996.847309737258|26.553781024523122|8.602970115863904|
| min| 0| 0| 0|
| max| 41485| 100| 60|
+-------+-----------------+------------------+-----------------+
#相关性.
df.corr("balance","numTrans")
0.0004452314017265386
可视化
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('ggplot')#样式美化
# %matplotlib inline是用来使输出图像在notebook中输出而不是单独打开一个窗口
#histogram 分桶，也就是画直方图中有多少个柱子
#返回的是一个tuple(List,List)，第一个list中是x，第二个是对应的y
hist = df.select("balance").rdd.flatMap(lambda x: x).histogram(20)
print(type(hist))
hist
<class 'tuple'>
([0.0,
2074.25,
4148.5,
6222.75,
8297.0,
10371.25,
12445.5,
14519.75,
16594.0,
18668.25,
20742.5,
22816.75,
24891.0,
26965.25,
29039.5,
31113.75,
33188.0,
35262.25,
37336.5,
39410.75,
41485],
[3640021,
2258778,
1718633,
1035019,
601524,
343461,
192766,
104486,
54517,
27570,
13117,
5805,
2650,
1103,
358,
127,
45,
15,
1,
4])
#x里面有21个值，y有20个，所以x的最后一个要去掉（x轴最后是无穷区间）
data = {"bins":hist[0][:-1],"freq":hist[1]}
#width 柱子的x轴方向上的宽度，一般近似为x1-x0
plt.bar(data["bins"],data["freq"],width=2000)
plt.xlabel("X")
plt.ylabel("Y")
plt.title("Just For Test")
<matplotlib.text.Text at 0x7f48b3d6fc18>
#sampleBy根据指定字段分层抽样
sample_data = df.sampleBy('gender',{1:0.0002,2:0.0002}).select("balance","numTrans","numIntlTrans")
sample_data.take(5)
[Row(balance=4000, numTrans=71, numIntlTrans=2),
Row(balance=0, numTrans=12, numIntlTrans=0),
Row(balance=3000, numTrans=44, numIntlTrans=0),
Row(balance=6000, numTrans=20, numIntlTrans=0),
Row(balance=3000, numTrans=3, numIntlTrans=13)]
#按字段将值分开
data_multi ={elem:sample_data.select(elem).rdd.flatMap(lambda x:x).collect()\
for elem in ["balance","numTrans","numIntlTrans"]}
#画散点图，plt.scatter(x,y)
plt.scatter(data_multi["numTrans"],data_multi["balance"],c ='r',marker='o',label="numTrans-balance")
plt.scatter(data_multi["numIntlTrans"],data_multi["balance"],c ='b',marker="x",label="numInt-balance")
plt.grid(True)#网格
plt.xlabel("x value")
plt.ylabel("y value")
plt.legend(loc='upper right')#图例的位置
plt.title("scatter view")
<matplotlib.text.Text at 0x7f48b3cdbf28>
