file1: Layer
class FullyConnection(object):
# in_size : X`s features number
# out_size : Z`s features number
# batch_size : how many sample in one batch
def __init__(self, in_size, out_size):
self.w = np.random.randn(in_size, out_size) # with R(in_size,out_size)
self.b = np.random.randn(out_size, 1) # with R(out_size,1)
self.cur_x = None
self.cur_z = None
"""
前向传播： 筛选X的特征 输出对应Z
"""
def forward(self, batch_x):
"""
Args:
batch_x: with R(batch_size,in_size)
Returns:
z : with R(batch_size,out_size)
"""
self.cur_x = batch_x
# z = (x*m) + b.T
z = np.matmul(batch_x, self.w) + self.b.T
self.cur_z = z
return z
"""
反向传播：根据 计算出的dz_w 和 传回的dl_z 计算dl_w 然后更新w的参数
"""
def backward(self, dl_z, lr):
"""
Args:
z: R(batch_size, out_size)
dl_z: R(batch_size, out_size)
Returns:
dl_x :R(batch_size,in_size)
"""
batch_size = self.cur_x.shape[0]
in_size = self.cur_x.shape[1]
out_size = self.cur_z.shape[1]
dz_w = np.expand_dims(self.cur_x, 1) # R(bt,1,in)
dz_b = np.ones([batch_size, out_size, 1])
dz_x = np.expand_dims(self.w, 0) # R(1(broadcast to bt),out,in)
dl_x = np.empty([batch_size, in_size])
for i in range(batch_size):
#dl_z[I] :R(1,out) dz_x[0].T :R(out,in) --> dl_x[i]:R(1,in)
dl_x[i] = np.matmul(dl_z[i], dz_x[0].T)
dl_z = np.expand_dims(dl_z, 2) # R(bt,out,1)
dl_w = dl_z * dz_w #R(bt,out,in)
dl_b = dl_z * dz_b #R(bt,out,1)
# update
self.step(dl_w, dl_b, lr)
return dl_x
def step(self, dw, db, lr):
# w: R(in,out) b:R(out,1)
self.w -= np.mean(dw, axis=0).T * lr
self.b -= np.mean(db, axis=0) * lr
file1: Activate
# in_size : Z`s features number
# out_size : A`s (pre_y) features number
# the in_size and out_size are equal in this activated layer
class SoftMax(object):
"""
softmax : 将所有输出特征值的概率归一化
"""
def __init__(self):
self.cur_z = None
self.exp_z = None
self.preY = None
def forward(self, z):
"""
Args:
z: with R(batch_size, in_size)
Returns:
a: with R(batch_size, out_size)
"""
self.cur_z = z
self.exp_z = np.exp(-z)
dom = np.sum(self.exp_z, axis=1).reshape([z.shape[0], 1])
a = self.exp_z / dom
self.preY = a
return a
def backward(self, dl_y):
"""
Args:
dl_y: R(batch_size, out_size)
Returns:
dl_z: R(batch_size, in_size)
"""
batch_size = self.cur_z.shape[0]
in_size = self.cur_z.shape[1]
dy_z = np.empty([batch_size, out_size, out_size])
for i in range(batch_size):
sing_y_sample = self.preY[i] # R(1，out_size)
sample_grad = np.matmul(sing_y_sample.T, sing_y_sample) # R(out_size,out_size)
diag = np.diag(sing_y_sample) # 生成对角矩阵
sample_grad = sample_grad - diag #更新对角线元素
dy_z[i] = sample_grad # 更新每个样本的剃度值
dl_z = np.empty([batch_size, in_size])
for i in range(batch_size):
# dl_y[i] : R(1,in_size)
# dy_z[i] : R(in_size,in_size)
dl_z[i] = np.matmul(dl_y[i], dy_z[i]) # R(1,in_size)
return dl_z # R(batch_size, in_size)
"""
Sigmoid : 把输出值控制在0,1范围内
"""
class Sigmoid(object):
def __init__(self):
self.a = None
def forward(self, z):
"""
Args:
z: R(batch_size, in_size)
Returns:
a: R(batch_size, out_size)
"""
self.a = 1 / (1 + np.exp(-z))
return self.a
def backward(self, dl_a):
"""
dl_a: R(batch_size,out_size)
Returns:
dl_z: R(batch_size, in_size)
"""
da_z = self.a * (1 - self.a) # R(batch_size,in_size)
dl_z = dl_a * da_z
return dl_z # R(batch_size,in_size)
"""
ReLU : 把输出值控制在0,1范围内
"""
def __init__(self):
self.a = None
def forward(self, z):
# z: R(batch_size,in_size)
a = z.copy()
a[a < 0] = 0
self.a = a
return a
def backward(self, dl_a):
"""
Args:
dl_a: R(batch_size,out_size)
Returns:
dl_z: R(batch_size,in_size)
"""
batch_size = self.a.shape[0]
in_size = self.a.shape[1]
da_z = self.a
da_z[da_z != 0] = 1
dl_z = dl_a * da_z
return dl_z
file3. Loss
class MSE(object):
def __init__(self):
self.pred_y = None
self.true_y = None
def forward(self, pred_y, true_y):
"""
Args:
pred_y: R(batch_size, in_size)
true_y: R(batch_size, in_size)
Returns:
loss: R(batch_size, 1)
"""
self.pred_y = pred_y
self.true_y = true_y
dif = pred_y - true_y
loss = np.sum(dif ** 2, axis=1)
return loss.reshape([pred_y.shape[0], 1])
def backward(self):
dl_y = self.pred_y - self.true_y # dl_y : R(batch_size, out_size)
return dl_y
file4. model
from layer import FullyConnection
from activate import SoftMax, Sigmoid,ReLU
# 三层
# 第一层： sigmoid
# 第二层：ReLU
# 第三层：SoftMax
class LogisticRegression(object):
def __init__(self, in_size, n_classes, lr):
self.fc1 = FullyConnection(in_size, 128) # init(in_size,out_size)
self.fc2 = FullyConnection(128, 50)
self.fc3 = FullyConnection(50,n_classes)
self.sigmoid1 = Sigmoid()
self.sigmoid2 = Sigmoid()
self.softmax = SoftMax()
self.ReLU1 = ReLU()
self.ReLU2 = ReLU()
self.lr = lr
def forward(self, batch_x):
z1 = self.fc1.forward(batch_x)
a1 = self.sigmoid1.forward(z1)
z2 = self.fc2.forward(a1)
a2 = self.ReLU1.forward(z2)
z3 = self.fc3.forward(a2)
y = self.softmax.forward(z3)
return y
def backward(self, dl_y):
"""
Args:
dl_y: R(batch_size, n_classes)
Returns:
"""
dl_z3 = self.softmax.backward(dl_y)
dl_a2 = self.fc3.backward(dl_z3,self.lr)
dl_z2 = self.ReLU1.backward(dl_a2) # R(batch_size, n_classes, n_classes)
dl_a1 = self.fc2.backward(dl_z2, self.lr) # R(1, n_classes, batch_size), R(1, 1, batch_size)
dl_z1 = self.sigmoid1.backward(dl_a1)
self.fc1.backward(dl_z1, self.lr)
经过训练之后，出现了很严重的梯度消失问题。
在识别手写数字的训练中，训练的准确度趋近于0.1 （小声bb:等于我的算法是在瞎JB猜)
问题出在ReLU中传出的a可能特别大,虽然传入ReLU的z经过了sigmoid的挤压，但由于我每一层的参数都是随机生成的，w可能特别大。这样会导致softmax中e^-z 趋近于0
最后我决定更换我的损失函数，采用交叉熵进行计算和更新
以下是更新后的loss.py
