前言
- Batch Normalization 中文翻译为 批量标准化
- 概念说明见参考文献
- 我们将批量归一化添加到一个基本的全连接神经网络，该神经网络有两个隐含层，每层神经元都有100个神经元，在两个隐含层应用批量标准化
Code
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
mnist = input_data.read_data_sets('/home/gcb/data/MNIST_data', one_hot = True)
w1_initial = np.random.normal(size = (784, 100)).astype(np.float32)
w2_initial = np.random.normal(size = (100, 100)).astype(np.float32)
w3_initial = np.random.normal(size = (100, 10)).astype(np.float32)
# 一个特别小的数，保证分母不为0
epsilon = 1e-3
构建 graph
# Placeholder
x = tf.placeholder(tf.float32, shape = [None, 784])
y_ = tf.placeholder(tf.float32, shape = [None, 10])
# layer 1 without BN
W1 = tf.Variable(w1_initial)
b1 = tf.Variable(tf.zeros([100]))
z1 = tf.matmul(x, W1) + b1 # (None, 100)
l1 = tf.nn.sigmoid(z1)
- Here is the same layer 1 with batch normalization:
# Layer 1 with BN
w1_BN = tf.Variable(w1_initial) # (784, 100)
# 请注意，批次前标准化偏差被省略。 这种偏见的影响是当减去批量平均值时消除。
# 相反，偏见的作用是由新的beta变量执行的。 参见BN2015论文的第3.2节。
z1_BN = tf.matmul(x,w1_BN) # (None, 784)x(784, 100) = (None, 100)
# Calculate batch mean and variance
batch_mean1, batch_var1 = tf.nn.moments(z1_BN,[0])
# Apply the initial batch normalizing transform
z1_hat = (z1_BN - batch_mean1) / tf.sqrt(batch_var1 + epsilon)
# Create two new parameters, scale and beta (shift)
scale1 = tf.Variable(tf.ones([100]))
beta1 = tf.Variable(tf.zeros([100]))
# Scale and shift to obtain the final output of the batch normalization
# this value is fed into the activation function (here a sigmoid)
BN1 = scale1 * z1_hat + beta1
l1_BN = tf.nn.sigmoid(BN1)
# Layer 2 without BN
w2 = tf.Variable(w2_initial)
b2 = tf.Variable(tf.zeros([100]))
z2 = tf.matmul(l1,w2)+b2 # (None, 100)
l2 = tf.nn.sigmoid(z2)
# Layer 2 with BN, using Tensorflows built-in BN function
w2_BN = tf.Variable(w2_initial)
z2_BN = tf.matmul(l1_BN,w2_BN)
batch_mean2, batch_var2 = tf.nn.moments(z2_BN,[0])
scale2 = tf.Variable(tf.ones([100]))
beta2 = tf.Variable(tf.zeros([100]))
BN2 = tf.nn.batch_normalization(z2_BN,batch_mean2,batch_var2,beta2,scale2,epsilon)
l2_BN = tf.nn.sigmoid(BN2)
# Softmax
w3 = tf.Variable(w3_initial)
b3 = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(l2,w3)+b3)
w3_BN = tf.Variable(w3_initial)
b3_BN = tf.Variable(tf.zeros([10]))
y_BN = tf.nn.softmax(tf.matmul(l2_BN,w3_BN)+b3_BN)
# Loss, optimizer and predictions
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
cross_entropy_BN = -tf.reduce_sum(y_*tf.log(y_BN))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
train_step_BN = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy_BN)
correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
correct_prediction_BN = tf.equal(tf.argmax(y_BN,1),tf.argmax(y_,1))
accuracy_BN = tf.reduce_mean(tf.cast(correct_prediction_BN,tf.float32))
训练神经网络
zs, BNs, acc, acc_BN = [], [], [], []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(40000):
batch_x, batch_y = mnist.train.next_batch(60)
sess.run(train_step, feed_dict={x: batch_x, y_: batch_y})
sess.run(train_step_BN, feed_dict={x: batch_x, y_: batch_y})
if i % 50 == 0:
res = sess.run([accuracy,accuracy_BN,z2,BN2],feed_dict={x: mnist.test.images, y_: mnist.test.labels})
acc.append(res[0])
acc_BN.append(res[1])
zs.append(np.mean(res[2],axis=0)) # record the mean value of z2 over the entire test set
BNs.append(np.mean(res[3],axis=0)) # record the mean value of BN2 over the entire test set
zs, BNs, acc, acc_BN = np.array(zs), np.array(BNs), np.array(acc), np.array(acc_BN)
# print(zs.shape) # (800, 100)
# print(acc.shape) # (800, )
比较 Accuracy
fig, ax = plt.subplots()
ax.plot(range(0,len(acc)*50,50),acc, label='Without BN')
ax.plot(range(0,len(acc)*50,50),acc_BN, label='With BN')
ax.set_xlabel('Training steps')
ax.set_ylabel('Accuracy')
ax.set_ylim([0.8,1])
ax.set_title('Batch Normalization Accuracy')
ax.legend(loc=4)
plt.show()
- 由图可以明显看出，添加 Batch Normalization 得到 Accuracy 较高。
使用模型进行预测
predictions = []
correct = 0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(100):
pred, corr = sess.run([tf.argmax(y_BN,1), accuracy_BN],
feed_dict={x: [mnist.test.images[i]], y_: [mnist.test.labels[i]]})
predictions.append(pred[0])
correct += corr
print("PREDICTIONS:", predictions)
print("ACCURACY:", correct/100)
- 注意，可以看到如果我们直接用模型进行预测，得到的预测会是一模一样的结果，很糟糕！！！
- 解决方法是，training 和 prediction 所用的 均值 和 方差 应该是不同的，下面是整理好的代码 (用 is_training 这个参数来判别是不是 training)
def batch_norm_wrapper(inputs, is_training, decay = 0.999):
scale = tf.Variable(tf.ones([inputs.get_shape()[-1]]))
beta = tf.Variable(tf.zeros([inputs.get_shape()[-1]]))
pop_mean = tf.Variable(tf.zeros([inputs.get_shape()[-1]]), trainable=False)
pop_var = tf.Variable(tf.ones([inputs.get_shape()[-1]]), trainable=False)
if is_training:
batch_mean, batch_var = tf.nn.moments(inputs,[0])
train_mean = tf.assign(pop_mean,
pop_mean * decay + batch_mean * (1 - decay))
train_var = tf.assign(pop_var,
pop_var * decay + batch_var * (1 - decay))
with tf.control_dependencies([train_mean, train_var]):
return tf.nn.batch_normalization(inputs,
batch_mean, batch_var, beta, scale, epsilon)
else:
return tf.nn.batch_normalization(inputs,
pop_mean, pop_var, beta, scale, epsilon)
参考文献
moments
a = tf.constant([[1, 2, 3], [4, 5, 6]], dtype = tf.float32)
mean, variance = tf.nn.moments(a, [0])
with tf.Session() as sess:
print(sess.run(a))
print(sess.run(mean))
print(sess.run(variance))
print('----------')
a = np.array([[1, 2, 3], [4, 5, 6]])
print(np.std(a, axis = 0) ** 2)
a = tf.constant([[1, 2, 3], [4, 5, 6]], dtype = tf.float32)
mean, variance = tf.nn.moments(a, [1])
with tf.Session() as sess:
print(sess.run(a))
print(sess.run(mean))
print(sess.run(variance))
print('----------')
a = np.array([[1, 2, 3], [4, 5, 6]])
print(np.std(a, axis = 1) ** 2)
