<<机器学习实战>>--策略梯度
def basic_policy(obs):
angle = obs[2]
return 0 if angle < 0 else 1
totals = []
for episode in range(500):
episode_rewards = 0
obs = env.reset()
for step in range(1000):
action = basic_policy(obs)
obs, reward, done, info = env.step(action)
episode_rewards += reward
if done:
break
totals.append(episode_rewards)
import gym
import numpy as np
import tensorflow as tf
from tensorflow.contrib.layers import fully_connected
#分数处理
def discount_rewards(rewards, discount_rate):
discounted_rewards = np.empty(len(rewards))
cumulative_rewards = 0
for step in reversed(range(len(rewards))):
cumulative_rewards = rewards[step] + cumulative_rewards*discount_rate
discounted_rewards[step] = cumulative_rewards
return discounted_rewards
def discount_and_normalize_rewards(all_rewards, discount_rate):
all_discounted_rewards = [discount_rewards(rewards,discount_rate) for rewards in all_rewards]
flat_rewards = np.concatenate(all_discounted_rewards)
reward_mean = flat_rewards.mean()
reward_std = flat_rewards.std()
return [(discounted_rewards-reward_mean)/reward_std for discounted_rewards in all_discounted_rewards]
#OpenAl
env = gym.make('CartPole-v0')
#建模阶段
n_inputs = 4
n_hidden = 4
n_outputs = 1
initializer = tf.initializers.variance_scaling()
x = tf.placeholder(tf.float32, shape=[None, n_inputs])
hidden = fully_connected(x, n_hidden,activation_fn=tf.nn.relu,
weights_initializer=initializer)
logits = fully_connected(hidden,n_outputs,activation_fn= None ,weights_initializer=initializer)
outputs = tf.nn.sigmoid(logits)
p_left_and_right = tf.concat(axis=1,values=[outputs,1-outputs])
action = tf.multinomial(tf.log(p_left_and_right),num_samples=1)
init = tf.global_variables_initializer()
#
y = 1.-tf.to_float(action)
learning_rate = 0.01
cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=logits)
optimizer = tf.train.AdamOptimizer(learning_rate)
#使用之前调整梯度
grads_and_vars = optimizer.compute_gradients(cross_entropy)
gradients = [grad for grad, variable in grads_and_vars]
gradient_placeholders = []
grads_and_vars_feed = []
for grad, variable in grads_and_vars:
gradient_placeholder = tf.placeholder(tf.float32,
shape=grad.get_shape())
gradient_placeholders.append(gradient_placeholder)
grads_and_vars_feed.append((gradient_placeholder, variable))
training_op = optimizer.apply_gradients(grads_and_vars_feed)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
#训练阶段
n_iterations = 250 #训练迭代次数
n_max_steps = 1000 #每一次的最大步长
n_games_per_update = 10 #每迭代十次更新一次策略网络
save_iterations = 10 #每十次迭代保存模型
discount_rate = 0.95
with tf.Session() as sess:
init.run()
for iteration in range(n_iterations):
all_rewards = [] #每一次的所有奖励
all_gradients = [] #每一次的所有梯度
for game in range(n_games_per_update):
current_rewards = [] #当前步的所有奖励
current_gradients = [] #当前步的所有梯度
obs = env.reset()
for step in range(n_max_steps):
action_val, gradients_val = sess.run([action, gradients],
feed_dict={x: obs.reshape(1,n_inputs)})
obs, reward, done, info = env.step(action_val[0][0])
current_rewards.append(reward)
current_gradients.append(gradients_val)
if done:
break
all_rewards.append(current_rewards)
all_gradients.append(current_gradients)
all_rewards = discount_and_normalize_rewards(all_rewards,discount_rate)
feed_dict = {}
for var_index,grad_placeholder in enumerate(gradient_placeholders):
mean_gradients = np.mean([reward*all_gradients[game_index][step][var_index]
for game_index, rewards in enumerate(all_rewards) for step,reward in enumerate(rewards)],axis=0)
feed_dict[grad_placeholder] = mean_gradients
sess.run(training_op, feed_dict=feed_dict)
if iteration % save_iterations == 0:
saver.save(sess, './my_policy_net_pg.ckpt')
#----------Test----------
# sess = tf.Session()
# saver.restore(sess, './my_policy_net_pg.ckpt')
# episode_rewards=0
# while episode_rewards<10000000001:
# obs = env.reset()
# action_val, gradients_val = sess.run([action, gradients],
# feed_dict={x: obs.reshape(1,n_inputs)})
# obs, reward, done, info = env.step(action_val[0][0])
# if episode_rewards%100==0:
# print(episode_rewards)
# episode_rewards += reward
