<<机器学习实战>>--策略梯度

def basic_policy(obs):
    angle = obs[2]
    return 0 if angle < 0 else 1
totals = []
for episode in range(500):
    episode_rewards = 0
    obs = env.reset()
    for step in range(1000):
        action = basic_policy(obs)
        obs, reward, done, info = env.step(action)
        episode_rewards += reward
        if done:
            break
    totals.append(episode_rewards)
import gym
import numpy as np
import tensorflow as tf
from tensorflow.contrib.layers import fully_connected
#分数处理
def discount_rewards(rewards, discount_rate):
    discounted_rewards = np.empty(len(rewards))
    cumulative_rewards = 0
    for step in reversed(range(len(rewards))):
        cumulative_rewards = rewards[step] + cumulative_rewards*discount_rate
        discounted_rewards[step] = cumulative_rewards
    return discounted_rewards
def discount_and_normalize_rewards(all_rewards, discount_rate):
    all_discounted_rewards = [discount_rewards(rewards,discount_rate) for rewards in all_rewards]
    
    flat_rewards = np.concatenate(all_discounted_rewards)
    reward_mean = flat_rewards.mean()
    reward_std = flat_rewards.std()
    return [(discounted_rewards-reward_mean)/reward_std for discounted_rewards in all_discounted_rewards]
#OpenAl
env = gym.make('CartPole-v0')
#建模阶段
n_inputs = 4
n_hidden = 4
n_outputs = 1
initializer = tf.initializers.variance_scaling()

x = tf.placeholder(tf.float32, shape=[None, n_inputs])
hidden = fully_connected(x, n_hidden,activation_fn=tf.nn.relu,
                         weights_initializer=initializer)
logits = fully_connected(hidden,n_outputs,activation_fn= None ,weights_initializer=initializer)
outputs = tf.nn.sigmoid(logits)

p_left_and_right = tf.concat(axis=1,values=[outputs,1-outputs])
action = tf.multinomial(tf.log(p_left_and_right),num_samples=1)
init = tf.global_variables_initializer()
#
y = 1.-tf.to_float(action)
learning_rate = 0.01
cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=logits)
optimizer = tf.train.AdamOptimizer(learning_rate)
#使用之前调整梯度
grads_and_vars = optimizer.compute_gradients(cross_entropy)
gradients = [grad for grad, variable in  grads_and_vars]
gradient_placeholders = []
grads_and_vars_feed = []
for grad, variable in grads_and_vars:
    gradient_placeholder = tf.placeholder(tf.float32, 
                                          shape=grad.get_shape())
    gradient_placeholders.append(gradient_placeholder)
    grads_and_vars_feed.append((gradient_placeholder, variable))
training_op = optimizer.apply_gradients(grads_and_vars_feed)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
#训练阶段
n_iterations = 250 #训练迭代次数
n_max_steps = 1000 #每一次的最大步长
n_games_per_update = 10 #每迭代十次更新一次策略网络
save_iterations = 10 #每十次迭代保存模型
discount_rate = 0.95
with tf.Session() as sess:
    init.run()
    for iteration in range(n_iterations):
        all_rewards = [] #每一次的所有奖励
        all_gradients = [] #每一次的所有梯度
        for game in range(n_games_per_update):
            current_rewards = [] #当前步的所有奖励
            current_gradients = [] #当前步的所有梯度
            obs = env.reset()
            for step in range(n_max_steps):
                action_val, gradients_val = sess.run([action, gradients],
                                        feed_dict={x: obs.reshape(1,n_inputs)})
                obs, reward, done, info = env.step(action_val[0][0])
                current_rewards.append(reward)
                current_gradients.append(gradients_val)
                if done:
                    break
            all_rewards.append(current_rewards)
            all_gradients.append(current_gradients)
        all_rewards = discount_and_normalize_rewards(all_rewards,discount_rate)
        feed_dict = {}
        for var_index,grad_placeholder in enumerate(gradient_placeholders):
            mean_gradients = np.mean([reward*all_gradients[game_index][step][var_index]
                for game_index, rewards in enumerate(all_rewards) for step,reward in enumerate(rewards)],axis=0)
            feed_dict[grad_placeholder] = mean_gradients
        sess.run(training_op, feed_dict=feed_dict)
        if iteration % save_iterations == 0:
            saver.save(sess, './my_policy_net_pg.ckpt')
#----------Test----------
# sess = tf.Session()
# saver.restore(sess, './my_policy_net_pg.ckpt')
# episode_rewards=0
# while episode_rewards<10000000001:
#     obs = env.reset()
#     action_val, gradients_val = sess.run([action, gradients],
#                                             feed_dict={x: obs.reshape(1,n_inputs)})
#     obs, reward, done, info = env.step(action_val[0][0])
#     if  episode_rewards%100==0:
#         print(episode_rewards)
#     episode_rewards += reward