强化学习--DDPG---tensorflow实现

完整代码：https://github.com/zle1992/Reinforcement_Learning_Game

论文《Continuous control with deep reinforcement learning》https://arxiv.org/pdf/1509.02971.pdf

Deep_Deterministic_Policy_Gradient

DDPG与AC的区别：

AC:

　　Actor: 利用td_error更新参数，td_error 来自Critic

　　Critic:根据value(s)函数的贝尔曼方程更新梯度

DDPG:

　　Actor:   maximize the q，输出action

　　Critic：根据Q(s,a)函数的贝尔曼方程更新梯度, 输出q值

DDPG 只能预测连续的动作输出。

逻辑梳理：

1、DDPG是AC 模型，输入包括（S,R,S_,A）

2、Actor

intput:(S)

output: a

loss :max(q)

q 来自Critic

3、Critic

input : S 、A

output: q

loss: R+ GAMMA * q_  - q

问题来了，q_ how to get? ---->Critic网络可以输入（S_,a_）得到q_ 但是，不能用同一个网络啊，所以，利用错位时间，我们使用Critic2（不可训练的）

Critic2需要a_ how to get?/----->Action网络可以输出（S_）得到a_，同理，我们使用Actor2(不可训练的)得到a_

流程

a = actor(s ,trian)

a_ = actor(s_,not_train)

q  = critic(s,a  trian)

q_critic(s_,a_,not_train)

a_loss = max(q)

c_loss = R+ GAMMA * q_  - q

代码：

DDPY.py

 import os
 import numpy as np
 import tensorflow as tf
 from abc import ABCMeta, abstractmethod
 np.random.seed(1)
 tf.set_random_seed(1)

 import logging  # 引入logging模块
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')  # logging.basicConfig函数对日志的输出格式及方式做相关配置
 # 由于日志基本配置中级别设置为DEBUG，所以一下打印信息将会全部显示在控制台上

 tfconfig = tf.ConfigProto()
 tfconfig.gpu_options.allow_growth = True
 session = tf.Session(config=tfconfig)

 class DDPG(object):
     __metaclass__ = ABCMeta
     """docstring for ACNetwork"""
     def __init__(self,
             n_actions,
             n_features,
             reward_decay,
             lr_a,
             lr_c,
             memory_size,
             output_graph,
             log_dir,
             model_dir,
             TAU,
             a_bound,
             ):
         super(DDPG, self).__init__()

         self.n_actions = n_actions
         self.n_features = n_features
         self.gamma=reward_decay
         self.memory_size =memory_size
         self.output_graph=output_graph
         self.lr_a =lr_a
         self.lr_c = lr_c
         self.log_dir = log_dir

         self.model_dir = model_dir
         # total learning step
         self.learn_step_counter = 0
         self.TAU = TAU     # soft replacement
         self.a_bound = a_bound

         self.s = tf.placeholder(tf.float32,[None]+self.n_features,name='s')
         self.s_next = tf.placeholder(tf.float32,[None]+self.n_features,name='s_next')
         self.r = tf.placeholder(tf.float32,[None,],name='r')

         #self.a = tf.placeholder(tf.int32,[None,1],name='a')

         with tf.variable_scope('Actor'):
             self.a = self._build_a_net(self.s, scope='eval', trainable=True)
             a_ = self._build_a_net(self.s_next, scope='target', trainable=False)

         with tf.variable_scope('Critic'):

             q  = self._build_c_net(self.s, self.a,scope='eval', trainable=True)
             q_  = self._build_c_net(self.s_next, a_,scope='target', trainable=False)

         # networks parameters
         self.ae_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval')
         self.at_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target')
         self.ce_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval')
         self.ct_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target')

         with tf.variable_scope('train_op_actor'):
             self.loss_actor = -tf.reduce_mean(q)
             self.train_op_actor = tf.train.AdamOptimizer(self.lr_a).minimize(self.loss_actor,var_list=self.ae_params)  

         with tf.variable_scope('train_op_critic'):

             q_target = self.r + self.gamma * q_
             self.loss_critic =tf.losses.mean_squared_error(labels=q_target, predictions=q)
             self.train_op_critic = tf.train.AdamOptimizer(self.lr_c).minimize(self.loss_critic,var_list=self.ce_params)

             # target net replacement
         self.soft_replace = [tf.assign(t, (1 - self.TAU) * t + self.TAU * e)
                                for t, e in zip(self.at_params + self.ct_params, self.ae_params + self.ce_params)]

         self.sess = tf.Session()
         if self.output_graph:
             tf.summary.FileWriter(self.log_dir,self.sess.graph)

         self.sess.run(tf.global_variables_initializer())

         self.cost_his =[0]
         self.cost =0 

         self.saver = tf.train.Saver()

         if not os.path.exists(self.model_dir):
             os.mkdir(self.model_dir)

         checkpoint = tf.train.get_checkpoint_state(self.model_dir)
         if checkpoint and checkpoint.model_checkpoint_path:
             self.saver.restore(self.sess, checkpoint.model_checkpoint_path)
             print ("Loading Successfully")
             self.learn_step_counter = int(checkpoint.model_checkpoint_path.split('-')[-1]) + 1

     @abstractmethod
     def _build_a_net(self,x,scope,trainable):

         raise NotImplementedError
     def _build_c_net(self,x,scope,trainable):

         raise NotImplementedError
     def learn(self,data):

         # soft target replacement
         self.sess.run(self.soft_replace)

         batch_memory_s = data['s']
         batch_memory_a =  data['a']
         batch_memory_r = data['r']
         batch_memory_s_ = data['s_']

         _, cost = self.sess.run(
             [self.train_op_actor, self.loss_actor],
             feed_dict={
                 self.s: batch_memory_s,
             })

         _, cost = self.sess.run(
             [self.train_op_critic, self.loss_critic],
             feed_dict={
                 self.s: batch_memory_s,
                 self.a: batch_memory_a,
                 self.r: batch_memory_r,
                 self.s_next: batch_memory_s_,

             })

         self.cost_his.append(cost)
         self.cost =cost
         self.learn_step_counter += 1
             # save network every 100000 iteration
         if self.learn_step_counter % 10000 == 0:
             self.saver.save(self.sess,self.model_dir,global_step=self.learn_step_counter)

     def choose_action(self,s): 

         return self.sess.run(self.a, {self.s: s[np.newaxis,:]})[0]
         # s = s[np.newaxis,:]

         # probs = self.sess.run(self.acts_prob,feed_dict={self.s:s})
         # return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel())   

game.py

 import sys
 import gym
 import numpy as np
 import tensorflow as tf
 sys.path.append('./')
 sys.path.append('model')

 from util import Memory ,StateProcessor
 from DDPG import DDPG
 from ACNetwork import ACNetwork
 np.random.seed(1)
 tf.set_random_seed(1)

 import logging  # 引入logging模块
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')  # logging.basicConfig函数对日志的输出格式及方式做相关配置
 # 由于日志基本配置中级别设置为DEBUG，所以一下打印信息将会全部显示在控制台上
 import os
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 tfconfig = tf.ConfigProto()
 tfconfig.gpu_options.allow_growth = True
 session = tf.Session(config=tfconfig)

 class DDPG4Pendulum(DDPG):
     """docstring for ClassName"""
     def __init__(self, **kwargs):
         super(DDPG4Pendulum, self).__init__(**kwargs)

     def _build_a_net(self,s,scope,trainable):
         w_initializer, b_initializer = tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1)
         #w_initializer, b_initializer = None,None
         with tf.variable_scope(scope):
             e1 = tf.layers.dense(inputs=s,
                     units=30,
                     bias_initializer = b_initializer,
                     kernel_initializer=w_initializer,
                     activation = tf.nn.relu,
                     trainable=trainable)
             a = tf.layers.dense(inputs=e1,
                     units=self.n_actions,
                     bias_initializer = b_initializer,
                     kernel_initializer=w_initializer,
                     activation = tf.nn.tanh,
                     trainable=trainable) 

         return tf.multiply(a, self.a_bound, name='scaled_a')  

     def _build_c_net(self,s,a,scope,trainable):
         w_initializer, b_initializer = tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1)

         with tf.variable_scope(scope):
             n_l1 = 30
             w1_s = tf.get_variable('w1_s',self.n_features+[n_l1],trainable=trainable)
             w1_a = tf.get_variable('w1_a',[self.n_actions,n_l1],trainable=trainable)
             b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable)
             net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)

             q = tf.layers.dense(inputs=net,
                     units=1,
                     bias_initializer = b_initializer,
                     kernel_initializer=w_initializer,
                     activation =None,
                     trainable=trainable) 

         return q   

 batch_size = 32

 memory_size  =10000
 env = gym.make('Pendulum-v0') #连续

 n_features= [env.observation_space.shape[0]]
 n_actions= env.action_space.shape[0]
 a_bound = env.action_space.high
 env = env.unwrapped
 MAX_EP_STEPS =200
 def run():

     RL = DDPG4Pendulum(
         n_actions=n_actions,
         n_features=n_features,
         reward_decay=0.9,
         lr_a = 0.001,
         lr_c = 0.002,
         memory_size=memory_size,
         TAU = 0.01,
         output_graph=False,
         log_dir = 'Pendulum/log/DDPG4Pendulum/',
         a_bound =a_bound,
         model_dir = 'Pendulum/model_dir/DDPG4Pendulum/'
         )

     memory = Memory(n_actions,n_features,memory_size=memory_size)

     var = 3  # control exploration
     step = 0

     for episode in range(2000):
         # initial observation
         observation = env.reset()
         ep_r = 0

         for j in range(MAX_EP_STEPS):

             # RL choose action based on observation
             action = RL.choose_action(observation)
             action = np.clip(np.random.normal(action, var), -2, 2)    # add randomness to action selection for exploration
             # RL take action and get_collectiot next observation and reward
             observation_, reward, done, info=env.step(action) # take a random action

             #print('step:%d---episode:%d----reward:%f---action:%f'%(step,episode,reward,action))
             memory.store_transition(observation, action, reward/10, observation_)

             if step > memory_size:
                 #env.render()
                 var *= .9995    # decay the action randomness
                 data = memory.sample(batch_size)
                 RL.learn(data)

             # swap observation
             observation = observation_
             ep_r += reward
             # break while loop when end of this episode
             if(episode>200):
                 env.render()  # render on the screen
             if j == MAX_EP_STEPS-1:
                 print('step: ',step,
                     'episode: ', episode,
                       'ep_r: ', round(ep_r, 2),
                       'var:',var,
                       #loss: ',RL.cost
                       )
                 break
             step += 1

     # end of game
     print('game over')
     env.destroy()

 def main():

     run()

 if __name__ == '__main__':
     main()
     #run2()