DeepNetwork---tensorflow实现

https://github.com/zle1992/Reinforcement_Learning_Game

DeepQNetwork.py

 import numpy as np
 import tensorflow as tf
 from abc import ABCMeta, abstractmethod
 np.random.seed(1)
 tf.set_random_seed(1)

 import logging  # 引入logging模块
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')  # logging.basicConfig函数对日志的输出格式及方式做相关配置
 # 由于日志基本配置中级别设置为DEBUG，所以一下打印信息将会全部显示在控制台上

 tfconfig = tf.ConfigProto()
 tfconfig.gpu_options.allow_growth = True
 session = tf.Session(config=tfconfig)

 class DeepQNetwork(object):
     __metaclass__ = ABCMeta
     """docstring for DeepQNetwork"""
     def __init__(self,
             n_actions,
             n_features,
             learning_rate,
             reward_decay,
             e_greedy,
             replace_target_iter,
             memory_size,
             e_greedy_increment,
             output_graph,
             log_dir,
             ):
         super(DeepQNetwork, self).__init__()

         self.n_actions = n_actions
         self.n_features = n_features
         self.learning_rate=learning_rate
         self.gamma=reward_decay
         self.epsilon_max=e_greedy
         self.replace_target_iter=replace_target_iter
         self.memory_size=memory_size
         self.epsilon_increment=e_greedy_increment
         self.output_graph=output_graph
         self.lr =learning_rate
         # total learning step
         self.learn_step_counter = 0
         self.log_dir = log_dir

         self.s = tf.placeholder(tf.float32,[None]+self.n_features,name='s')
         self.s_next = tf.placeholder(tf.float32,[None]+self.n_features,name='s_next')

         self.r = tf.placeholder(tf.float32,[None,],name='r')
         self.a = tf.placeholder(tf.int32,[None,],name='a')

         self.q_eval = self._build_q_net(self.s, scope='eval_net', trainable=True)
         self.q_next = self._build_q_net(self.s_next, scope='target_net', trainable=False)

         with tf.variable_scope('q_target'):
             self.q_target = self.r + self.gamma * tf.reduce_max(self.q_next, axis=1, name='Qmax_s_')    # shape=(None, )
         with tf.variable_scope('q_eval'):
             a_indices = tf.stack([tf.range(tf.shape(self.a)[0], dtype=tf.int32), self.a], axis=1)
             self.q_eval_wrt_a = tf.gather_nd(params=self.q_eval, indices=a_indices)    # shape=(None, )
         with tf.variable_scope('loss'):
             self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval_wrt_a, name='TD_error'))
         with tf.variable_scope('train'):
             self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)

         t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='target_net')
         e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='eval_net')

         with tf.variable_scope("hard_replacement"):
             self.target_replace_op=[tf.assign(t,e) for t,e in zip(t_params,e_params)]

         self.sess = tf.Session()
         if self.output_graph:
             tf.summary.FileWriter(self.log_dir,self.sess.graph)

         self.sess.run(tf.global_variables_initializer())

         self.cost_his =[]

     @abstractmethod
     def _build_q_net(self,x,scope,trainable):
         raise NotImplementedError

     def learn(self,data):

          # check to replace target parameters
         if self.learn_step_counter % self.replace_target_iter == 0:
             self.sess.run(self.target_replace_op)
             print('\ntarget_params_replaced\n')

         batch_memory_s = data['s'],
         batch_memory_a =  data['a'],
         batch_memory_r = data['r'],
         batch_memory_s_ = data['s_'],
         _, cost = self.sess.run(
             [self._train_op, self.loss],
             feed_dict={
                 self.s: batch_memory_s,
                 self.a: batch_memory_a,
                 self.r: batch_memory_r,
                 self.s_next: batch_memory_s_,
             })
         self.cost_his.append(cost)

         # increasing epsilon
         self.epsilon_max = self.epsilon_max + self.epsilon_increment if self.epsilon_max < self.epsilon_max else self.epsilon_max
         self.learn_step_counter += 1

     def choose_action(self,s):
         s = s[np.newaxis,:]
         aa = np.random.uniform()
         #print("epsilon_max",self.epsilon_max)
         if aa < self.epsilon_max:
             action_value = self.sess.run(self.q_eval,feed_dict={self.s:s})
             action = np.argmax(action_value)
         else:
             action = np.random.randint(0,self.n_actions)
         return action

Memory.py

 import numpy as np
 np.random.seed(1)
 class Memory(object):
     """docstring for Memory"""
     def __init__(self,
             n_actions,
             n_features,
             memory_size):
         super(Memory, self).__init__()
         self.memory_size = memory_size
         self.cnt =0 

         self.s = np.zeros([memory_size]+n_features)
         self.a = np.zeros([memory_size,])
         self.r =  np.zeros([memory_size,])
         self.s_ = np.zeros([memory_size]+n_features)

     def store_transition(self,s, a, r, s_):
         #logging.info('store_transition')
         index = self.cnt % self.memory_size
         self.s[index] = s
         self.a[index] = a
         self.r[index] =  r
         self.s_[index] =s_
         self.cnt+=1

     def sample(self,n):
         #logging.info('sample')
         #assert self.cnt>=self.memory_size,'Memory has not been fulfilled'
         N = min(self.memory_size,self.cnt)
         indices = np.random.choice(N,size=n)
         d ={}
         d['s'] = self.s[indices][0]
         d['s_'] = self.s_[indices][0]
         d['r'] = self.r[indices][0]
         d['a'] = self.a[indices][0]
         return d

主函数

 import gym
 import numpy as np
 import tensorflow as tf

 from Memory import Memory
 from DeepQNetwork import DeepQNetwork

 np.random.seed(1)
 tf.set_random_seed(1)

 import logging  # 引入logging模块
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')  # logging.basicConfig函数对日志的输出格式及方式做相关配置
 # 由于日志基本配置中级别设置为DEBUG，所以一下打印信息将会全部显示在控制台上

 tfconfig = tf.ConfigProto()
 tfconfig.gpu_options.allow_growth = True
 session = tf.Session(config=tfconfig)

 class DeepQNetwork4CartPole(DeepQNetwork):
     """docstring for ClassName"""
     def __init__(self, **kwargs):
         super(DeepQNetwork4CartPole, self).__init__(**kwargs)

     def _build_q_net(self,x,scope,trainable):
         w_initializer, b_initializer = tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1)

         with tf.variable_scope(scope):
             e1 = tf.layers.dense(inputs=x,
                     units=32,
                     bias_initializer = b_initializer,
                     kernel_initializer=w_initializer,
                     activation = tf.nn.relu,
                     trainable=trainable)
             q = tf.layers.dense(inputs=e1,
                     units=self.n_actions,
                     bias_initializer = b_initializer,
                     kernel_initializer=w_initializer,
                     activation = tf.nn.sigmoid,
                     trainable=trainable) 

         return q  

 batch_size = 64

 memory_size  =2000
 #env = gym.make('Breakout-v0') #离散
 env = gym.make('CartPole-v0') #离散

 n_features= list(env.observation_space.shape)
 n_actions= env.action_space.n

 env = env.unwrapped

 def run():

     RL = DeepQNetwork4CartPole(
         n_actions=n_actions,
         n_features=n_features,
         learning_rate=0.01,
         reward_decay=0.9,
         e_greedy=0.9,
         replace_target_iter=200,
         memory_size=memory_size,
         e_greedy_increment=None,
         output_graph=True,
         log_dir = 'log/DeepQNetwork4CartPole/',
         )

     memory = Memory(n_actions,n_features,memory_size=memory_size)

     step = 0
     ep_r = 0
     for episode in range(2000):
         # initial observation
         observation = env.reset()

         while True:

             # RL choose action based on observation
             action = RL.choose_action(observation)
             # logging.debug('action')
             # print(action)
             # RL take action and get_collectiot next observation and reward
             observation_, reward, done, info=env.step(action) # take a random action

             # the smaller theta and closer to center the better
             x, x_dot, theta, theta_dot = observation_
             r1 = (env.x_threshold - abs(x))/env.x_threshold - 0.8
             r2 = (env.theta_threshold_radians - abs(theta))/env.theta_threshold_radians - 0.5
             reward = r1 + r2

             memory.store_transition(observation, action, reward, observation_)

             if (step > 200) and (step % 5 == 0):

                 data = memory.sample(batch_size)
                 RL.learn(data)
                 #print('step:%d----reward:%f---action:%d'%(step,reward,action))
             # swap observation
             observation = observation_
             ep_r += reward
             # break while loop when end of this episode
             if(episode>700):
                 env.render()  # render on the screen
             if done:
                 print('episode: ', episode,
                       'ep_r: ', round(ep_r, 2),
                       ' epsilon: ', round(RL.epsilon_max, 2))
                 ep_r = 0

                 break
             step += 1

     # end of game
     print('game over')
     env.destroy()

 def main():

     run()

 if __name__ == '__main__':
     main()
     #run2()