Update DDPG.py

Fine tune

Author: sweetice

Char05 DDPG/DDPG.py

@@ -28,29 +28,28 @@
 parser.add_argument('--target_update_interval', default=1, type=int)
 parser.add_argument('--test_iteration', default=10, type=int)
 
-parser.add_argument('--learning_rate', default=1e-3, type=float)
+parser.add_argument('--learning_rate', default=1e-4, type=float)
 parser.add_argument('--gamma', default=0.99, type=int) # discounted factor
-parser.add_argument('--capacity', default=50000, type=int) # replay buffer size
-parser.add_argument('--batch_size', default=64, type=int) # mini batch size
+parser.add_argument('--capacity', default=1000000, type=int) # replay buffer size
+parser.add_argument('--batch_size', default=100, type=int) # mini batch size
 parser.add_argument('--seed', default=False, type=bool)
 parser.add_argument('--random_seed', default=9527, type=int)
 # optional parameters
 
-parser.add_argument('--sample_frequency', default=256, type=int)
+parser.add_argument('--sample_frequency', default=2000, type=int)
 parser.add_argument('--render', default=False, type=bool) # show UI or not
 parser.add_argument('--log_interval', default=50, type=int) #
 parser.add_argument('--load', default=False, type=bool) # load model
 parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work
 parser.add_argument('--exploration_noise', default=0.1, type=float)
 parser.add_argument('--max_episode', default=100000, type=int) # num of games
-parser.add_argument('--max_length_of_trajectory', default=2000, type=int) # num of games
 parser.add_argument('--print_log', default=5, type=int)
-parser.add_argument('--update_iteration', default=10, type=int)
+parser.add_argument('--update_iteration', default=200, type=int)
 args = parser.parse_args()
 
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
 script_name = os.path.basename(__file__)
-env = gym.make(args.env_name).unwrapped
+env = gym.make(args.env_name)
 
 if args.seed:
     env.seed(args.random_seed)
@@ -134,14 +133,15 @@ def __init__(self, state_dim, action_dim, max_action):
         self.actor = Actor(state_dim, action_dim, max_action).to(device)
         self.actor_target = Actor(state_dim, action_dim, max_action).to(device)
         self.actor_target.load_state_dict(self.actor.state_dict())
-        self.actor_optimizer = optim.Adam(self.actor.parameters(), args.learning_rate)
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=1e-4)
 
         self.critic = Critic(state_dim, action_dim).to(device)
         self.critic_target = Critic(state_dim, action_dim).to(device)
         self.critic_target.load_state_dict(self.critic.state_dict())
-        self.critic_optimizer = optim.Adam(self.critic.parameters(), args.learning_rate)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=1e-3)
         self.replay_buffer = Replay_buffer()
         self.writer = SummaryWriter(directory)
+
         self.num_critic_update_iteration = 0
         self.num_actor_update_iteration = 0
         self.num_training = 0
@@ -158,12 +158,12 @@ def update(self):
             state = torch.FloatTensor(x).to(device)
             action = torch.FloatTensor(u).to(device)
             next_state = torch.FloatTensor(y).to(device)
-            done = torch.FloatTensor(d).to(device)
+            done = torch.FloatTensor(1-d).to(device)
             reward = torch.FloatTensor(r).to(device)
 
             # Compute the target Q value
             target_Q = self.critic_target(next_state, self.actor_target(next_state))
-            target_Q = reward + ((1 - done) * args.gamma * target_Q).detach()
+            target_Q = reward + (done * args.gamma * target_Q).detach()
 
             # Get current Q estimate
             current_Q = self.critic(state, action)
@@ -228,39 +228,33 @@ def main():
                 state = next_state
 
     elif args.mode == 'train':
-        print("====================================")
-        print("Collection Experience...")
-        print("====================================")
         if args.load: agent.load()
+        total_step = 0
         for i in range(args.max_episode):
+            total_reward = 0
+            step =0
             state = env.reset()
             for t in count():
                 action = agent.select_action(state)
-
-                # issue 3 add noise to action
                 action = (action + np.random.normal(0, args.exploration_noise, size=env.action_space.shape[0])).clip(
                     env.action_space.low, env.action_space.high)
 
                 next_state, reward, done, info = env.step(action)
-                ep_r += reward
                 if args.render and i >= args.render_interval : env.render()
                 agent.replay_buffer.push((state, next_state, action, reward, np.float(done)))
-                # if (i+1) % 10 == 0:
-                #     print('Episode {},  The memory size is {} '.format(i, len(agent.replay_buffer.storage)))
 
                 state = next_state
-                if done or t >= args.max_length_of_trajectory:
-                    agent.writer.add_scalar('ep_r', ep_r, global_step=i)
-                    if i % args.print_log == 0:
-                        print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t))
-                    ep_r = 0
+                if done:
                     break
+                step += 1
+                total_reward += reward
+            total_step += step+1
+            print("Total T:{} Episode: \t{} Total Reward: \t{:0.2f}".format(total_step, i, total_reward))
+            agent.update()
+           # "Total T: %d Episode Num: %d Episode T: %d Reward: %f
 
             if i % args.log_interval == 0:
                 agent.save()
-            if len(agent.replay_buffer.storage) >= args.capacity-1:
-                agent.update()
-
     else:
         raise NameError("mode wrong!!!")