SAC discrete updated

Author: p-christ

agents/Base_Agent.py

@@ -6,7 +6,7 @@
 import numpy as np
 import torch
 import time
-import tensorflow as tf
+# import tensorflow as tf
 from nn_builder.pytorch.NN import NN
 # from tensorboardX import SummaryWriter
 from torch.optim import optimizer
@@ -139,7 +139,7 @@ def set_random_seeds(self, random_seed):
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
         torch.manual_seed(random_seed)
-        tf.set_random_seed(random_seed)
+        # tf.set_random_seed(random_seed)
         random.seed(random_seed)
         np.random.seed(random_seed)
         if torch.cuda.is_available():

agents/DQN_agents/DQN.py

@@ -17,7 +17,7 @@ def __init__(self, config):
         self.memory = Replay_Buffer(self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"], config.seed)
         self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size)
         self.q_network_optimizer = optim.Adam(self.q_network_local.parameters(),
-                                              lr=self.hyperparameters["learning_rate"])
+                                              lr=self.hyperparameters["learning_rate"], eps=1e-4)
         self.exploration_strategy = Epsilon_Greedy_Exploration(config)
 
     def reset_game(self):

agents/DQN_agents/Dueling_DDQN.py

@@ -1,7 +1,7 @@
 import torch
 from torch import optim
 from agents.Base_Agent import Base_Agent
-from .DDQN import DDQN
+from agents.DQN_agents.DDQN import DDQN
 
 class Dueling_DDQN(DDQN):
     """A dueling double DQN agent as described in the paper http://proceedings.mlr.press/v48/wangf16.pdf"""
@@ -10,7 +10,7 @@ class Dueling_DDQN(DDQN):
     def __init__(self, config):
         DDQN.__init__(self, config)
         self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size + 1)
-        self.q_network_optimizer = optim.Adam(self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"])
+        self.q_network_optimizer = optim.Adam(self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4)
         self.q_network_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size + 1)
         Base_Agent.copy_model_over(from_model=self.q_network_local, to_model=self.q_network_target)
 

agents/actor_critic_agents/A2C.py

@@ -1,4 +1,4 @@
-from .A3C import A3C
+from agents.actor_critic_agents.A3C import A3C
 
 class A2C(A3C):
     """Synchronous version of A2C algorithm from deepmind paper https://arxiv.org/pdf/1602.01783.pdf. The only

agents/actor_critic_agents/A3C.py

@@ -17,7 +17,7 @@ def __init__(self, config):
         self.num_processes = multiprocessing.cpu_count()
         self.worker_processes = max(1, self.num_processes - 2)
         self.actor_critic = self.create_NN(input_dim=self.state_size, output_dim=[self.action_size, 1])
-        self.actor_critic_optimizer = SharedAdam(self.actor_critic.parameters(), lr=self.hyperparameters["learning_rate"])
+        self.actor_critic_optimizer = SharedAdam(self.actor_critic.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4)
 
     def run_n_episodes(self):
         """Runs game to completion n times and then summarises results and saves model (if asked to)"""
@@ -89,7 +89,7 @@ def __init__(self, worker_num, environment, shared_model, counter, optimizer_loc
         self.set_seeds(self.worker_num)
         self.shared_model = shared_model
         self.local_model = local_model
-        self.local_optimizer = Adam(self.local_model.parameters(), lr=0.0)
+        self.local_optimizer = Adam(self.local_model.parameters(), lr=0.0, eps=1e-4)
         self.counter = counter
         self.optimizer_lock = optimizer_lock
         self.shared_optimizer = shared_optimizer

agents/actor_critic_agents/DDPG.py

@@ -17,15 +17,15 @@ def __init__(self, config):
         Base_Agent.copy_model_over(self.critic_local, self.critic_target)
 
         self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
-                                           lr=self.hyperparameters["Critic"]["learning_rate"])
+                                           lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.memory = Replay_Buffer(self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"],
                                     self.config.seed)
         self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
         self.actor_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
         Base_Agent.copy_model_over(self.actor_local, self.actor_target)
 
         self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"])
+                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
         self.exploration_strategy = OU_Noise_Exploration(self.config)
 
     def step(self):

agents/actor_critic_agents/SAC.py

@@ -26,9 +26,9 @@ def __init__(self, config):
         self.critic_local_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
                                            key_to_use="Critic", override_seed=self.config.seed + 1)
         self.critic_optimizer = torch.optim.Adam(self.critic_local.parameters(),
-                                                 lr=self.hyperparameters["Critic"]["learning_rate"])
+                                                 lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.critic_optimizer_2 = torch.optim.Adam(self.critic_local_2.parameters(),
-                                                   lr=self.hyperparameters["Critic"]["learning_rate"])
+                                                   lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
                                            key_to_use="Critic")
         self.critic_target_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1,
@@ -39,13 +39,13 @@ def __init__(self, config):
                                     self.config.seed)
         self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size * 2, key_to_use="Actor")
         self.actor_optimizer = torch.optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"])
+                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
         self.automatic_entropy_tuning = self.hyperparameters["automatically_tune_entropy_hyperparameter"]
         if self.automatic_entropy_tuning:
             self.target_entropy = -torch.prod(torch.Tensor(self.environment.action_space.shape).to(self.device)).item() # heuristic value from the paper
             self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device)
             self.alpha = self.log_alpha.exp()
-            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"])
+            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
         else:
             self.alpha = self.hyperparameters["entropy_term_weight"]
 

agents/actor_critic_agents/SAC_Discrete.py

@@ -4,7 +4,7 @@
 import numpy as np
 from agents.Base_Agent import Base_Agent
 from utilities.data_structures.Replay_Buffer import Replay_Buffer
-from .SAC import SAC
+from agents.actor_critic_agents.SAC import SAC
 from utilities.Utility_Functions import create_actor_distribution
 
 class SAC_Discrete(SAC):
@@ -20,9 +20,9 @@ def __init__(self, config):
         self.critic_local_2 = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
                                            key_to_use="Critic", override_seed=self.config.seed + 1)
         self.critic_optimizer = torch.optim.Adam(self.critic_local.parameters(),
-                                                 lr=self.hyperparameters["Critic"]["learning_rate"])
+                                                 lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.critic_optimizer_2 = torch.optim.Adam(self.critic_local_2.parameters(),
-                                                   lr=self.hyperparameters["Critic"]["learning_rate"])
+                                                   lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.critic_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
                                            key_to_use="Critic")
         self.critic_target_2 = self.create_NN(input_dim=self.state_size, output_dim=self.action_size,
@@ -34,14 +34,14 @@ def __init__(self, config):
 
         self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor")
         self.actor_optimizer = torch.optim.Adam(self.actor_local.parameters(),
-                                          lr=self.hyperparameters["Actor"]["learning_rate"])
+                                          lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
         self.automatic_entropy_tuning = self.hyperparameters["automatically_tune_entropy_hyperparameter"]
         if self.automatic_entropy_tuning:
             # we set the max possible entropy as the target entropy
             self.target_entropy = -np.log((1.0 / self.action_size)) * 0.98
             self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device)
             self.alpha = self.log_alpha.exp()
-            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"])
+            self.alpha_optim = Adam([self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4)
         else:
             self.alpha = self.hyperparameters["entropy_term_weight"]
         assert not self.hyperparameters["add_extra_noise"], "There is no add extra noise option for the discrete version of SAC at moment"
@@ -65,11 +65,11 @@ def calculate_critic_losses(self, state_batch, action_batch, reward_batch, next_
         """Calculates the losses for the two critics. This is the ordinary Q-learning loss except the additional entropy
          term is taken into account"""
         with torch.no_grad():
-            next_state_action, (_, log_action_probabilities), _ = self.produce_action_and_action_info(next_state_batch)
-            next_state_log_pi = log_action_probabilities.gather(1, next_state_action.unsqueeze(-1).long())
-            qf1_next_target = self.critic_target(next_state_batch).gather(1, next_state_action.unsqueeze(-1).long())
-            qf2_next_target = self.critic_target_2(next_state_batch).gather(1, next_state_action.unsqueeze(-1).long())
-            min_qf_next_target = torch.min(qf1_next_target, qf2_next_target) - self.alpha * next_state_log_pi
+            next_state_action, (action_probabilities, log_action_probabilities), _ = self.produce_action_and_action_info(next_state_batch)
+            qf1_next_target = self.critic_target(next_state_batch)
+            qf2_next_target = self.critic_target_2(next_state_batch)
+            min_qf_next_target = action_probabilities * (torch.min(qf1_next_target, qf2_next_target) - self.alpha * log_action_probabilities)
+            min_qf_next_target = min_qf_next_target.mean(dim=1).unsqueeze(-1)
             next_q_value = reward_batch + (1.0 - mask_batch) * self.hyperparameters["discount_rate"] * (min_qf_next_target)
             self.critic_target(next_state_batch).gather(1, next_state_action.unsqueeze(-1).long())
 

agents/actor_critic_agents/TD3.py

@@ -18,7 +18,7 @@ def __init__(self, config):
                                             key_to_use="Critic")
         Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2)
         self.critic_optimizer_2 = optim.Adam(self.critic_local_2.parameters(),
-                                           lr=self.hyperparameters["Critic"]["learning_rate"])
+                                           lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4)
         self.exploration_strategy_critic = Gaussian_Exploration(self.config)
 
     def compute_critic_values_for_next_states(self, next_states):

agents/policy_gradient_agents/PPO.py

@@ -18,7 +18,7 @@ def __init__(self, config):
         self.policy_new = self.create_NN(input_dim=self.state_size, output_dim=self.policy_output_size)
         self.policy_old = self.create_NN(input_dim=self.state_size, output_dim=self.policy_output_size)
         self.policy_old.load_state_dict(copy.deepcopy(self.policy_new.state_dict()))
-        self.policy_new_optimizer = optim.Adam(self.policy_new.parameters(), lr=self.hyperparameters["learning_rate"])
+        self.policy_new_optimizer = optim.Adam(self.policy_new.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4)
         self.episode_number = 0
         self.many_episode_states = []
         self.many_episode_actions = []