fix some commas and indexes (labmlai#106)

* fix some commas and indexes

* equation in GATv2 edited accordingly to paper in archive.org

Author: astramax57

labml_nn/graphs/gat/__init__.py

@@ -140,10 +140,10 @@ def forward(self, h: torch.Tensor, adj_mat: torch.Tensor):
         g_repeat_interleave = g.repeat_interleave(n_nodes, dim=0)
         # Now we concatenate to get
         # $$\{\overrightarrow{g_1} \Vert \overrightarrow{g_1},
-        # \overrightarrow{g_1}, \Vert \overrightarrow{g_2},
+        # \overrightarrow{g_1} \Vert \overrightarrow{g_2},
         # \dots, \overrightarrow{g_1}  \Vert \overrightarrow{g_N},
         # \overrightarrow{g_2} \Vert \overrightarrow{g_1},
-        # \overrightarrow{g_2}, \Vert \overrightarrow{g_2},
+        # \overrightarrow{g_2} \Vert \overrightarrow{g_2},
         # \dots, \overrightarrow{g_2}  \Vert \overrightarrow{g_N}, ...\}$$
         g_concat = torch.cat([g_repeat_interleave, g_repeat], dim=-1)
         # Reshape so that `g_concat[i, j]` is $\overrightarrow{g_i} \Vert \overrightarrow{g_j}$
@@ -170,7 +170,7 @@ def forward(self, h: torch.Tensor, adj_mat: torch.Tensor):
 
         # We then normalize attention scores (or coefficients)
         # $$\alpha_{ij} = \text{softmax}_j(e_{ij}) =
-        # \frac{\exp(e_{ij})}{\sum_{j \in \mathcal{N}_i} \exp(e_{ij})}$$
+        # \frac{\exp(e_{ij})}{\sum_{k \in \mathcal{N}_i} \exp(e_{ik})}$$
         #
         # where $\mathcal{N}_i$ is the set of nodes connected to $i$.
         #

labml_nn/graphs/gatv2/__init__.py

@@ -184,10 +184,10 @@ def forward(self, h: torch.Tensor, adj_mat: torch.Tensor):
         g_r_repeat_interleave = g_r.repeat_interleave(n_nodes, dim=0)
         # Now we add the two tensors to get
         # $$\{\overrightarrow{{g_l}_1} + \overrightarrow{{g_r}_1},
-        # \overrightarrow{{g_l}_1}, + \overrightarrow{{g_r}_2},
+        # \overrightarrow{{g_l}_1} + \overrightarrow{{g_r}_2},
         # \dots, \overrightarrow{{g_l}_1}  +\overrightarrow{{g_r}_N},
         # \overrightarrow{{g_l}_2} + \overrightarrow{{g_r}_1},
-        # \overrightarrow{{g_l}_2}, + \overrightarrow{{g_r}_2},
+        # \overrightarrow{{g_l}_2} + \overrightarrow{{g_r}_2},
         # \dots, \overrightarrow{{g_l}_2}  + \overrightarrow{{g_r}_N}, ...\}$$
         g_sum = g_l_repeat + g_r_repeat_interleave
         # Reshape so that `g_sum[i, j]` is $\overrightarrow{{g_l}_i} + \overrightarrow{{g_r}_j}$
@@ -214,7 +214,7 @@ def forward(self, h: torch.Tensor, adj_mat: torch.Tensor):
 
         # We then normalize attention scores (or coefficients)
         # $$\alpha_{ij} = \text{softmax}_j(e_{ij}) =
-        # \frac{\exp(e_{ij})}{\sum_{j \in \mathcal{N}_i} \exp(e_{ij})}$$
+        # \frac{\exp(e_{ij})}{\sum_{j' \in \mathcal{N}_i} \exp(e_{ij'})}$$
         #
         # where $\mathcal{N}_i$ is the set of nodes connected to $i$.
         #