headings

Author: vpj

labml_nn/cfr/__init__.py

@@ -68,6 +68,7 @@
 i.e. all those histories look the same in the eye of the player.
 
 <a id="Strategy"></a>
+
 ### Strategy
 
 **Strategy of player** $i$, $\sigma_i \in \Sigma_i$ is a distribution over actions $A(I_i)$,
@@ -84,6 +85,7 @@
 $\sigma_{-i}$ is strategies of all players except $\sigma_i$
 
 <a id="HistoryProbability"></a>
+
 ### Probability of History
 
 $\pi^\sigma(h)$ is the probability of reaching the history $h$ with strategy profile $\sigma$.
@@ -109,6 +111,7 @@
 $$u_i(\sigma) = \sum_{h \in Z} u_i(h) \pi^\sigma(h)$$
 
 <a id="NashEquilibrium"></a>
+
 ### Nash Equilibrium
 
 Nash equilibrium is a state where none of the players can increase their expected utility (or payoff)
@@ -204,6 +207,7 @@
 So we need to minimize $R^T_i$ to get close to a Nash equilibrium.
 
 <a id="CounterfactualRegret"></a>
+
 ### Counterfactual regret
 
 **Counterfactual value** $\color{pink}{v_i(\sigma, I)}$ is the expected utility for player $i$ if
@@ -235,6 +239,7 @@
 where $$R^{T,+}_{i,imm}(I) = \max(R^T_{i,imm}(I), 0)$$
 
 <a id="RegretMatching"></a>
+
 ### Regret Matching
 
 The strategy is calculated using regret matching.
@@ -271,6 +276,7 @@
 therefore reaches $\epsilon$-[Nash equilibrium](#NashEquilibrium).
 
 <a id="MCCFR"></a>
+
 ### Monte Carlo CFR (MCCFR)
 
 Computing $\color{coral}{r^t_i(I, a)}$ requires expanding the full game tree
@@ -331,6 +337,7 @@
 class History:
     """
     <a id="History"></a>
+
     ## History
 
     History $h \in H$ is a sequence of actions including chance events,
@@ -404,6 +411,7 @@ def __repr__(self):
 class InfoSet:
     """
     <a id="InfoSet"></a>
+
     ## Information Set $I_i$
     """
 

labml_nn/conv_mixer/__init__.py

@@ -45,6 +45,7 @@
 class ConvMixerLayer(Module):
     """
     <a id="ConvMixerLayer"></a>
+
     ## ConvMixer layer
 
     This is a single ConvMixer layer. The model will have a series of these.
@@ -100,6 +101,7 @@ def forward(self, x: torch.Tensor):
 class PatchEmbeddings(Module):
     """
     <a id="PatchEmbeddings"></a>
+
     ## Get patch embeddings
 
     This splits the image into patches of size $p \times p$ and gives an embedding for each patch.
@@ -139,6 +141,7 @@ def forward(self, x: torch.Tensor):
 class ClassificationHead(Module):
     """
     <a id="ClassificationHead"></a>
+
     ## Classification Head
 
     They do average pooling (taking the mean of all patch embeddings) and a final linear transformation

labml_nn/experiments/mnist.py

@@ -24,6 +24,7 @@
 class MNISTConfigs(MNISTDatasetConfigs, TrainValidConfigs):
     """
     <a id="MNISTConfigs"></a>
+
     ## Trainer configurations
     """
 

labml_nn/experiments/nlp_autoregression.py

@@ -41,6 +41,7 @@ def forward(self, outputs, targets):
 class NLPAutoRegressionConfigs(TrainValidConfigs):
     """
     <a id="NLPAutoRegressionConfigs"></a>
+
     ## Trainer configurations
 
     This has the basic configurations for NLP auto-regressive task training.

labml_nn/experiments/nlp_classification.py

@@ -29,6 +29,7 @@
 class NLPClassificationConfigs(TrainValidConfigs):
     """
     <a id="NLPClassificationConfigs"></a>
+
     ## Trainer configurations
 
     This has the basic configurations for NLP classification task training.

labml_nn/gan/stylegan/__init__.py

@@ -158,6 +158,7 @@
 class MappingNetwork(nn.Module):
     """
     <a id="mapping_network"></a>
+
     ## Mapping Network
 
     ![Mapping Network](mapping_network.svg)
@@ -196,6 +197,7 @@ def forward(self, z: torch.Tensor):
 class Generator(nn.Module):
     """
     <a id="generator"></a>
+
     ## StyleGAN2 Generator
 
     ![Generator](style_gan2.svg)
@@ -276,6 +278,7 @@ def forward(self, w: torch.Tensor, input_noise: List[Tuple[Optional[torch.Tensor
 class GeneratorBlock(nn.Module):
     """
     <a id="generator_block"></a>
+
     ### Generator Block
 
     ![Generator block](generator_block.svg)
@@ -327,6 +330,7 @@ def forward(self, x: torch.Tensor, w: torch.Tensor, noise: Tuple[Optional[torch.
 class StyleBlock(nn.Module):
     """
     <a id="style_block"></a>
+
     ### Style Block
 
     ![Style block](style_block.svg)
@@ -377,6 +381,7 @@ def forward(self, x: torch.Tensor, w: torch.Tensor, noise: Optional[torch.Tensor
 class ToRGB(nn.Module):
     """
     <a id="to_rgb"></a>
+
     ### To RGB
 
     ![To RGB](to_rgb.svg)
@@ -489,6 +494,7 @@ def forward(self, x: torch.Tensor, s: torch.Tensor):
 class Discriminator(nn.Module):
     """
     <a id="discriminator"></a>
+
     ## StyleGAN 2 Discriminator
 
     ![Discriminator](style_gan2_disc.svg)
@@ -557,6 +563,7 @@ def forward(self, x: torch.Tensor):
 class DiscriminatorBlock(nn.Module):
     """
     <a id="discriminator_black"></a>
+
     ### Discriminator Block
 
     ![Discriminator block](discriminator_block.svg)
@@ -645,6 +652,7 @@ def forward(self, x: torch.Tensor):
 class DownSample(nn.Module):
     """
     <a id="down_sample"></a>
+
     ### Down-sample
 
     The down-sample operation [smoothens](#smooth) each feature channel and
@@ -668,6 +676,7 @@ def forward(self, x: torch.Tensor):
 class UpSample(nn.Module):
     """
     <a id="up_sample"></a>
+
     ### Up-sample
 
     The up-sample operation scales the image up by $2 \times$ and [smoothens](#smooth) each feature channel.
@@ -690,6 +699,7 @@ def forward(self, x: torch.Tensor):
 class Smooth(nn.Module):
     """
     <a id="smooth"></a>
+
     ### Smoothing Layer
 
     This layer blurs each channel
@@ -729,6 +739,7 @@ def forward(self, x: torch.Tensor):
 class EqualizedLinear(nn.Module):
     """
     <a id="equalized_linear"></a>
+
     ## Learning-rate Equalized Linear Layer
 
     This uses [learning-rate equalized weights](#equalized_weights) for a linear layer.
@@ -755,6 +766,7 @@ def forward(self, x: torch.Tensor):
 class EqualizedConv2d(nn.Module):
     """
     <a id="equalized_conv2d"></a>
+
     ## Learning-rate Equalized 2D Convolution Layer
 
     This uses [learning-rate equalized weights](#equalized_weights) for a convolution layer.
@@ -784,6 +796,7 @@ def forward(self, x: torch.Tensor):
 class EqualizedWeight(nn.Module):
     """
     <a id="equalized_weight"></a>
+
     ## Learning-rate Equalized Weights Parameter
 
     This is based on equalized learning rate introduced in the Progressive GAN paper.
@@ -821,6 +834,7 @@ def forward(self):
 class GradientPenalty(nn.Module):
     """
     <a id="gradient_penalty"></a>
+
     ## Gradient Penalty
 
     This is the $R_1$ regularization penality from the paper
@@ -862,6 +876,7 @@ def forward(self, x: torch.Tensor, d: torch.Tensor):
 class PathLengthPenalty(nn.Module):
     """
     <a id="path_length_penalty"></a>
+
     ## Path Length Penalty
 
     This regularization encourages a fixed-size step in $w$ to result in a fixed-magnitude

labml_nn/optimizers/adam_warmup_cosine_decay.py

@@ -18,6 +18,7 @@
 class AdamWarmupCosineDecay(AMSGrad):
     """
     <a id="EmbeddingsWithPositionalEncoding"></a>
+
     ## Adam Optimizer with Warmup and Cosine Decay
 
     This class extends from AMSGrad optimizer defined in [`amsgrad.py`](amsgrad.html).

labml_nn/optimizers/configs.py

@@ -18,6 +18,7 @@
 class OptimizerConfigs(BaseConfigs):
     """
     <a id="OptimizerConfigs"></a>
+
     ## Optimizer Configurations
     """
 

labml_nn/resnet/__init__.py

@@ -91,6 +91,7 @@ def forward(self, x: torch.Tensor):
 class ResidualBlock(Module):
     """
     <a id="residual_block"></a>
+
     ## Residual Block
 
     This implements the residual block described in the paper.
@@ -157,6 +158,7 @@ def forward(self, x: torch.Tensor):
 class BottleneckResidualBlock(Module):
     """
     <a id="bottleneck_residual_block"></a>
+
     ## Bottleneck Residual Block
 
     This implements the bottleneck block described in the paper.

labml_nn/transformers/configs.py

@@ -21,6 +21,7 @@
 class FeedForwardConfigs(BaseConfigs):
     """
     <a id="FFN"></a>
+
     ## FFN Configurations
 
     Creates a Position-wise FeedForward Network defined in
@@ -143,6 +144,7 @@ def _feed_forward(c: FeedForwardConfigs):
 class TransformerConfigs(BaseConfigs):
     """
     <a id="TransformerConfigs"></a>
+
     ## Transformer Configurations
 
     This defines configurations for a transformer.