RoPER (labmlai#126)

Author: vpj

.gitignore

@@ -15,4 +15,5 @@ logs
 html/
 diagrams/
 .comet.config
-settings.md
+settings.md
+labml_app.log

.labml.yaml

@@ -19,3 +19,4 @@ indicators:
     name: optim.*
     options:
       comet: false
+web_api: http://localhost:5005/api/v1/track?

docs/experiments/arithmetic_dataset.html

@@ -70,7 +70,8 @@
                 <a href='#section-0'>#</a>
             </div>
             <h1><a href="index.html">DeepNorm</a> Experiment</h1>
-<p><a href="https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/normalization/deep_norm/experiment.ipynb"><img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://app.labml.ai/run/ec8e4dacb7f311ec8d1cd37d50b05c3d"><img alt="View Run" src="https://img.shields.io/badge/labml-experiment-brightgreen"></a></p>
+<p><a href="https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/normalization/deep_norm/experiment.ipynb"><img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://app.labml.ai/run/ec8e4dacb7f311ec8d1cd37d50b05c3d"><img alt="View Run" src="https://img.shields.io/badge/labml-experiment-brightgreen"></a> <a href="https://www.comet.ml/labml/deep-norm/61d817f80ff143c8825fba4aacd431d4?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step"><img alt="Open In Comet" src="https://images.labml.ai/images/comet.svg?experiment=deep_norm&file=experiment"></a></p>
+
         </div>
         <div class='code'>
             <div class="highlight"><pre><span class="lineno">15</span><span></span><span class="kn">import</span> <span class="nn">copy</span>

docs/normalization/deep_norm/experiment.html

@@ -204,7 +204,7 @@
 
     <url>
       <loc>https://nn.labml.ai/normalization/deep_norm/index.html</loc>
-      <lastmod>2022-04-23T16:30:00+00:00</lastmod>
+      <lastmod>2022-05-18T16:30:00+00:00</lastmod>
       <priority>1.00</priority>
     </url>
 
@@ -244,6 +244,13 @@
     </url>
 
 
+    <url>
+      <loc>https://nn.labml.ai/experiments/arithmetic_dataset.html</loc>
+      <lastmod>2022-06-02T16:30:00+00:00</lastmod>
+      <priority>1.00</priority>
+    </url>
+    
+
     <url>
       <loc>https://nn.labml.ai/experiments/index.html</loc>
       <lastmod>2020-12-26T16:30:00+00:00</lastmod>
@@ -603,14 +610,35 @@
 
     <url>
       <loc>https://nn.labml.ai/transformers/rope/index.html</loc>
-      <lastmod>2022-04-05T16:30:00+00:00</lastmod>
+      <lastmod>2022-05-31T16:30:00+00:00</lastmod>
+      <priority>1.00</priority>
+    </url>
+    
+
+    <url>
+      <loc>https://nn.labml.ai/transformers/rope/value_pe/arithmetic_experiment.html</loc>
+      <lastmod>2022-06-02T16:30:00+00:00</lastmod>
+      <priority>1.00</priority>
+    </url>
+    
+
+    <url>
+      <loc>https://nn.labml.ai/transformers/rope/value_pe/index.html</loc>
+      <lastmod>2022-06-02T16:30:00+00:00</lastmod>
+      <priority>1.00</priority>
+    </url>
+    
+
+    <url>
+      <loc>https://nn.labml.ai/transformers/rope/value_pe/experiment.html</loc>
+      <lastmod>2022-05-31T16:30:00+00:00</lastmod>
       <priority>1.00</priority>
     </url>
 
 
     <url>
       <loc>https://nn.labml.ai/transformers/rope/experiment.html</loc>
-      <lastmod>2022-03-12T16:30:00+00:00</lastmod>
+      <lastmod>2022-05-31T16:30:00+00:00</lastmod>
       <priority>1.00</priority>
     </url>
 

docs/sitemap.xml

@@ -92,7 +92,7 @@ <h3>Rotary PE attention</h3>
         <div class='code'>
             <div class="highlight"><pre><span class="lineno">21</span><span class="k">def</span> <span class="nf">_rotary_pe_mha</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">TransformerConfigs</span><span class="p">):</span>
 <span class="lineno">22</span>    <span class="kn">from</span> <span class="nn">labml_nn.transformers.rope</span> <span class="kn">import</span> <span class="n">RotaryPEMultiHeadAttention</span>
-<span class="lineno">23</span>    <span class="k">return</span> <span class="n">RotaryPEMultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">)</span></pre></div>
+<span class="lineno">23</span>    <span class="k">return</span> <span class="n">RotaryPEMultiHeadAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">n_heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="mf">1.</span><span class="p">)</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-2'>
@@ -157,7 +157,7 @@ <h3>Rotary PE attention</h3>
 
         </div>
         <div class='code'>
-            <div class="highlight"><pre><span class="lineno">46</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;rotary_pe_transformer&quot;</span><span class="p">)</span></pre></div>
+            <div class="highlight"><pre><span class="lineno">46</span>    <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;rotary_pe_transformer&quot;</span><span class="p">,</span> <span class="n">writers</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;screen&#39;</span><span class="p">})</span></pre></div>
         </div>
     </div>
     <div class='section' id='section-7'>