Skip to content

Commit aa311eb

Browse files
vpjvpj
committed
experiemnt logs
1 parent 4d4cbef commit aa311eb

File tree

8 files changed

+592
-478
lines changed

8 files changed

+592
-478
lines changed

‎docs/experiments/nlp_autoregression.html‎

Lines changed: 192 additions & 168 deletions
Large diffs are not rendered by default.

‎docs/experiments/nlp_classification.html‎

Lines changed: 185 additions & 161 deletions
Large diffs are not rendered by default.

‎docs/normalization/deep_norm/experiment.html‎

Lines changed: 74 additions & 74 deletions
Large diffs are not rendered by default.

‎docs/normalization/deep_norm/index.html‎

Lines changed: 49 additions & 49 deletions
Large diffs are not rendered by default.

‎docs/sitemap.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -190,14 +190,14 @@
190190

191191
<url>
192192
<loc>https://nn.labml.ai/normalization/deep_norm/index.html</loc>
193-
<lastmod>2022-04-10T16:30:00+00:00</lastmod>
193+
<lastmod>2022-04-23T16:30:00+00:00</lastmod>
194194
<priority>1.00</priority>
195195
</url>
196196

197197

198198
<url>
199199
<loc>https://nn.labml.ai/normalization/deep_norm/experiment.html</loc>
200-
<lastmod>2022-04-10T16:30:00+00:00</lastmod>
200+
<lastmod>2022-04-23T16:30:00+00:00</lastmod>
201201
<priority>1.00</priority>
202202
</url>
203203

@@ -260,7 +260,7 @@
260260

261261
<url>
262262
<loc>https://nn.labml.ai/resnet/index.html</loc>
263-
<lastmod>2021-10-21T16:30:00+00:00</lastmod>
263+
<lastmod>2022-04-10T16:30:00+00:00</lastmod>
264264
<priority>1.00</priority>
265265
</url>
266266

‎labml_nn/experiments/nlp_autoregression.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,16 @@ class NLPAutoRegressionConfigs(TrainValidConfigs):
9191
# Data loaders shuffle with replacement
9292
dataloader_shuffle_with_replacement: bool = False
9393

94+
# Whether to log model parameters and gradients (once per epoch).
95+
# These are summarized stats per layer, but it could still lead
96+
# to many indicators for very deep networks.
97+
is_log_model_params_grads: bool = False
98+
99+
# Whether to log model activations (once per epoch).
100+
# These are summarized stats per layer, but it could still lead
101+
# to many indicators for very deep networks.
102+
is_log_model_activations: bool = False
103+
94104
def init(self):
95105
"""
96106
### Initialization
@@ -126,7 +136,7 @@ def step(self, batch: any, batch_idx: BatchIndex):
126136
tracker.add_global_step(data.shape[0] * data.shape[1])
127137

128138
# Whether to capture model outputs
129-
with self.mode.update(is_log_activations=batch_idx.is_last):
139+
with self.mode.update(is_log_activations=batch_idx.is_last and self.is_log_model_activations):
130140
# Get model outputs.
131141
# It's returning a tuple for states when using RNNs.
132142
# This is not implemented yet. 😜
@@ -151,7 +161,7 @@ def step(self, batch: any, batch_idx: BatchIndex):
151161
# Take optimizer step
152162
self.optimizer.step()
153163
# Log the model parameters and gradients on last batch of every epoch
154-
if batch_idx.is_last:
164+
if batch_idx.is_last and self.is_log_model_params_grads:
155165
tracker.add('model', self.model)
156166
# Clear the gradients
157167
self.optimizer.zero_grad()

‎labml_nn/experiments/nlp_classification.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,16 @@ class NLPClassificationConfigs(TrainValidConfigs):
7474
# Validation data loader
7575
valid_loader: DataLoader = 'ag_news'
7676

77+
# Whether to log model parameters and gradients (once per epoch).
78+
# These are summarized stats per layer, but it could still lead
79+
# to many indicators for very deep networks.
80+
is_log_model_params_grads: bool = False
81+
82+
# Whether to log model activations (once per epoch).
83+
# These are summarized stats per layer, but it could still lead
84+
# to many indicators for very deep networks.
85+
is_log_model_activations: bool = False
86+
7787
def init(self):
7888
"""
7989
### Initialization
@@ -102,7 +112,7 @@ def step(self, batch: any, batch_idx: BatchIndex):
102112
tracker.add_global_step(data.shape[1])
103113

104114
# Whether to capture model outputs
105-
with self.mode.update(is_log_activations=batch_idx.is_last):
115+
with self.mode.update(is_log_activations=batch_idx.is_last and self.is_log_model_activations):
106116
# Get model outputs.
107117
# It's returning a tuple for states when using RNNs.
108118
# This is not implemented yet. 😜
@@ -125,7 +135,7 @@ def step(self, batch: any, batch_idx: BatchIndex):
125135
# Take optimizer step
126136
self.optimizer.step()
127137
# Log the model parameters and gradients on last batch of every epoch
128-
if batch_idx.is_last:
138+
if batch_idx.is_last and self.is_log_model_params_grads:
129139
tracker.add('model', self.model)
130140
# Clear the gradients
131141
self.optimizer.zero_grad()

‎labml_nn/transformers/basic/autoregressive_experiment.ipynb‎

Lines changed: 65 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717
{
1818
"cell_type": "markdown",
1919
"metadata": {
20-
"id": "AYV_dMVDxyc2"
20+
"id": "AYV_dMVDxyc2",
21+
"pycharm": {
22+
"name": "#%% md\n"
23+
}
2124
},
2225
"source": [
2326
"[![Github](https://img.shields.io/github/stars/labmlai/annotated_deep_learning_paper_implementations?style=social)](https://github.com/labmlai/annotated_deep_learning_paper_implementations)\n",
@@ -34,7 +37,10 @@
3437
{
3538
"cell_type": "markdown",
3639
"metadata": {
37-
"id": "AahG_i2y5tY9"
40+
"id": "AahG_i2y5tY9",
41+
"pycharm": {
42+
"name": "#%% md\n"
43+
}
3844
},
3945
"source": [
4046
"Install the packages"
@@ -47,7 +53,10 @@
4753
"colab": {
4854
"base_uri": "https://localhost:8080/"
4955
},
50-
"outputId": "cf107fb2-4d50-4c67-af34-367624553421"
56+
"outputId": "cf107fb2-4d50-4c67-af34-367624553421",
57+
"pycharm": {
58+
"name": "#%%\n"
59+
}
5160
},
5261
"source": [
5362
"!pip install labml-nn comet_ml"
@@ -58,7 +67,10 @@
5867
{
5968
"cell_type": "markdown",
6069
"metadata": {
61-
"id": "SE2VUQ6L5zxI"
70+
"id": "SE2VUQ6L5zxI",
71+
"pycharm": {
72+
"name": "#%% md\n"
73+
}
6274
},
6375
"source": [
6476
"Imports"
@@ -67,7 +79,10 @@
6779
{
6880
"cell_type": "code",
6981
"metadata": {
70-
"id": "0hJXx_g0wS2C"
82+
"id": "0hJXx_g0wS2C",
83+
"pycharm": {
84+
"name": "#%%\n"
85+
}
7186
},
7287
"source": [
7388
"import torch\n",
@@ -112,7 +127,10 @@
112127
{
113128
"cell_type": "markdown",
114129
"metadata": {
115-
"id": "Lpggo0wM6qb-"
130+
"id": "Lpggo0wM6qb-",
131+
"pycharm": {
132+
"name": "#%% md\n"
133+
}
116134
},
117135
"source": [
118136
"Create an experiment"
@@ -121,7 +139,10 @@
121139
{
122140
"cell_type": "code",
123141
"metadata": {
124-
"id": "bFcr9k-l4cAg"
142+
"id": "bFcr9k-l4cAg",
143+
"pycharm": {
144+
"name": "#%%\n"
145+
}
125146
},
126147
"source": [
127148
"experiment.create(name=\"transformer\", writers={'screen', 'web_api', 'comet'})"
@@ -132,7 +153,10 @@
132153
{
133154
"cell_type": "markdown",
134155
"metadata": {
135-
"id": "-OnHLi626tJt"
156+
"id": "-OnHLi626tJt",
157+
"pycharm": {
158+
"name": "#%% md\n"
159+
}
136160
},
137161
"source": [
138162
"Initialize configurations"
@@ -141,7 +165,10 @@
141165
{
142166
"cell_type": "code",
143167
"metadata": {
144-
"id": "Piz0c5f44hRo"
168+
"id": "Piz0c5f44hRo",
169+
"pycharm": {
170+
"name": "#%%\n"
171+
}
145172
},
146173
"source": [
147174
"conf = Configs()"
@@ -152,7 +179,10 @@
152179
{
153180
"cell_type": "markdown",
154181
"metadata": {
155-
"id": "wwMzCqpD6vkL"
182+
"id": "wwMzCqpD6vkL",
183+
"pycharm": {
184+
"name": "#%% md\n"
185+
}
156186
},
157187
"source": [
158188
"Set experiment configurations and assign a configurations dictionary to override configurations"
@@ -166,7 +196,10 @@
166196
"height": 17
167197
},
168198
"id": "e6hmQhTw4nks",
169-
"outputId": "29634715-42f4-4405-fb11-fc9522608627"
199+
"outputId": "29634715-42f4-4405-fb11-fc9522608627",
200+
"pycharm": {
201+
"name": "#%%\n"
202+
}
170203
},
171204
"source": [
172205
"experiment.configs(conf, {\n",
@@ -205,7 +238,10 @@
205238
{
206239
"cell_type": "markdown",
207240
"metadata": {
208-
"id": "EvI7MtgJ61w5"
241+
"id": "EvI7MtgJ61w5",
242+
"pycharm": {
243+
"name": "#%% md\n"
244+
}
209245
},
210246
"source": [
211247
"Set PyTorch models for loading and saving"
@@ -219,7 +255,10 @@
219255
"height": 255
220256
},
221257
"id": "GDlt7dp-5ALt",
222-
"outputId": "e7548e8f-c541-4618-dc5a-1597cae42003"
258+
"outputId": "e7548e8f-c541-4618-dc5a-1597cae42003",
259+
"pycharm": {
260+
"name": "#%%\n"
261+
}
223262
},
224263
"source": [
225264
"experiment.add_pytorch_models({'model': conf.model})"
@@ -230,7 +269,10 @@
230269
{
231270
"cell_type": "markdown",
232271
"metadata": {
233-
"id": "KJZRf8527GxL"
272+
"id": "KJZRf8527GxL",
273+
"pycharm": {
274+
"name": "#%% md\n"
275+
}
234276
},
235277
"source": [
236278
"Start the experiment and run the training loop."
@@ -244,7 +286,10 @@
244286
"height": 1000
245287
},
246288
"id": "aIAWo7Fw5DR8",
247-
"outputId": "db979785-bfe3-4eda-d3eb-8ccbe61053e5"
289+
"outputId": "db979785-bfe3-4eda-d3eb-8ccbe61053e5",
290+
"pycharm": {
291+
"name": "#%%\n"
292+
}
248293
},
249294
"source": [
250295
"# Start the experiment\n",
@@ -257,11 +302,12 @@
257302
{
258303
"cell_type": "code",
259304
"metadata": {
260-
"id": "oBXXlP2b7XZO"
305+
"id": "oBXXlP2b7XZO",
306+
"pycharm": {
307+
"name": "#%%\n"
308+
}
261309
},
262-
"source": [
263-
""
264-
],
310+
"source": [],
265311
"execution_count": null,
266312
"outputs": []
267313
}

0 commit comments

Comments
 (0)