notebooks/check_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare current implementation with TF implementation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Minimal model implementation in TF\n",
    "https://github.com/google-research/fixmatch/blob/08d9b83d7cc87e853e6afc5a86b12aacff56cdea/libml/models.py#L62"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install tensorflow==1.14.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'1.14.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "\n",
    "\n",
    "class ResNet:\n",
    "    \n",
    "    class Dataset: pass\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.dataset = self.Dataset()\n",
    "        self.dataset.mean = 0.0\n",
    "        self.dataset.std = 1.0    \n",
    "        self.nclass = 10\n",
    "\n",
    "    def classifier(self, x, scales=3, filters=32, repeat=4, training=False, getter=None, dropout=0, **kwargs):\n",
    "        del kwargs\n",
    "        leaky_relu = functools.partial(tf.nn.leaky_relu, alpha=0.1)\n",
    "        bn_args = dict(training=training, momentum=0.999)\n",
    "\n",
    "        def conv_args(k, f):\n",
    "            return dict(padding='same',\n",
    "                        kernel_initializer=tf.random_normal_initializer(stddev=tf.rsqrt(0.5 * k * k * f)))\n",
    "#                         kernel_initializer=tf.constant_initializer(0.00123))\n",
    "\n",
    "        def residual(x0, filters, stride=1, activate_before_residual=False):\n",
    "            x = leaky_relu(tf.layers.batch_normalization(x0, **bn_args))\n",
    "            if activate_before_residual:\n",
    "                x0 = x\n",
    "\n",
    "            x = tf.layers.conv2d(x, filters, 3, strides=stride, **conv_args(3, filters))\n",
    "            x = leaky_relu(tf.layers.batch_normalization(x, **bn_args))\n",
    "            x = tf.layers.conv2d(x, filters, 3, **conv_args(3, filters))\n",
    "\n",
    "            if x0.get_shape()[3] != filters:\n",
    "                x0 = tf.layers.conv2d(x0, filters, 1, strides=stride, **conv_args(1, filters))\n",
    "\n",
    "            return x0 + x\n",
    "\n",
    "        out = {}\n",
    "        \n",
    "        with tf.variable_scope('classify', reuse=tf.AUTO_REUSE, custom_getter=getter):\n",
    "            out[\"c1\"] = y = tf.layers.conv2d((x - self.dataset.mean) / self.dataset.std, 16, 3, **conv_args(3, 16))\n",
    "            for scale in range(scales):\n",
    "                out[\"res_{}\".format(scale)] = y = residual(y, filters << scale, stride=2 if scale else 1, activate_before_residual=scale == 0)\n",
    "                for i in range(repeat - 1):\n",
    "                    out[\"res_{}_{}\".format(scale, i)] = y = residual(y, filters << scale)\n",
    "\n",
    "            embeds = y = leaky_relu(tf.layers.batch_normalization(y, **bn_args))\n",
    "            y = tf.reduce_mean(y, [1, 2])\n",
    "            if dropout and training:\n",
    "                y = tf.nn.dropout(y, 1 - dropout)\n",
    "            logits = tf.layers.dense(y, self.nclass, kernel_initializer=tf.glorot_normal_initializer())\n",
    "#             logits = tf.layers.dense(y, self.nclass, kernel_initializer=tf.constant_initializer(0.01432))\n",
    "\n",
    "        return logits, embeds, out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.set_random_seed(0)\n",
    "tf_model = ResNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-b0702be9988b>:41: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv2D` instead.\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:From <ipython-input-4-b0702be9988b>:25: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.BatchNormalization instead.  In particular, `tf.control_dependencies(tf.GraphKeys.UPDATE_OPS)` should not be used (consult the `tf.keras.layers.batch_normalization` documentation).\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce06efe4d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce06efe4d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce06efe4d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce06efe4d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a3a710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a3a710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a3a710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a3a710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce446b5050>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce446b5050>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce446b5050>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce446b5050>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a57a50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce00a57a50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce0002ced0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce0002ced0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce0002ced0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce0002ced0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce01093910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfff52390>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfff52390>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfff52390>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfff52390>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce4470a6d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fce02c09d50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff4f46d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff4f46d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff4f46d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff4f46d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfffa7b90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfffa7b90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfffa7b90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdfffa7b90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff567290>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff567290>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff567290>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff567290>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff379590>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff379590>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff379590>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff379590>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff4fc710>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff45d1d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff45d1d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff45d1d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff45d1d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff39a910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff39a910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff39a910>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff39a910>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff27ddd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff27ddd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff27ddd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff27ddd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff394490>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff394490>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff394490>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff394490>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce000c8090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce000c8090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce000c8090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fce000c8090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdffed41d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdffed41d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdffed41d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdffed41d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff387f50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff387f50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff387f50>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff387f50>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff160fd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff160fd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff160fd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff160fd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdffed41d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdffed41d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdffed41d0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdffed41d0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff561750>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff160fd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff160fd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff160fd0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff160fd0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff650e90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff650e90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff650e90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff650e90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff30bed0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff30bed0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff30bed0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff30bed0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff014050>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff014050>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff014050>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff014050>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff019590>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff019590>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff019590>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff019590>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff379a10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdfeea5190>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdfeea5190>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdfeea5190>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x7fcdfeea5190>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x7fcdff297090>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1288: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From <ipython-input-4-b0702be9988b>:51: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fcdff637c10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fcdff637c10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fcdff637c10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7fcdff637c10>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "tf_x = tf.placeholder(tf.float32, [None] + [32, 32, 3], 'x')\n",
    "tf_logits, tf_embeds, tf_out = tf_model.classifier(tf_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_dict = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)\n",
    "weights_dict = {v.name: v for v in weights_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "np_x = np.random.rand(8, 32, 32, 3).astype(\"float32\")\n",
    "\n",
    "\n",
    "with tf.Session() as s:\n",
    "    s.run(tf.global_variables_initializer())    \n",
    "    res = s.run(\n",
    "        [tf_logits, tf_embeds, tf_out],\n",
    "        feed_dict={\n",
    "            tf_x: np_x\n",
    "        }\n",
    "    )\n",
    "    np_weights_dict = s.run(weights_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-4.826606 , -9.9340925,  3.8899462, -2.6317124,  5.45304  ,\n",
       "        -5.7924585,  4.752457 ,  1.5139014,  8.014108 , -3.4554937],\n",
       "       [-4.890322 , -9.340751 ,  3.6937258, -2.844387 ,  5.346814 ,\n",
       "        -6.0465345,  4.6167617,  1.7033981,  7.9195976, -3.6339433],\n",
       "       [-5.194474 , -9.558173 ,  4.1587205, -2.3802965,  5.5766435,\n",
       "        -6.065986 ,  4.8269773,  1.7820188,  8.294075 , -3.2630131],\n",
       "       [-4.4036365, -9.257714 ,  3.7970524, -2.2597632,  5.687299 ,\n",
       "        -5.9189053,  4.6535378,  1.601984 ,  7.9605904, -3.756865 ],\n",
       "       [-4.565281 , -9.504535 ,  3.5524337, -2.6782646,  5.3547196,\n",
       "        -5.421666 ,  5.0375605,  1.4634992,  8.065021 , -3.617325 ],\n",
       "       [-4.792373 , -9.602575 ,  3.3932242, -2.8439019,  5.5520005,\n",
       "        -6.3028493,  4.731669 ,  1.5132834,  8.200872 , -3.8719633],\n",
       "       [-5.025964 , -9.571217 ,  4.203959 , -2.7074006,  5.699357 ,\n",
       "        -6.0045753,  4.875746 ,  2.1217115,  8.213163 , -3.7344334],\n",
       "       [-4.4215446, -9.851782 ,  3.8279457, -2.2979133,  5.843791 ,\n",
       "        -6.123222 ,  4.453551 ,  2.0486345,  7.8676434, -3.5743713]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classify/conv2d/kernel:0 (3, 3, 3, 16)\n",
      "classify/conv2d/bias:0 (16,)\n",
      "classify/batch_normalization/gamma:0 (16,)\n",
      "classify/batch_normalization/beta:0 (16,)\n",
      "classify/batch_normalization/moving_mean:0 (16,)\n",
      "classify/batch_normalization/moving_variance:0 (16,)\n",
      "classify/conv2d_1/kernel:0 (3, 3, 16, 32)\n",
      "classify/conv2d_1/bias:0 (32,)\n",
      "classify/batch_normalization_1/gamma:0 (32,)\n",
      "classify/batch_normalization_1/beta:0 (32,)\n",
      "classify/batch_normalization_1/moving_mean:0 (32,)\n",
      "classify/batch_normalization_1/moving_variance:0 (32,)\n",
      "classify/conv2d_2/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_2/bias:0 (32,)\n",
      "classify/conv2d_3/kernel:0 (1, 1, 16, 32)\n",
      "classify/conv2d_3/bias:0 (32,)\n",
      "classify/batch_normalization_2/gamma:0 (32,)\n",
      "classify/batch_normalization_2/beta:0 (32,)\n",
      "classify/batch_normalization_2/moving_mean:0 (32,)\n",
      "classify/batch_normalization_2/moving_variance:0 (32,)\n",
      "classify/conv2d_4/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_4/bias:0 (32,)\n",
      "classify/batch_normalization_3/gamma:0 (32,)\n",
      "classify/batch_normalization_3/beta:0 (32,)\n",
      "classify/batch_normalization_3/moving_mean:0 (32,)\n",
      "classify/batch_normalization_3/moving_variance:0 (32,)\n",
      "classify/conv2d_5/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_5/bias:0 (32,)\n",
      "classify/batch_normalization_4/gamma:0 (32,)\n",
      "classify/batch_normalization_4/beta:0 (32,)\n",
      "classify/batch_normalization_4/moving_mean:0 (32,)\n",
      "classify/batch_normalization_4/moving_variance:0 (32,)\n",
      "classify/conv2d_6/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_6/bias:0 (32,)\n",
      "classify/batch_normalization_5/gamma:0 (32,)\n",
      "classify/batch_normalization_5/beta:0 (32,)\n",
      "classify/batch_normalization_5/moving_mean:0 (32,)\n",
      "classify/batch_normalization_5/moving_variance:0 (32,)\n",
      "classify/conv2d_7/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_7/bias:0 (32,)\n",
      "classify/batch_normalization_6/gamma:0 (32,)\n",
      "classify/batch_normalization_6/beta:0 (32,)\n",
      "classify/batch_normalization_6/moving_mean:0 (32,)\n",
      "classify/batch_normalization_6/moving_variance:0 (32,)\n",
      "classify/conv2d_8/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_8/bias:0 (32,)\n",
      "classify/batch_normalization_7/gamma:0 (32,)\n",
      "classify/batch_normalization_7/beta:0 (32,)\n",
      "classify/batch_normalization_7/moving_mean:0 (32,)\n",
      "classify/batch_normalization_7/moving_variance:0 (32,)\n",
      "classify/conv2d_9/kernel:0 (3, 3, 32, 32)\n",
      "classify/conv2d_9/bias:0 (32,)\n",
      "classify/batch_normalization_8/gamma:0 (32,)\n",
      "classify/batch_normalization_8/beta:0 (32,)\n",
      "classify/batch_normalization_8/moving_mean:0 (32,)\n",
      "classify/batch_normalization_8/moving_variance:0 (32,)\n",
      "classify/conv2d_10/kernel:0 (3, 3, 32, 64)\n",
      "classify/conv2d_10/bias:0 (64,)\n",
      "classify/batch_normalization_9/gamma:0 (64,)\n",
      "classify/batch_normalization_9/beta:0 (64,)\n",
      "classify/batch_normalization_9/moving_mean:0 (64,)\n",
      "classify/batch_normalization_9/moving_variance:0 (64,)\n",
      "classify/conv2d_11/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_11/bias:0 (64,)\n",
      "classify/conv2d_12/kernel:0 (1, 1, 32, 64)\n",
      "classify/conv2d_12/bias:0 (64,)\n",
      "classify/batch_normalization_10/gamma:0 (64,)\n",
      "classify/batch_normalization_10/beta:0 (64,)\n",
      "classify/batch_normalization_10/moving_mean:0 (64,)\n",
      "classify/batch_normalization_10/moving_variance:0 (64,)\n",
      "classify/conv2d_13/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_13/bias:0 (64,)\n",
      "classify/batch_normalization_11/gamma:0 (64,)\n",
      "classify/batch_normalization_11/beta:0 (64,)\n",
      "classify/batch_normalization_11/moving_mean:0 (64,)\n",
      "classify/batch_normalization_11/moving_variance:0 (64,)\n",
      "classify/conv2d_14/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_14/bias:0 (64,)\n",
      "classify/batch_normalization_12/gamma:0 (64,)\n",
      "classify/batch_normalization_12/beta:0 (64,)\n",
      "classify/batch_normalization_12/moving_mean:0 (64,)\n",
      "classify/batch_normalization_12/moving_variance:0 (64,)\n",
      "classify/conv2d_15/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_15/bias:0 (64,)\n",
      "classify/batch_normalization_13/gamma:0 (64,)\n",
      "classify/batch_normalization_13/beta:0 (64,)\n",
      "classify/batch_normalization_13/moving_mean:0 (64,)\n",
      "classify/batch_normalization_13/moving_variance:0 (64,)\n",
      "classify/conv2d_16/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_16/bias:0 (64,)\n",
      "classify/batch_normalization_14/gamma:0 (64,)\n",
      "classify/batch_normalization_14/beta:0 (64,)\n",
      "classify/batch_normalization_14/moving_mean:0 (64,)\n",
      "classify/batch_normalization_14/moving_variance:0 (64,)\n",
      "classify/conv2d_17/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_17/bias:0 (64,)\n",
      "classify/batch_normalization_15/gamma:0 (64,)\n",
      "classify/batch_normalization_15/beta:0 (64,)\n",
      "classify/batch_normalization_15/moving_mean:0 (64,)\n",
      "classify/batch_normalization_15/moving_variance:0 (64,)\n",
      "classify/conv2d_18/kernel:0 (3, 3, 64, 64)\n",
      "classify/conv2d_18/bias:0 (64,)\n",
      "classify/batch_normalization_16/gamma:0 (64,)\n",
      "classify/batch_normalization_16/beta:0 (64,)\n",
      "classify/batch_normalization_16/moving_mean:0 (64,)\n",
      "classify/batch_normalization_16/moving_variance:0 (64,)\n",
      "classify/conv2d_19/kernel:0 (3, 3, 64, 128)\n",
      "classify/conv2d_19/bias:0 (128,)\n",
      "classify/batch_normalization_17/gamma:0 (128,)\n",
      "classify/batch_normalization_17/beta:0 (128,)\n",
      "classify/batch_normalization_17/moving_mean:0 (128,)\n",
      "classify/batch_normalization_17/moving_variance:0 (128,)\n",
      "classify/conv2d_20/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_20/bias:0 (128,)\n",
      "classify/conv2d_21/kernel:0 (1, 1, 64, 128)\n",
      "classify/conv2d_21/bias:0 (128,)\n",
      "classify/batch_normalization_18/gamma:0 (128,)\n",
      "classify/batch_normalization_18/beta:0 (128,)\n",
      "classify/batch_normalization_18/moving_mean:0 (128,)\n",
      "classify/batch_normalization_18/moving_variance:0 (128,)\n",
      "classify/conv2d_22/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_22/bias:0 (128,)\n",
      "classify/batch_normalization_19/gamma:0 (128,)\n",
      "classify/batch_normalization_19/beta:0 (128,)\n",
      "classify/batch_normalization_19/moving_mean:0 (128,)\n",
      "classify/batch_normalization_19/moving_variance:0 (128,)\n",
      "classify/conv2d_23/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_23/bias:0 (128,)\n",
      "classify/batch_normalization_20/gamma:0 (128,)\n",
      "classify/batch_normalization_20/beta:0 (128,)\n",
      "classify/batch_normalization_20/moving_mean:0 (128,)\n",
      "classify/batch_normalization_20/moving_variance:0 (128,)\n",
      "classify/conv2d_24/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_24/bias:0 (128,)\n",
      "classify/batch_normalization_21/gamma:0 (128,)\n",
      "classify/batch_normalization_21/beta:0 (128,)\n",
      "classify/batch_normalization_21/moving_mean:0 (128,)\n",
      "classify/batch_normalization_21/moving_variance:0 (128,)\n",
      "classify/conv2d_25/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_25/bias:0 (128,)\n",
      "classify/batch_normalization_22/gamma:0 (128,)\n",
      "classify/batch_normalization_22/beta:0 (128,)\n",
      "classify/batch_normalization_22/moving_mean:0 (128,)\n",
      "classify/batch_normalization_22/moving_variance:0 (128,)\n",
      "classify/conv2d_26/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_26/bias:0 (128,)\n",
      "classify/batch_normalization_23/gamma:0 (128,)\n",
      "classify/batch_normalization_23/beta:0 (128,)\n",
      "classify/batch_normalization_23/moving_mean:0 (128,)\n",
      "classify/batch_normalization_23/moving_variance:0 (128,)\n",
      "classify/conv2d_27/kernel:0 (3, 3, 128, 128)\n",
      "classify/conv2d_27/bias:0 (128,)\n",
      "classify/batch_normalization_24/gamma:0 (128,)\n",
      "classify/batch_normalization_24/beta:0 (128,)\n",
      "classify/batch_normalization_24/moving_mean:0 (128,)\n",
      "classify/batch_normalization_24/moving_variance:0 (128,)\n",
      "classify/dense/kernel:0 (128, 10)\n",
      "classify/dense/bias:0 (10,)\n"
     ]
    }
   ],
   "source": [
    "for k, v in np_weights_dict.items():\n",
    "    print(k, v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from wrn import WideResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = WideResNet(num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv.weight torch.Size([16, 3, 3, 3])\n",
      "conv.bias torch.Size([16])\n",
      "res_blocks.0.0.bn1.weight torch.Size([16])\n",
      "res_blocks.0.0.bn1.bias torch.Size([16])\n",
      "res_blocks.0.0.conv1.weight torch.Size([32, 16, 3, 3])\n",
      "res_blocks.0.0.conv1.bias torch.Size([32])\n",
      "res_blocks.0.0.bn2.weight torch.Size([32])\n",
      "res_blocks.0.0.bn2.bias torch.Size([32])\n",
      "res_blocks.0.0.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.0.conv2.bias torch.Size([32])\n",
      "res_blocks.0.0.skip.weight torch.Size([32, 16, 1, 1])\n",
      "res_blocks.0.0.skip.bias torch.Size([32])\n",
      "res_blocks.0.1.bn1.weight torch.Size([32])\n",
      "res_blocks.0.1.bn1.bias torch.Size([32])\n",
      "res_blocks.0.1.conv1.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.1.conv1.bias torch.Size([32])\n",
      "res_blocks.0.1.bn2.weight torch.Size([32])\n",
      "res_blocks.0.1.bn2.bias torch.Size([32])\n",
      "res_blocks.0.1.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.1.conv2.bias torch.Size([32])\n",
      "res_blocks.0.2.bn1.weight torch.Size([32])\n",
      "res_blocks.0.2.bn1.bias torch.Size([32])\n",
      "res_blocks.0.2.conv1.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.2.conv1.bias torch.Size([32])\n",
      "res_blocks.0.2.bn2.weight torch.Size([32])\n",
      "res_blocks.0.2.bn2.bias torch.Size([32])\n",
      "res_blocks.0.2.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.2.conv2.bias torch.Size([32])\n",
      "res_blocks.0.3.bn1.weight torch.Size([32])\n",
      "res_blocks.0.3.bn1.bias torch.Size([32])\n",
      "res_blocks.0.3.conv1.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.3.conv1.bias torch.Size([32])\n",
      "res_blocks.0.3.bn2.weight torch.Size([32])\n",
      "res_blocks.0.3.bn2.bias torch.Size([32])\n",
      "res_blocks.0.3.conv2.weight torch.Size([32, 32, 3, 3])\n",
      "res_blocks.0.3.conv2.bias torch.Size([32])\n",
      "res_blocks.1.0.bn1.weight torch.Size([32])\n",
      "res_blocks.1.0.bn1.bias torch.Size([32])\n",
      "res_blocks.1.0.conv1.weight torch.Size([64, 32, 3, 3])\n",
      "res_blocks.1.0.conv1.bias torch.Size([64])\n",
      "res_blocks.1.0.bn2.weight torch.Size([64])\n",
      "res_blocks.1.0.bn2.bias torch.Size([64])\n",
      "res_blocks.1.0.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.0.conv2.bias torch.Size([64])\n",
      "res_blocks.1.0.skip.weight torch.Size([64, 32, 1, 1])\n",
      "res_blocks.1.0.skip.bias torch.Size([64])\n",
      "res_blocks.1.1.bn1.weight torch.Size([64])\n",
      "res_blocks.1.1.bn1.bias torch.Size([64])\n",
      "res_blocks.1.1.conv1.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.1.conv1.bias torch.Size([64])\n",
      "res_blocks.1.1.bn2.weight torch.Size([64])\n",
      "res_blocks.1.1.bn2.bias torch.Size([64])\n",
      "res_blocks.1.1.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.1.conv2.bias torch.Size([64])\n",
      "res_blocks.1.2.bn1.weight torch.Size([64])\n",
      "res_blocks.1.2.bn1.bias torch.Size([64])\n",
      "res_blocks.1.2.conv1.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.2.conv1.bias torch.Size([64])\n",
      "res_blocks.1.2.bn2.weight torch.Size([64])\n",
      "res_blocks.1.2.bn2.bias torch.Size([64])\n",
      "res_blocks.1.2.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.2.conv2.bias torch.Size([64])\n",
      "res_blocks.1.3.bn1.weight torch.Size([64])\n",
      "res_blocks.1.3.bn1.bias torch.Size([64])\n",
      "res_blocks.1.3.conv1.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.3.conv1.bias torch.Size([64])\n",
      "res_blocks.1.3.bn2.weight torch.Size([64])\n",
      "res_blocks.1.3.bn2.bias torch.Size([64])\n",
      "res_blocks.1.3.conv2.weight torch.Size([64, 64, 3, 3])\n",
      "res_blocks.1.3.conv2.bias torch.Size([64])\n",
      "res_blocks.2.0.bn1.weight torch.Size([64])\n",
      "res_blocks.2.0.bn1.bias torch.Size([64])\n",
      "res_blocks.2.0.conv1.weight torch.Size([128, 64, 3, 3])\n",
      "res_blocks.2.0.conv1.bias torch.Size([128])\n",
      "res_blocks.2.0.bn2.weight torch.Size([128])\n",
      "res_blocks.2.0.bn2.bias torch.Size([128])\n",
      "res_blocks.2.0.conv2.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.0.conv2.bias torch.Size([128])\n",
      "res_blocks.2.0.skip.weight torch.Size([128, 64, 1, 1])\n",
      "res_blocks.2.0.skip.bias torch.Size([128])\n",
      "res_blocks.2.1.bn1.weight torch.Size([128])\n",
      "res_blocks.2.1.bn1.bias torch.Size([128])\n",
      "res_blocks.2.1.conv1.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.1.conv1.bias torch.Size([128])\n",
      "res_blocks.2.1.bn2.weight torch.Size([128])\n",
      "res_blocks.2.1.bn2.bias torch.Size([128])\n",
      "res_blocks.2.1.conv2.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.1.conv2.bias torch.Size([128])\n",
      "res_blocks.2.2.bn1.weight torch.Size([128])\n",
      "res_blocks.2.2.bn1.bias torch.Size([128])\n",
      "res_blocks.2.2.conv1.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.2.conv1.bias torch.Size([128])\n",
      "res_blocks.2.2.bn2.weight torch.Size([128])\n",
      "res_blocks.2.2.bn2.bias torch.Size([128])\n",
      "res_blocks.2.2.conv2.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.2.conv2.bias torch.Size([128])\n",
      "res_blocks.2.3.bn1.weight torch.Size([128])\n",
      "res_blocks.2.3.bn1.bias torch.Size([128])\n",
      "res_blocks.2.3.conv1.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.3.conv1.bias torch.Size([128])\n",
      "res_blocks.2.3.bn2.weight torch.Size([128])\n",
      "res_blocks.2.3.bn2.bias torch.Size([128])\n",
      "res_blocks.2.3.conv2.weight torch.Size([128, 128, 3, 3])\n",
      "res_blocks.2.3.conv2.bias torch.Size([128])\n",
      "bn.weight torch.Size([128])\n",
      "bn.bias torch.Size([128])\n",
      "fc.weight torch.Size([10, 128])\n",
      "fc.bias torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "pt_weights_dict = {}\n",
    "for n, p in model.named_parameters():    \n",
    "    print(n, p.shape)\n",
    "    pt_weights_dict[n] = p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pt_weights_names = list(pt_weights_dict.keys())\n",
    "\n",
    "c = 0\n",
    "for tf_name in np_weights_dict:\n",
    "    \n",
    "    if \"moving_\" in tf_name:\n",
    "        continue\n",
    "\n",
    "    pt_name = pt_weights_names[c]\n",
    "    c += 1\n",
    "        \n",
    "    w = np_weights_dict[tf_name]\n",
    "    if w.ndim == 4:\n",
    "        w = w.transpose((-1, -2, 0, 1))\n",
    "    elif w.ndim == 2:\n",
    "        w = w.transpose((1, 0))\n",
    "\n",
    "    pt_weights_dict[pt_name].data = torch.from_numpy(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check forward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 32, 32])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.from_numpy(np_x.transpose(0, 3, 1, 2))\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 16, 32, 32])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    y = model.conv(x)\n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(y.detach().numpy() - res[2]['c1'].transpose(0, 3, 1, 2)).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C1 + res_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 32, 32, 32])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    y = model.conv(x)\n",
    "    y = model.res_blocks[0][0](y)\n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0002001329"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(y.detach().numpy() - res[2]['res_0'].transpose(0, 3, 1, 2)).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C1 + res_0 + res_0_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 32, 32, 32])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    y = model.conv(x)\n",
    "    y = model.res_blocks[0][0](y)\n",
    "    y = model.res_blocks[0][1](y)    \n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.000370559"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(y.detach().numpy() - res[2]['res_0_0'].transpose(0, 3, 1, 2)).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C1 + res_0 + res_0_0 + res_0_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 32, 32, 32])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    y = model.conv(x)\n",
    "    y = model.res_blocks[0][0](y)\n",
    "    y = model.res_blocks[0][1](y)    \n",
    "    y = model.res_blocks[0][2](y)\n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0008550567"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(y.detach().numpy() - res[2]['res_0_1'].transpose(0, 3, 1, 2)).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Complete model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 10])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    y = model(x)\n",
    "\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-5.1307, -9.5296,  4.1101, -2.7820,  6.1873, -5.7317,  4.8811,  1.3497,\n",
       "         8.0847, -3.2070])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-4.826606 , -9.9340925,  3.8899462, -2.6317124,  5.45304  ,\n",
       "       -5.7924585,  4.752457 ,  1.5139014,  8.014108 , -3.4554937],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[0][0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2863981"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(y.detach().numpy() - res[0]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "base_model = nn.Linear(1, 1)\n",
    "nn.init.constant_(base_model.weight, 1.0)\n",
    "\n",
    "ema_model = nn.Linear(1, 1)\n",
    "ema_model.load_state_dict(base_model.state_dict())\n",
    "\n",
    "for param in ema_model.parameters():\n",
    "    param.detach_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Parameter containing:\n",
       " tensor([[1.]]),\n",
       " Parameter containing:\n",
       " tensor([[1.]], requires_grad=True))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ema_model.weight, base_model.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "ema_decay = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model.weight.data.sub_(0.01)\n",
    "\n",
    "for ema_param, param in zip(ema_model.parameters(), base_model.parameters()):\n",
    "    ema_param.data.mul_(ema_decay).add_(param.data, alpha=1.0 - ema_decay)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Parameter containing:\n",
       " tensor([[0.9822]]),\n",
       " Parameter containing:\n",
       " tensor([[0.9400]], requires_grad=True))"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ema_model.weight, base_model.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EMA model with shared buffers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ema_buf 140581377615744\n",
      "buf 140581377615744\n",
      "ema_buf 140581380140720\n",
      "buf 140581380140720\n",
      "ema_buf 140581380139120\n",
      "buf 140581380113344\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "base_model = nn.Sequential(\n",
    "    nn.Conv2d(3, 16, 3),\n",
    "    nn.BatchNorm2d(16),\n",
    "    nn.ReLU(True)\n",
    ")\n",
    "\n",
    "opt = optim.SGD(base_model.parameters(), lr=0.01)\n",
    "\n",
    "ema_model = nn.Sequential(\n",
    "    nn.Conv2d(3, 16, 3),\n",
    "    nn.BatchNorm2d(16),\n",
    "    nn.ReLU(True)\n",
    ")\n",
    "ema_model.load_state_dict(base_model.state_dict())\n",
    "\n",
    "for param in ema_model.parameters():\n",
    "    param.detach_()\n",
    "\n",
    "for m1, m2 in zip(base_model.modules(), ema_model.modules()):\n",
    "    if isinstance(m1, nn.BatchNorm2d) and isinstance(m2, nn.BatchNorm2d):\n",
    "        m2.running_mean = m1.running_mean\n",
    "        m2.running_var = m1.running_var\n",
    "\n",
    "for ema_buf, buf in zip(ema_model.buffers(), base_model.buffers()):\n",
    "    print(\"ema_buf\", id(ema_buf))\n",
    "    print(\"buf\", id(buf))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- ema\n",
      "0.weight torch.Size([16, 3, 3, 3]) tensor(2.3749)\n",
      "0.bias torch.Size([16]) tensor(0.4341)\n",
      "1.weight torch.Size([16]) tensor(4.)\n",
      "1.bias torch.Size([16]) tensor(0.)\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])\n",
      "-- base\n",
      "0.weight torch.Size([16, 3, 3, 3]) tensor(2.3749, grad_fn=<NormBackward0>)\n",
      "0.bias torch.Size([16]) tensor(0.4341, grad_fn=<NormBackward0>)\n",
      "1.weight torch.Size([16]) tensor(4., grad_fn=<NormBackward0>)\n",
      "1.bias torch.Size([16]) tensor(0., grad_fn=<NormBackward0>)\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])\n"
     ]
    }
   ],
   "source": [
    "print(\"-- ema\")\n",
    "for n, ema_p in ema_model.named_parameters():\n",
    "    print(n, ema_p.shape, ema_p.norm())\n",
    "print(ema_model[1].running_mean, ema_model[1].running_var)    \n",
    "    \n",
    "print(\"-- base\")\n",
    "for n, p in base_model.named_parameters():\n",
    "    print(n, p.shape, p.norm())\n",
    "print(base_model[1].running_mean, base_model[1].running_var)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "ema_decay = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt.zero_grad()\n",
    "x = torch.rand(4, 3, 12, 12)\n",
    "base_model(x).sum().backward()\n",
    "opt.step()\n",
    "\n",
    "\n",
    "for ema_param, param in zip(ema_model.parameters(), base_model.parameters()):\n",
    "    ema_param.data.mul_(ema_decay).add_(param.data, alpha=1.0 - ema_decay)\n",
    "\n",
    "# for ema_buf, buf in zip(ema_model.buffers(), base_model.buffers()):\n",
    "#     ema_buf.data = buf.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- ema\n",
      "0.weight torch.Size([16, 3, 3, 3]) tensor(2.4063)\n",
      "0.bias torch.Size([16]) tensor(0.4341)\n",
      "1.weight torch.Size([16]) tensor(2.7863)\n",
      "1.bias torch.Size([16]) tensor(1.5250)\n",
      "tensor([ 0.0406, -0.0479, -0.0161, -0.1272, -0.0419, -0.0548,  0.0111,  0.0539,\n",
      "        -0.0344,  0.0687, -0.0264, -0.0902,  0.0322, -0.0502,  0.0072, -0.0373]) tensor([0.8193, 0.8177, 0.8190, 0.8172, 0.8176, 0.8185, 0.8170, 0.8172, 0.8184,\n",
      "        0.8195, 0.8177, 0.8181, 0.8187, 0.8179, 0.8177, 0.8200])\n",
      "-- base\n",
      "0.weight torch.Size([16, 3, 3, 3]) tensor(3.1440, grad_fn=<NormBackward0>)\n",
      "0.bias torch.Size([16]) tensor(0.4341, grad_fn=<NormBackward0>)\n",
      "1.weight torch.Size([16]) tensor(2.3727, grad_fn=<NormBackward0>)\n",
      "1.bias torch.Size([16]) tensor(8.0311, grad_fn=<NormBackward0>)\n",
      "tensor([ 0.0406, -0.0479, -0.0161, -0.1272, -0.0419, -0.0548,  0.0111,  0.0539,\n",
      "        -0.0344,  0.0687, -0.0264, -0.0902,  0.0322, -0.0502,  0.0072, -0.0373]) tensor([0.8193, 0.8177, 0.8190, 0.8172, 0.8176, 0.8185, 0.8170, 0.8172, 0.8184,\n",
      "        0.8195, 0.8177, 0.8181, 0.8187, 0.8179, 0.8177, 0.8200])\n"
     ]
    }
   ],
   "source": [
    "print(\"-- ema\")\n",
    "for n, ema_p in ema_model.named_parameters():\n",
    "    print(n, ema_p.shape, ema_p.norm())\n",
    "print(ema_model[1].running_mean, ema_model[1].running_var)    \n",
    "    \n",
    "print(\"-- base\")\n",
    "for n, p in base_model.named_parameters():\n",
    "    print(n, p.shape, p.norm())\n",
    "print(base_model[1].running_mean, base_model[1].running_var)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}