More Eager Execution stuff on TensorFlow 1.5

Author: TarrySingh

deep-learning/tensor-flow-exercises/TensorFlow 1.5 Developer Version with Eager Execution - CPU Mac OSX.ipynb

@@ -906,7 +906,37 @@
       "Step 500: Loss on training set : 0.085323\n",
       "Step 600: Loss on training set : 0.231022\n",
       "Step 700: Loss on training set : 0.061058\n",
-      "Step 800: Loss on training set : 0.196906\n"
+      "Step 800: Loss on training set : 0.196906\n",
+      "Step 900: Loss on training set : 0.156831\n",
+      "Step 1000: Loss on training set : 0.091805\n",
+      "Step 1100: Loss on training set : 0.024188\n",
+      "Step 1200: Loss on training set : 0.092624\n",
+      "Step 1300: Loss on training set : 0.019488\n",
+      "Step 1400: Loss on training set : 0.064160\n",
+      "Step 1500: Loss on training set : 0.044069\n",
+      "Step 1600: Loss on training set : 0.088605\n",
+      "Step 1700: Loss on training set : 0.004956\n",
+      "Step 1800: Loss on training set : 0.044108\n",
+      "Step 1900: Loss on training set : 0.050574\n",
+      "Step 2000: Loss on training set : 0.013534\n",
+      "Step 2100: Loss on training set : 0.068764\n",
+      "Step 2200: Loss on training set : 0.061247\n",
+      "Step 2300: Loss on training set : 0.134102\n",
+      "Step 2400: Loss on training set : 0.002189\n",
+      "Step 2500: Loss on training set : 0.002621\n",
+      "Step 2600: Loss on training set : 0.084751\n",
+      "Step 2700: Loss on training set : 0.073403\n",
+      "Step 2800: Loss on training set : 0.034124\n",
+      "Step 2900: Loss on training set : 0.068016\n",
+      "Step 3000: Loss on training set : 0.026844\n",
+      "Step 3100: Loss on training set : 0.008452\n",
+      "Step 3200: Loss on training set : 0.052670\n",
+      "Step 3300: Loss on training set : 0.095155\n",
+      "Step 3400: Loss on training set : 0.019506\n",
+      "Step 3500: Loss on training set : 0.015484\n",
+      "Step 3600: Loss on training set : 0.007086\n",
+      "Step 3700: Loss on training set : 0.045831\n",
+      "Step 3800: Loss on training set : 0.058367\n"
      ]
     }
    ],
@@ -975,7 +1005,83 @@
    },
    "outputs": [],
    "source": [
-    "# Running with tf.nn.softmax_cross_entropy_with_logits_v2\n"
+    "help(tf.nn.softmax_cross_entropy_with_logits_v2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Running with tf.nn.softmax_cross_entropy_with_logits_v2\n",
+    "class MNISTModel(object):\n",
+    "      def __init__(self, data_format):\n",
+    "        # 'channels_first' is typically faster on GPUs\n",
+    "        # while 'channels_last' is typically faster on CPUs.\n",
+    "        # See: https://www.tensorflow.org/performance/performance_guide#data_formats\n",
+    "        if data_format == 'channels_first':\n",
+    "            self._input_shape = [-1, 1, 28, 28]\n",
+    "        else:\n",
+    "            self._input_shape = [-1, 28, 28, 1]\n",
+    "    self.conv1 = tf.layers.Conv2D(32, 5,\n",
+    "                                  padding='same',\n",
+    "                                  activation=tf.nn.relu,\n",
+    "                                  data_format=data_format)\n",
+    "    self.max_pool2d = tf.layers.MaxPooling2D(\n",
+    "        (2, 2), (2, 2), padding='same', data_format=data_format)\n",
+    "    self.conv2 = tf.layers.Conv2D(64, 5,\n",
+    "                                  padding='same',\n",
+    "                                  activation=tf.nn.relu,\n",
+    "                                  data_format=data_format)\n",
+    "    self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu)\n",
+    "    self.dropout = tf.layers.Dropout(0.5)\n",
+    "    self.dense2 = tf.layers.Dense(10)\n",
+    "\n",
+    "  def predict(self, inputs):\n",
+    "    x = tf.reshape(inputs, self._input_shape)\n",
+    "    x = self.max_pool2d(self.conv1(x))\n",
+    "    x = self.max_pool2d(self.conv2(x))\n",
+    "    x = tf.layers.flatten(x)\n",
+    "    x = self.dropout(self.dense1(x))\n",
+    "    return self.dense2(x)\n",
+    "\n",
+    "def loss(model, inputs, targets):\n",
+    "  return tf.reduce_mean(\n",
+    "      tf.nn.softmax_cross_entropy_with_logits(\n",
+    "          logits=model.predict(inputs), labels=targets))\n",
+    "\n",
+    "\n",
+    "# Load the training and validation data\n",
+    "from tensorflow.examples.tutorials.mnist import input_data\n",
+    "data = input_data.read_data_sets(\"./mnist_data\", one_hot=True)\n",
+    "\n",
+    "# Train\n",
+    "device = \"gpu:0\" if tfe.num_gpus() else \"cpu:0\"\n",
+    "model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last')\n",
+    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)\n",
+    "grad = tfe.implicit_gradients(loss)\n",
+    "for i in range(20001):\n",
+    "  with tf.device(device):\n",
+    "    (inputs, targets) = data.train.next_batch(50)\n",
+    "    optimizer.apply_gradients(grad(model, inputs, targets))\n",
+    "    if i % 100 == 0:\n",
+    "      print(\"Step %d: Loss on training set : %f\" %\n",
+    "            (i, loss(model, inputs, targets).numpy()))\n",
+    "print(\"Loss on test set: %f\" % loss(model, data.test.images, data.test.labels).numpy())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Checkpointing trained variables\n",
+    "\n",
+    "TensorFlow Eager Execution Variables `tfe.Variables` provide a way to represent shared, persistent state of the model you make. The `tfe.Saver` class -- which is a thin wrapper over the `tf.train.Saver` class provides means to save and restore variables to and from checkpoints.\n",
+    "\n",
+    "As an examples:"
    ]
   }
  ],