More TensorFlow Eager Execution

Author: TarrySingh

deep-learning/tensor-flow-exercises/TensorFlow 1.5 Developer Version with Eager Execution - CPU Mac OSX.ipynb

@@ -848,10 +848,124 @@
     "**NOTE to self**: (Planned test on CPU/GPU for 25th Nov "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "class Model(object):\n",
+    "    def __init__(self):\n",
+    "        self.layer = tf.layers.Dense(1)\n",
+    "        \n",
+    "    def predict(self, inputs):\n",
+    "        return self.layer(inputs)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
-   "source": []
+   "source": [
+    "#### Note: What does tf.layers API do\n",
+    "\n",
+    "Next, you'll see how tf.layers API makes it easy to define sophisticated models. Well, we're used to the beauty of Keras already, lets see how it plays out in TensorFlow 1.4.x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
+      "Extracting ./mnist_data/train-images-idx3-ubyte.gz\n",
+      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
+      "Extracting ./mnist_data/train-labels-idx1-ubyte.gz\n",
+      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
+      "Extracting ./mnist_data/t10k-images-idx3-ubyte.gz\n",
+      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
+      "Extracting ./mnist_data/t10k-labels-idx1-ubyte.gz\n",
+      "WARNING:tensorflow:From <ipython-input-43-9ff01676a66a>:35: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "\n",
+      "Future major versions of TensorFlow will allow gradients to flow\n",
+      "into the labels input on backprop by default.\n",
+      "\n",
+      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
+      "\n",
+      "Step 0: Loss on training set : 2.256627\n",
+      "Step 100: Loss on training set : 0.399705\n",
+      "Step 200: Loss on training set : 0.214346\n",
+      "Step 300: Loss on training set : 0.211841\n",
+      "Step 400: Loss on training set : 0.119034\n",
+      "Step 500: Loss on training set : 0.085323\n",
+      "Step 600: Loss on training set : 0.231022\n",
+      "Step 700: Loss on training set : 0.061058\n",
+      "Step 800: Loss on training set : 0.196906\n"
+     ]
+    }
+   ],
+   "source": [
+    "class MNISTModel(object):\n",
+    "  def __init__(self, data_format):\n",
+    "    # 'channels_first' is typically faster on GPUs\n",
+    "    # while 'channels_last' is typically faster on CPUs.\n",
+    "    # See: https://www.tensorflow.org/performance/performance_guide#data_formats\n",
+    "    if data_format == 'channels_first':\n",
+    "      self._input_shape = [-1, 1, 28, 28]\n",
+    "    else:\n",
+    "      self._input_shape = [-1, 28, 28, 1]\n",
+    "    self.conv1 = tf.layers.Conv2D(32, 5,\n",
+    "                                  padding='same',\n",
+    "                                  activation=tf.nn.relu,\n",
+    "                                  data_format=data_format)\n",
+    "    self.max_pool2d = tf.layers.MaxPooling2D(\n",
+    "        (2, 2), (2, 2), padding='same', data_format=data_format)\n",
+    "    self.conv2 = tf.layers.Conv2D(64, 5,\n",
+    "                                  padding='same',\n",
+    "                                  activation=tf.nn.relu,\n",
+    "                                  data_format=data_format)\n",
+    "    self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu)\n",
+    "    self.dropout = tf.layers.Dropout(0.5)\n",
+    "    self.dense2 = tf.layers.Dense(10)\n",
+    "\n",
+    "  def predict(self, inputs):\n",
+    "    x = tf.reshape(inputs, self._input_shape)\n",
+    "    x = self.max_pool2d(self.conv1(x))\n",
+    "    x = self.max_pool2d(self.conv2(x))\n",
+    "    x = tf.layers.flatten(x)\n",
+    "    x = self.dropout(self.dense1(x))\n",
+    "    return self.dense2(x)\n",
+    "\n",
+    "def loss(model, inputs, targets):\n",
+    "  return tf.reduce_mean(\n",
+    "      tf.nn.softmax_cross_entropy_with_logits(\n",
+    "          logits=model.predict(inputs), labels=targets))\n",
+    "\n",
+    "\n",
+    "# Load the training and validation data\n",
+    "from tensorflow.examples.tutorials.mnist import input_data\n",
+    "data = input_data.read_data_sets(\"./mnist_data\", one_hot=True)\n",
+    "\n",
+    "# Train\n",
+    "device = \"gpu:0\" if tfe.num_gpus() else \"cpu:0\"\n",
+    "model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last')\n",
+    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)\n",
+    "grad = tfe.implicit_gradients(loss)\n",
+    "for i in range(20001):\n",
+    "  with tf.device(device):\n",
+    "    (inputs, targets) = data.train.next_batch(50)\n",
+    "    optimizer.apply_gradients(grad(model, inputs, targets))\n",
+    "    if i % 100 == 0:\n",
+    "      print(\"Step %d: Loss on training set : %f\" %\n",
+    "            (i, loss(model, inputs, targets).numpy()))\n",
+    "print(\"Loss on test set: %f\" % loss(model, data.test.images, data.test.labels).numpy())"
+   ]
   },
   {
    "cell_type": "code",
@@ -860,7 +974,9 @@
     "collapsed": true
    },
    "outputs": [],
-   "source": []
+   "source": [
+    "# Running with tf.nn.softmax_cross_entropy_with_logits_v2\n"
+   ]
   }
  ],
  "metadata": {