deep-learning/tensor-flow-exercises/TensorFlow+1.4+--+Eager+Execution+With+TensorFlow+1.5+Dev+Release+-+GPU+Ubuntu+16.04LTS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 0 : Upgrading to TensorFlow 1.4\n",
    "\n",
    "### check your system requirements as per TensorFlow Documentation\n",
    "\n",
    "For me this was good enough...\n",
    "\n",
    "|Version:|\tCPU/GPU:|\tPython Version:|\tCompiler:|\tBuild Tools:|\tcuDNN:|\tCUDA:|\n",
    "--------|-----------|------------------|-------------|--------------|---------|------|\n",
    "|tensorflow-1.4.0|\tCPU|\t2.7, 3.3-3.6|\tGCC 4.8|\tBazel 0.5.4|\tN/A|\tN/A|\n",
    "|tensorflow_gpu-1.4.0|\tGPU|\t2.7, 3.3-3.6|\tGCC 4.8|\tBazel 0.5.4|\t6|\t8|\n",
    "|tensorflow-1.3.0|\tCPU|\t2.7, 3.3-3.6|\tGCC 4.8|\tBazel 0.4.5|\tN/A\t|N/A|\n",
    "|tensorflow_gpu-1.3.0|\tGPU|\t2.7, 3.3-3.6|\tGCC 4.8|\tBazel 0.4.5\t|6|\t8|\n",
    "\n",
    "#### Check info for Bazel version\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ bazel version\n",
    "Build label: 0.5.4\n",
    "Build target: bazel-out/local-fastbuild/bin/src/main/java/com/google/devtools/build/lib/bazel/BazelServer_deploy.jar\n",
    "Build time: Fri Aug 25 10:00:00 2017 (1503655200)\n",
    "Build timestamp: 1503655200\n",
    "Build timestamp as int: 1503655200\n",
    "```\n",
    "\n",
    "#### Clone Tensorflow Git and switch to branch 1.4\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads$ mkdir temp\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads$ cd temp/\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp$ git clone https://github.com/tensorflow/tensorflow\n",
    "Cloning into 'tensorflow'...\n",
    "remote: Counting objects: 267022, done.\n",
    "remote: Compressing objects: 100% (4/4), done.\n",
    "remote: Total 267022 (delta 0), reused 0 (delta 0), pack-reused 267018\n",
    "Receiving objects: 100% (267022/267022), 134.13 MiB | 24.28 MiB/s, done.\n",
    "Resolving deltas: 100% (208394/208394), done.\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp$ cd tensorflow/\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ git checkout r\n",
    "r0.10                     r0.8                      r1.2                      revert-14730-master \n",
    "r0.11                     r0.9                      r1.3                      revert-14765-patch-1 \n",
    "r0.12                     r1.0                      r1.4                      \n",
    "r0.7                      r1.1                      revert-14513-add_conv1d   \n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ git checkout r\n",
    "r0.10                     r0.8                      r1.2                      revert-14730-master \n",
    "r0.11                     r0.9                      r1.3                      revert-14765-patch-1 \n",
    "r0.12                     r1.0                      r1.4                      \n",
    "r0.7                      r1.1                      revert-14513-add_conv1d   \n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ git checkout r1.\n",
    "r1.0   r1.1   r1.2   r1.3   r1.4   \n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ git checkout r1.4\n",
    "Branch r1.4 set up to track remote branch r1.4 from origin.\n",
    "Switched to a new branch 'r1.4'\n",
    "\n",
    "```\n",
    "\n",
    "#### Stay in the same directory and configure\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ ./configure \n",
    "You have bazel 0.5.4 installed.\n",
    "Please specify the location of python. [Default is /home/deeplearn/anaconda3/bin/python]: \n",
    "\n",
    "\n",
    "Found possible Python library paths:\n",
    "  /home/deeplearn/anaconda3/lib/python3.6/site-packages\n",
    "Please input the desired Python library path to use.  Default is [/home/deeplearn/anaconda3/lib/python3.6/site-packages]\n",
    "\n",
    "Do you wish to build TensorFlow with jemalloc as malloc support? [Y/n]: \n",
    "jemalloc as malloc support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with Google Cloud Platform support? [Y/n]: n\n",
    "No Google Cloud Platform support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with Hadoop File System support? [Y/n]: \n",
    "Hadoop File System support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with Amazon S3 File System support? [Y/n]: \n",
    "Amazon S3 File System support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with XLA JIT support? [y/N]: \n",
    "No XLA JIT support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with GDR support? [y/N]: \n",
    "No GDR support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with VERBS support? [y/N]: \n",
    "No VERBS support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with OpenCL support? [y/N]: \n",
    "No OpenCL support will be enabled for TensorFlow.\n",
    "\n",
    "Do you wish to build TensorFlow with CUDA support? [y/N]: y\n",
    "CUDA support will be enabled for TensorFlow.\n",
    "\n",
    "Please specify the CUDA SDK version you want to use, e.g. 7.0. [Leave empty to default to CUDA 8.0]: \n",
    "\n",
    "\n",
    "Please specify the location where CUDA 8.0 toolkit is installed. Refer to README.md for more details. [Default is /usr/local/cuda]: \n",
    "\n",
    "\n",
    "Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 6.0]: \n",
    "\n",
    "\n",
    "Please specify the location where cuDNN 6 library is installed. Refer to README.md for more details. [Default is /usr/local/cuda]:\n",
    "\n",
    "\n",
    "Please specify a list of comma-separated Cuda compute capabilities you want to build with.\n",
    "You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.\n",
    "Please note that each additional compute capability significantly increases your build time and binary size. [Default is: 6.1]\n",
    "\n",
    "\n",
    "Do you want to use clang as CUDA compiler? [y/N]: \n",
    "nvcc will be used as CUDA compiler.\n",
    "\n",
    "Please specify which gcc should be used by nvcc as the host compiler. [Default is /usr/bin/gcc]: \n",
    "\n",
    "\n",
    "Do you wish to build TensorFlow with MPI support? [y/N]: \n",
    "No MPI support will be enabled for TensorFlow.\n",
    "\n",
    "Please specify optimization flags to use during compilation when bazel option \"--config=opt\" is specified [Default is -march=native]: \n",
    "\n",
    "\n",
    "Add \"--config=mkl\" to your bazel command to build with MKL support.\n",
    "Please note that MKL on MacOS or windows is still not supported.\n",
    "If you would like to use a local MKL instead of downloading, please set the environment variable \"TF_MKL_ROOT\" every time before build.\n",
    "Configuration finished\n",
    "\n",
    "\n",
    "```\n",
    "\n",
    "#### Build with Bazel\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$ bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package \n",
    ".......\n",
    "WARNING: /home/deeplearn/downloads/temp/tensorflow/tensorflow/core/BUILD:1781:1: in includes attribute of cc_library rule //tensorflow/core:framework_headers_lib: '../../external/nsync/public' resolves to 'external/nsync/public' not below the relative path of its package 'tensorflow/core'. This will be an error in the future. Since this rule was created by the macro 'cc_header_only_library', the error might have been caused by the macro implementation in /home/deeplearn/downloads/temp/tensorflow/tensorflow/tensorflow.bzl:1044:30\n",
    "WARNING: /home/deeplearn/downloads/temp/tensorflow/tensorflow/contrib/learn/BUILD:15:1: in py_library rule //tensorflow/contrib/learn:learn: target '//tensorflow/contrib/learn:learn' depends on deprecated target '//tensorflow/contrib/session_bundle:exporter': No longer supported. Switch to SavedModel immediately.\n",
    "WARNING: /home/deeplearn/downloads/temp/tensorflow/tensorflow/contrib/learn/BUILD:15:1: in py_library rule //tensorflow/contrib/learn:learn: target '//tensorflow/contrib/learn:learn' depends on deprecated target '//tensorflow/contrib/session_bundle:gc': No longer supported. Switch to SavedModel immediately.\n",
    "INFO: Analysed target //tensorflow/tools/pip_package:build_pip_package (205 packages loaded).\n",
    "INFO: Found 1 target...\n",
    "INFO: From Compiling external/snappy/snappy-c.cc [for host]:\n",
    ".\n",
    ".\n",
    ".\n",
    ".\n",
    ".\n",
    ".\n",
    "        ^\n",
    "Target //tensorflow/tools/pip_package:build_pip_package up-to-date:\n",
    "  bazel-bin/tensorflow/tools/pip_package/build_pip_package\n",
    "INFO: Elapsed time: 1507.903s, Critical Path: 120.82s\n",
    "INFO: Build completed successfully, 6006 total actions\n",
    "\n",
    "```\n",
    "\n",
    "#### Build the pip package to a /tmp folder\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~/downloads/temp/tensorflow$  bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg\n",
    "Thu Nov 23 22:29:59 CET 2017 : === Using tmpdir: /tmp/tmp.COfRom0W7d\n",
    "~/downloads/temp/tensorflow/bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles ~/downloads/temp/tensorflow\n",
    "~/downloads/temp/tensorflow\n",
    "/tmp/tmp.COfRom0W7d ~/downloads/temp/tensorflow\n",
    "Thu Nov 23 22:30:00 CET 2017 : === Building wheel\n",
    "warning: no files found matching '*.dll' under directory '*'\n",
    "warning: no files found matching '*.lib' under directory '*'\n",
    "warning: no files found matching '*.h' under directory 'tensorflow/include/tensorflow'\n",
    "warning: no files found matching '*' under directory 'tensorflow/include/Eigen'\n",
    "warning: no files found matching '*' under directory 'tensorflow/include/external'\n",
    "warning: no files found matching '*.h' under directory 'tensorflow/include/google'\n",
    "warning: no files found matching '*' under directory 'tensorflow/include/third_party'\n",
    "warning: no files found matching '*' under directory 'tensorflow/include/unsupported'\n",
    "~/downloads/temp/tensorflow\n",
    "Thu Nov 23 22:30:11 CET 2017 : === Output wheel file is in: /tmp/tensorflow_pkg\n",
    "\n",
    "\n",
    "```\n",
    "\n",
    "#### Upgrade your installation to TensorFlow 1.4 😀\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip install --upgrade /tmp/tensorflow_pkg/tensorflow-1.4.1-cp36-cp36m-linux_x86_64.whl \n",
    "Processing /tmp/tensorflow_pkg/tensorflow-1.4.1-cp36-cp36m-linux_x86_64.whl\n",
    "Requirement already up-to-date: numpy>=1.12.1 in ./anaconda3/lib/python3.6/site-packages (from tensorflow==1.4.1)\n",
    "Requirement already up-to-date: six>=1.10.0 in ./anaconda3/lib/python3.6/site-packages (from tensorflow==1.4.1)\n",
    "Collecting wheel>=0.26 (from tensorflow==1.4.1)\n",
    "  Downloading wheel-0.30.0-py2.py3-none-any.whl (49kB)\n",
    "    100% |████████████████████████████████| 51kB 7.5MB/s \n",
    "Collecting protobuf>=3.3.0 (from tensorflow==1.4.1)\n",
    "  Downloading protobuf-3.5.0.post1-py2.py3-none-any.whl (389kB)\n",
    "    100% |████████████████████████████████| 389kB 2.9MB/s \n",
    "Collecting enum34>=1.1.6 (from tensorflow==1.4.1)\n",
    "  Downloading enum34-1.1.6-py3-none-any.whl\n",
    "Collecting tensorflow-tensorboard<0.5.0,>=0.4.0rc1 (from tensorflow==1.4.1)\n",
    "  Downloading tensorflow_tensorboard-0.4.0rc3-py3-none-any.whl (1.7MB)\n",
    "    100% |████████████████████████████████| 1.7MB 996kB/s \n",
    "Collecting setuptools (from protobuf>=3.3.0->tensorflow==1.4.1)\n",
    "  Downloading setuptools-37.0.0-py2.py3-none-any.whl (481kB)\n",
    "    100% |████████████████████████████████| 491kB 4.0MB/s \n",
    "Collecting html5lib==0.9999999 (from tensorflow-tensorboard<0.5.0,>=0.4.0rc1->tensorflow==1.4.1)\n",
    "Collecting bleach==1.5.0 (from tensorflow-tensorboard<0.5.0,>=0.4.0rc1->tensorflow==1.4.1)\n",
    "  Downloading bleach-1.5.0-py2.py3-none-any.whl\n",
    "Requirement already up-to-date: werkzeug>=0.11.10 in ./anaconda3/lib/python3.6/site-packages (from tensorflow-tensorboard<0.5.0,>=0.4.0rc1->tensorflow==1.4.1)\n",
    "Requirement already up-to-date: markdown>=2.6.8 in ./anaconda3/lib/python3.6/site-packages (from tensorflow-tensorboard<0.5.0,>=0.4.0rc1->tensorflow==1.4.1)\n",
    "Installing collected packages: wheel, setuptools, protobuf, enum34, html5lib, bleach, tensorflow-tensorboard, tensorflow\n",
    "  Found existing installation: wheel 0.29.0\n",
    "    Uninstalling wheel-0.29.0:\n",
    "      Successfully uninstalled wheel-0.29.0\n",
    "  Found existing installation: setuptools 36.6.0\n",
    "    Uninstalling setuptools-36.6.0:\n",
    "      Successfully uninstalled setuptools-36.6.0\n",
    "  Found existing installation: protobuf 3.4.0\n",
    "    Uninstalling protobuf-3.4.0:\n",
    "      Successfully uninstalled protobuf-3.4.0\n",
    "  Found existing installation: html5lib 1.0b10\n",
    "    Uninstalling html5lib-1.0b10:\n",
    "      Successfully uninstalled html5lib-1.0b10\n",
    "  Found existing installation: bleach 2.1.1\n",
    "    Uninstalling bleach-2.1.1:\n",
    "      Successfully uninstalled bleach-2.1.1\n",
    "  Found existing installation: tensorflow-tensorboard 0.1.5\n",
    "    Uninstalling tensorflow-tensorboard-0.1.5:\n",
    "      Successfully uninstalled tensorflow-tensorboard-0.1.5\n",
    "  Found existing installation: tensorflow 1.3.0\n",
    "    Uninstalling tensorflow-1.3.0:\n",
    "      Successfully uninstalled tensorflow-1.3.0\n",
    "Successfully installed bleach-1.5.0 enum34-1.1.6 html5lib-0.9999999 protobuf-3.5.0.post1 setuptools-37.0.0 tensorflow-1.4.1 tensorflow-tensorboard-0.4.0rc3 wheel-0.30.0\n",
    "\n",
    "```\n",
    "\n",
    "#### Finally, lets check if we're done?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Google's protobuf 3.5.0.post1 was giving me trouble\n",
    "\n",
    "So I uninstalled and solved it by uninstalling 3.5.0.post1, installing previous release 3.4.0 and then installing 3.5.0.post1 again.\n",
    "\n",
    "See my solution on [Github/Tensorflow] doc(https://github.com/tensorflow/tensorflow/issues/14689)\n",
    "\n",
    "Here's what I did (raw stuff):\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip uninstall protobuf\n",
    "Uninstalling protobuf-3.5.0.post1:\n",
    "  /home/deeplearn/anaconda3/lib/python3.6/site-packages/google/protobuf/__init__.py\n",
    "  /home/deeplearn/anaconda3/lib/python3.6/site-packages/google/protobuf/__pycache__/__init__.cpython-36.pyc\n",
    "  .\n",
    "  .\n",
    "  .\n",
    "  .\n",
    "  /home/deeplearn/anaconda3/lib/python3.6/site-packages/protobuf-3.5.0.post1.dist-info/top_level.txt\n",
    "Proceed (y/n)? y\n",
    "  Successfully uninstalled protobuf-3.5.0.post1\n",
    "  \n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip install protobuf==3.5.0\n",
    "Collecting protobuf==3.5.0\n",
    "  Could not find a version that satisfies the requirement protobuf==3.5.0 (from versions: 2.0.0b0, 2.0.3, 2.3.0, 2.4.1, 2.5.0, 2.6.0, 2.6.1, 3.0.0a2, 3.0.0a3, 3.0.0b1, 3.0.0b1.post1, 3.0.0b1.post2, 3.0.0b2, 3.0.0b2.post1, 3.0.0b2.post2, 3.0.0b3, 3.0.0b4, 3.0.0, 3.1.0, 3.1.0.post1, 3.2.0rc1, 3.2.0rc1.post1, 3.2.0rc2, 3.2.0, 3.3.0, 3.4.0, 3.5.0.post1)\n",
    "No matching distribution found for protobuf==3.5.0\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip install protobuf==3.4.0\n",
    "Requirement already satisfied: protobuf==3.4.0 in ./anaconda3/lib/python3.6/site-packages\n",
    "Requirement already satisfied: setuptools in ./anaconda3/lib/python3.6/site-packages (from protobuf==3.4.0)\n",
    "Requirement already satisfied: six>=1.9 in ./anaconda3/lib/python3.6/site-packages (from protobuf==3.4.0)\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip install protobuf==3.5.0.post1\n",
    "Collecting protobuf==3.5.0.post1\n",
    "  Using cached protobuf-3.5.0.post1-py2.py3-none-any.whl\n",
    "Requirement already satisfied: six>=1.9 in ./anaconda3/lib/python3.6/site-packages (from protobuf==3.5.0.post1)\n",
    "Requirement already satisfied: setuptools in ./anaconda3/lib/python3.6/site-packages (from protobuf==3.5.0.post1)\n",
    "Installing collected packages: protobuf\n",
    "  Found existing installation: protobuf 3.4.0\n",
    "    Uninstalling protobuf-3.4.0:\n",
    "      Successfully uninstalled protobuf-3.4.0\n",
    "Successfully installed protobuf-3.5.0.post1\n",
    "\n",
    "```\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1 : Testing the installation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: Getting the nightly build for eager execution / Upgrading to TensorFlow 1.5 OMG 😱\n",
    "\n",
    "```shell\n",
    "deeplearn@Ubuntu-1604-xenial-64-minimal:~$ pip install tf-nightly-gpu\n",
    "Collecting tf-nightly-gpu\n",
    "  Downloading tf_nightly_gpu-1.5.0.dev20171122-cp36-cp36m-manylinux1_x86_64.whl (121.7MB)\n",
    "    100% |████████████████████████████████| 121.7MB 17kB/s \n",
    "Requirement already satisfied: wheel>=0.26 in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: tb-nightly<1.6.0a0,>=1.5.0a0 in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: numpy>=1.12.1 in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: protobuf>=3.4.0 in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: six>=1.10.0 in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: absl-py in ./anaconda3/lib/python3.6/site-packages (from tf-nightly-gpu)\n",
    "Requirement already satisfied: html5lib==0.9999999 in ./anaconda3/lib/python3.6/site-packages (from tb-nightly<1.6.0a0,>=1.5.0a0->tf-nightly-gpu)\n",
    "Requirement already satisfied: bleach==1.5.0 in ./anaconda3/lib/python3.6/site-packages (from tb-nightly<1.6.0a0,>=1.5.0a0->tf-nightly-gpu)\n",
    "Requirement already satisfied: futures>=3.1.1 in ./anaconda3/lib/python3.6/site-packages (from tb-nightly<1.6.0a0,>=1.5.0a0->tf-nightly-gpu)\n",
    "Requirement already satisfied: werkzeug>=0.11.10 in ./anaconda3/lib/python3.6/site-packages (from tb-nightly<1.6.0a0,>=1.5.0a0->tf-nightly-gpu)\n",
    "Requirement already satisfied: markdown>=2.6.8 in ./anaconda3/lib/python3.6/site-packages (from tb-nightly<1.6.0a0,>=1.5.0a0->tf-nightly-gpu)\n",
    "Requirement already satisfied: setuptools in ./anaconda3/lib/python3.6/site-packages (from protobuf>=3.4.0->tf-nightly-gpu)\n",
    "Installing collected packages: tf-nightly-gpu\n",
    "Successfully installed tf-nightly-gpu-1.5.0.dev20171122\n",
    "\n",
    "```\n",
    "\n",
    "**Trying to execute eager execution** gave me this error (weird as I don't get it on my mac)\n",
    "\n",
    "```shell\n",
    "\n",
    "tfe.enable_eager_execution()\n",
    "---------------------------------------------------------------------------\n",
    "ValueError                                Traceback (most recent call last)\n",
    "<ipython-input-9-cf53c6a364bc> in <module>()\n",
    "----> 1 tfe.enable_eager_execution()\n",
    "\n",
    "~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in enable_eager_execution(config, device_policy)\n",
    "   4694     if graph_mode_has_been_used:\n",
    "   4695       raise ValueError(\n",
    "-> 4696           \"tfe.enable_eager_execution has to be called at program startup.\")\n",
    "   4697   context._default_mode = context.EAGER_MODE\n",
    "   4698   if context._context is None:\n",
    "\n",
    "ValueError: tfe.enable_eager_execution has to be called at program startup.\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deeplearn/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.eager as tfe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# solution is to bring down the protobuf version \n",
    "from google.protobuf import descriptor_pb2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.5.0-dev20171122\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfe.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# multiple 3 X 3 matrix\n",
    "x = tf.matmul([[1, 2, 3],\n",
    "               [4, 5, 6],\n",
    "               [7, 8, 9]\n",
    "              ],\n",
    "              [\n",
    "               [10, 11, 12],\n",
    "               [13, 14, 15],\n",
    "               [16, 17, 18],\n",
    "              ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add one to each element using tf.add for broadcasting\n",
    "y = tf.add(x, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a random random matrix of, say 6 by 4 matrix\n",
    "z = tf.random_uniform([6, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"MatMul:0\", shape=(3, 3), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(x) # This returned as per older situation without eager execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[ 84  90  96]\n",
      " [201 216 231]\n",
      " [318 342 366]], shape=(3, 3), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(x) # printed with eager execution , you can see kept execution cell 8 as it is :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[ 85  91  97]\n",
      " [202 217 232]\n",
      " [319 343 367]], shape=(3, 3), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[ 0.42166519  0.74874997  0.30295897  0.18815947]\n",
      " [ 0.13613379  0.41560864  0.78463495  0.76817131]\n",
      " [ 0.00482869  0.90712023  0.97884417  0.07941568]\n",
      " [ 0.85587406  0.86932099  0.58015025  0.09645569]\n",
      " [ 0.26412416  0.30468106  0.40509927  0.29306555]\n",
      " [ 0.57711077  0.2724632   0.35810065  0.68696404]], shape=(6, 4), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cool huh!\n",
    "\n",
    "What you see is that with eager execution enabled, these ops consume and return multi-dimensional arrays as `Tensor` objects pretty much like what you see in `Torch` or `Numpy ndarray` or other imperative constructs.\n",
    "\n",
    "[**its experimental, so don't go about playing with it in production!**] there I said it again 😌\n",
    "\n",
    "These operations can also be triggered via operator overloading of the `Tensor` object.\n",
    "\n",
    "Col thing is that you can use the `+` instead of tf.add, `-` for tf.subtract etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 3.], shape=(1,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "a = (tf.ones([1], dtype=tf.float32) + 1) *2 - 1\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=30, shape=(1,), dtype=float32, numpy=array([ 3.], dtype=float32)>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a # here you get more info about the numpy array"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Coverting to and from Numpy\n",
    "\n",
    "This operations converts python objects to `Tensor` objects and vice versa."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(2, shape=(), dtype=int32)\n",
      "tf.Tensor(2, shape=(), dtype=int64)\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "x = tf.add(1, 1)                             # tf.Tensor with a value of 2\n",
    "y = tf.add(np.array(1), np.array(1))         # tf.Tensor w a value of 2\n",
    "z = np.multiply(x, y)                        # numpu.int64 with a value of 4\n",
    "print(x)\n",
    "print(y)\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### you can explicitly convert using...\n",
    "\n",
    "`tf.constant` as you see next...\n",
    "\n",
    "You can also call `numpy()` method of a `Tensor` object to obtain its Numpy `ndarray` value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(6.0, shape=(), dtype=float32)\n",
      "6.0\n"
     ]
    }
   ],
   "source": [
    "np_x = np.array(2., dtype=np.float32)\n",
    "x = tf.constant(np_x)\n",
    "py_y = 3.\n",
    "y = tf.constant(py_y)\n",
    "\n",
    "# add them up + 1\n",
    "\n",
    "z = x + y + 1\n",
    "\n",
    "print(z)\n",
    "print(z.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CPU vs GPU acceleration and telling Eager Exec where to offload the computation\n",
    "\n",
    "Easiest way to do this is to enclose your computation in ith a `tf.device('/gpu:0')` block. In case you have multiple GPUs, this function might come handy `tfe.num_gpus()`, that should return the number of GPUs you have on your cluster.\n",
    "\n",
    "Here we multiply 1000 x 1000 matrices on a CPU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU: Took 0.17011046409606934 seconds to multiply a (1000, 1000) matrix by itself 10 times\n",
      "GPU: Took 0.00032329559326171875 seconds to multiply a (1000, 1000) matrix by itself 10 times\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "def measure(x):\n",
    "  # The very first time a GPU is used by TensorFlow, it is initialized.\n",
    "  # So exclude the first run from timing.\n",
    "  tf.matmul(x, x)\n",
    "\n",
    "  start = time.time()\n",
    "  for i in range(10):\n",
    "    tf.matmul(x, x)\n",
    "  end = time.time()\n",
    "\n",
    "  return \"Took %s seconds to multiply a %s matrix by itself 10 times\" % (end - start, x.shape)\n",
    "\n",
    "# Run on CPU:\n",
    "with tf.device(\"/cpu:0\"):\n",
    "  print(\"CPU: %s\" % measure(tf.random_normal([1000, 1000])))\n",
    "\n",
    "# If a GPU is available, run on GPU:\n",
    "if tfe.num_gpus() > 0:\n",
    "  with tf.device(\"/gpu:0\"):\n",
    "    print(\"GPU: %s\" % measure(tf.random_normal([1000, 1000])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# you can copy Tensors to different devices\n",
    "\n",
    "x = tf.random_normal([10, 10])\n",
    "\n",
    "x_gpu0 = x.gpu()\n",
    "x_cpu = x.cpu()\n",
    "\n",
    "_ = tf.matmul(x_cpu, x_cpu) # will run this operation on CPU\n",
    "_ = tf.matmul(x_gpu0, x_gpu0)# will run on your first GPU device, you can chang it to gpu1, gpu2 if you have many\n",
    "\n",
    "if tfe.num_gpus() > 1:\n",
    "    x_gpu1 = c.gpu()\n",
    "    _ = tf.matmul(x_gpu1, x_gpu1)  # will run on your second GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Automatic Differentiation with Eager Execution\n",
    "\n",
    "As per [Wikipedia](https://en.wikipedia.org/wiki/Automatic_differentiation):\n",
    "\n",
    "In mathematics and computer algebra, automatic differentiation (AD), also called algorithmic differentiation or computational differentiation,[1][2] is a set of techniques to numerically evaluate the derivative of a function specified by a computer program. AD exploits the fact that every computer program, no matter how complicated, executes a sequence of elementary arithmetic operations (addition, subtraction, multiplication, division, etc.) and elementary functions (exp, log, sin, cos, etc.). By applying the chain rule repeatedly to these operations, derivatives of arbitrary order can be computed automatically, accurately to working precision, and using at most a small constant factor more arithmetic operations than the original program.\n",
    "\n",
    "Automatic differentiation is not:\n",
    "- Symbolic differentiation, nor\n",
    "- Numerical differentiation (the method of finite differences).\n",
    "\n",
    "These classical methods run into problems: symbolic differentiation leads to inefficient code (unless carefully done) and faces the difficulty of converting a computer program into a single expression, while numerical differentiation can introduce round-off errors in the discretization process and cancellation. Both classical methods have problems with calculating higher derivatives, where the complexity and errors increase. Finally, both classical methods are slow at computing the partial derivatives of a function with respect to many inputs, as is needed for gradient-based optimization algorithms. Automatic differentiation solves all of these problems, at the expense of introducing more software dependencies.\n",
    "\n",
    "<img src=\"images/AutomaticDifferentiationNutshell.png\">\n",
    "\n",
    "**Automatic Differentiation** in TensorFlow is useful when implementing several deep learning algorithms. Eager Execution provides an autograd-like API for automatic differentiation, especially for functions such as these:\n",
    "\n",
    "- `tfe.gradients_function(f)` : This returns a python function that computes the derivatives of the python function `f` w.r.t its arguments. `f` should return a scalr value. When invoeked this function returns a list of `Tensor` objects (an element for each argument of `f`)\n",
    "- `tfe.value_and_gradients_function(f)` : Just like the above function this one returns , when invoked, the value of `f` in addition to the list of derivatives of `f` w.r.t its arguments.\n",
    "\n",
    "Lets take a look how this works with an example:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'numpy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-df6df4e4bd34>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# Third order derivative.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0md3f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradients_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0md2f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0md3f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'numpy'"
     ]
    }
   ],
   "source": [
    "def f(x):\n",
    "    return tf.multiply(x, x)  # Or x * x\n",
    "assert 9 == f(3.).numpy()\n",
    "\n",
    "df = tfe.gradients_function(f)\n",
    "assert 6 == df(3.)[0].numpy()\n",
    "\n",
    "# Second order deriviative.\n",
    "d2f = tfe.gradients_function(lambda x : df(x)[0])\n",
    "assert 2 == d2f(3.)[0].numpy()\n",
    "\n",
    "# Third order derivative.\n",
    "d3f = tfe.gradients_function(lambda x : d2f(x)[0])\n",
    "assert 0 == d3f(3.)[0].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss: 68.281876\n",
      "Loss at step 0: 65.645905\n",
      "Loss at step 20: 30.064030\n",
      "Loss at step 40: 14.059760\n",
      "Loss at step 60: 6.859705\n",
      "Loss at step 80: 3.619853\n",
      "Loss at step 100: 2.161713\n",
      "Loss at step 120: 1.505334\n",
      "Loss at step 140: 1.209814\n",
      "Loss at step 160: 1.076741\n",
      "Loss at step 180: 1.016808\n",
      "Final loss: 0.990712\n",
      "W, B = 3.074969, 2.132225\n"
     ]
    }
   ],
   "source": [
    "def prediction(input, weight, bias):\n",
    "    return input * weight + bias\n",
    "\n",
    "# A toy dataset of points around 3 * x + 2\n",
    "NUM_EXAMPLES = 1000\n",
    "training_inputs = tf.random_normal([NUM_EXAMPLES])\n",
    "noise = tf.random_normal([NUM_EXAMPLES])\n",
    "training_outputs = training_inputs * 3 + 2 + noise\n",
    "\n",
    "# A loss function: Mean-squared error\n",
    "def loss(weight, bias):\n",
    "    error = prediction(training_inputs, weight, bias) - training_outputs\n",
    "    return tf.reduce_mean(tf.square(error))\n",
    "\n",
    "# Function that returns the the derivative of loss with respect to\n",
    "# weight and bias\n",
    "grad = tfe.gradients_function(loss)\n",
    "\n",
    "# Train for 200 steps (starting from some random choice for W and B, on the same\n",
    "# batch of data).\n",
    "W = 5.\n",
    "B = 10.\n",
    "learning_rate = 0.01\n",
    "print(\"Initial loss: %f\" % loss(W, B).numpy())\n",
    "for i in range(200):\n",
    "  (dW, dB) = grad(W, B)\n",
    "  W -= dW * learning_rate\n",
    "  B -= dB * learning_rate\n",
    "  if i % 20 == 0:\n",
    "    print(\"Loss at step %d: %f\" % (i, loss(W, B).numpy()))\n",
    "print(\"Final loss: %f\" % loss(W, B).numpy())\n",
    "print(\"W, B = %f, %f\" % (W.numpy(), B.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}