02-supervised-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from preamble import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supervised Learning\n",
    "### Classification and Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generalization, Overfitting, and Underfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![model_complexity](images/overfitting_underfitting_cartoon.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Relation of Model Complexity to Dataset Size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Supervised Machine Learning Algorithms\n",
    "#### Some Sample Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "caption": "Forge dataset",
    "label": "forge_scatter"
   },
   "outputs": [],
   "source": [
    "# generate dataset\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "# plot dataset\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.legend([\"Class 0\", \"Class 1\"], loc=4)\n",
    "plt.xlabel(\"First feature\")\n",
    "plt.ylabel(\"Second feature\")\n",
    "print(\"X.shape:\", X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.make_wave(n_samples=40)\n",
    "plt.plot(X, y, 'o')\n",
    "plt.ylim(-3, 3)\n",
    "plt.xlabel(\"Feature\")\n",
    "plt.ylabel(\"Target\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "print(\"cancer.keys():\\n\", cancer.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Shape of cancer data:\", cancer.data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Sample counts per class:\\n\",\n",
    "      {n: v for n, v in zip(cancer.target_names, np.bincount(cancer.target))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Feature names:\\n\", cancer.feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "print(\"Data shape:\", boston.data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.load_extended_boston()\n",
    "print(\"X.shape:\", X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### k-Nearest Neighbors\n",
    "##### k-Neighbors classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_classification(n_neighbors=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_classification(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "clf = KNeighborsClassifier(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test set predictions:\", clf.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test set accuracy: {:.2f}\".format(clf.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Analyzing KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(10, 3))\n",
    "\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    # the fit method returns the object self, so we can instantiate\n",
    "    # and fit in one line\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors).fit(X, y)\n",
    "    mglearn.plots.plot_2d_separator(clf, X, fill=True, eps=0.5, ax=ax, alpha=.4)\n",
    "    mglearn.discrete_scatter(X[:, 0], X[:, 1], y, ax=ax)\n",
    "    ax.set_title(\"{} neighbor(s)\".format(n_neighbors))\n",
    "    ax.set_xlabel(\"feature 0\")\n",
    "    ax.set_ylabel(\"feature 1\")\n",
    "axes[0].legend(loc=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, stratify=cancer.target, random_state=66)\n",
    "\n",
    "training_accuracy = []\n",
    "test_accuracy = []\n",
    "# try n_neighbors from 1 to 10\n",
    "neighbors_settings = range(1, 11)\n",
    "\n",
    "for n_neighbors in neighbors_settings:\n",
    "    # build the model\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors)\n",
    "    clf.fit(X_train, y_train)\n",
    "    # record training set accuracy\n",
    "    training_accuracy.append(clf.score(X_train, y_train))\n",
    "    # record generalization accuracy\n",
    "    test_accuracy.append(clf.score(X_test, y_test))\n",
    "    \n",
    "plt.plot(neighbors_settings, training_accuracy, label=\"training accuracy\")\n",
    "plt.plot(neighbors_settings, test_accuracy, label=\"test accuracy\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.xlabel(\"n_neighbors\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### k-neighbors regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_regression(n_neighbors=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_regression(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "X, y = mglearn.datasets.make_wave(n_samples=40)\n",
    "\n",
    "# split the wave dataset into a training and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "# instantiate the model and set the number of neighbors to consider to 3\n",
    "reg = KNeighborsRegressor(n_neighbors=3)\n",
    "# fit the model using the training data and training targets\n",
    "reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test set predictions:\\n\", reg.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test set R^2: {:.2f}\".format(reg.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyzing KNeighborsRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n",
    "# create 1,000 data points, evenly spaced between -3 and 3\n",
    "line = np.linspace(-3, 3, 1000).reshape(-1, 1)\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    # make predictions using 1, 3, or 9 neighbors\n",
    "    reg = KNeighborsRegressor(n_neighbors=n_neighbors)\n",
    "    reg.fit(X_train, y_train)\n",
    "    ax.plot(line, reg.predict(line))\n",
    "    ax.plot(X_train, y_train, '^', c=mglearn.cm2(0), markersize=8)\n",
    "    ax.plot(X_test, y_test, 'v', c=mglearn.cm2(1), markersize=8)\n",
    "\n",
    "    ax.set_title(\n",
    "        \"{} neighbor(s)\\n train score: {:.2f} test score: {:.2f}\".format(\n",
    "            n_neighbors, reg.score(X_train, y_train),\n",
    "            reg.score(X_test, y_test)))\n",
    "    ax.set_xlabel(\"Feature\")\n",
    "    ax.set_ylabel(\"Target\")\n",
    "axes[0].legend([\"Model predictions\", \"Training data/target\",\n",
    "                \"Test data/target\"], loc=\"best\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Strengths, weaknesses, and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear Models\n",
    "##### Linear models for regression\n",
    "\\begin{align*}\n",
    "\\end{align*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_linear_regression_wave()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear regression aka ordinary least squares"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "X, y = mglearn.datasets.make_wave(n_samples=60)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
    "\n",
    "lr = LinearRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"lr.coef_:\", lr.coef_)\n",
    "print(\"lr.intercept_:\", lr.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training set score: {:.2f}\".format(lr.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lr.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.load_extended_boston()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "lr = LinearRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training set score: {:.2f}\".format(lr.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lr.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Ridge regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "ridge = Ridge().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge10 = Ridge(alpha=10).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge10.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge10.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge01 = Ridge(alpha=0.1).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge01.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge01.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(ridge.coef_, 's', label=\"Ridge alpha=1\")\n",
    "plt.plot(ridge10.coef_, '^', label=\"Ridge alpha=10\")\n",
    "plt.plot(ridge01.coef_, 'v', label=\"Ridge alpha=0.1\")\n",
    "\n",
    "plt.plot(lr.coef_, 'o', label=\"LinearRegression\")\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "xlims = plt.xlim()\n",
    "plt.hlines(0, xlims[0], xlims[1])\n",
    "plt.xlim(xlims)\n",
    "plt.ylim(-25, 25)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_ridge_n_samples()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "lasso = Lasso().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso.score(X_test, y_test)))\n",
    "print(\"Number of features used:\", np.sum(lasso.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we increase the default setting of \"max_iter\",\n",
    "# otherwise the model would warn us that we should increase max_iter.\n",
    "lasso001 = Lasso(alpha=0.01, max_iter=100000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso001.score(X_test, y_test)))\n",
    "print(\"Number of features used:\", np.sum(lasso001.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso00001 = Lasso(alpha=0.0001, max_iter=100000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso00001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso00001.score(X_test, y_test)))\n",
    "print(\"Number of features used:\", np.sum(lasso00001.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(lasso.coef_, 's', label=\"Lasso alpha=1\")\n",
    "plt.plot(lasso001.coef_, '^', label=\"Lasso alpha=0.01\")\n",
    "plt.plot(lasso00001.coef_, 'v', label=\"Lasso alpha=0.0001\")\n",
    "\n",
    "plt.plot(ridge01.coef_, 'o', label=\"Ridge alpha=0.1\")\n",
    "plt.legend(ncol=2, loc=(0, 1.05))\n",
    "plt.ylim(-25, 25)\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Linear models for classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{align*}\n",
    "\\end{align*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(10, 3))\n",
    "\n",
    "for model, ax in zip([LinearSVC(), LogisticRegression()], axes):\n",
    "    clf = model.fit(X, y)\n",
    "    mglearn.plots.plot_2d_separator(clf, X, fill=False, eps=0.5,\n",
    "                                    ax=ax, alpha=.7)\n",
    "    mglearn.discrete_scatter(X[:, 0], X[:, 1], y, ax=ax)\n",
    "    ax.set_title(clf.__class__.__name__)\n",
    "    ax.set_xlabel(\"Feature 0\")\n",
    "    ax.set_ylabel(\"Feature 1\")\n",
    "axes[0].legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_linear_svc_regularization()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, stratify=cancer.target, random_state=42)\n",
    "logreg = LogisticRegression().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg100 = LogisticRegression(C=100).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg100.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg100.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg001 = LogisticRegression(C=0.01).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg001.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(logreg.coef_.T, 'o', label=\"C=1\")\n",
    "plt.plot(logreg100.coef_.T, '^', label=\"C=100\")\n",
    "plt.plot(logreg001.coef_.T, 'v', label=\"C=0.001\")\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "xlims = plt.xlim()\n",
    "plt.hlines(0, xlims[0], xlims[1])\n",
    "plt.xlim(xlims)\n",
    "plt.ylim(-5, 5)\n",
    "plt.xlabel(\"Feature\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for C, marker in zip([0.001, 1, 100], ['o', '^', 'v']):\n",
    "    lr_l1 = LogisticRegression(C=C, solver='liblinear', penalty=\"l1\").fit(X_train, y_train)\n",
    "    print(\"Training accuracy of l1 logreg with C={:.3f}: {:.2f}\".format(\n",
    "          C, lr_l1.score(X_train, y_train)))\n",
    "    print(\"Test accuracy of l1 logreg with C={:.3f}: {:.2f}\".format(\n",
    "          C, lr_l1.score(X_test, y_test)))\n",
    "    plt.plot(lr_l1.coef_.T, marker, label=\"C={:.3f}\".format(C))\n",
    "\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "xlims = plt.xlim()\n",
    "plt.hlines(0, xlims[0], xlims[1])\n",
    "plt.xlim(xlims)\n",
    "plt.xlabel(\"Feature\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "\n",
    "plt.ylim(-5, 5)\n",
    "plt.legend(loc=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Linear models for multiclass classification\n",
    "\\begin{align*}\n",
    "\\end{align*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "\n",
    "X, y = make_blobs(random_state=42)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")\n",
    "plt.legend([\"Class 0\", \"Class 1\", \"Class 2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_svm = LinearSVC().fit(X, y)\n",
    "print(\"Coefficient shape: \", linear_svm.coef_.shape)\n",
    "print(\"Intercept shape: \", linear_svm.intercept_.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept, color in zip(linear_svm.coef_, linear_svm.intercept_,\n",
    "                                  mglearn.cm3.colors):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1], c=color)\n",
    "plt.ylim(-10, 15)\n",
    "plt.xlim(-10, 8)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")\n",
    "plt.legend(['Class 0', 'Class 1', 'Class 2', 'Line class 0', 'Line class 1',\n",
    "            'Line class 2'], loc=(1.01, 0.3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_2d_classification(linear_svm, X, fill=True, alpha=.7)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept, color in zip(linear_svm.coef_, linear_svm.intercept_,\n",
    "                                  mglearn.cm3.colors):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1], c=color)\n",
    "plt.legend(['Class 0', 'Class 1', 'Class 2', 'Line class 0', 'Line class 1',\n",
    "            'Line class 2'], loc=(1.01, 0.3))\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strengths, weaknesses and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate model and fit it in one line\n",
    "logreg = LogisticRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "y_pred = logreg.fit(X_train, y_train).predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = LogisticRegression().fit(X_train, y_train).predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[0, 1, 0, 1],\n",
    "              [1, 0, 1, 1],\n",
    "              [0, 0, 0, 1],\n",
    "              [1, 0, 1, 0]])\n",
    "y = np.array([0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = {}\n",
    "for label in np.unique(y):\n",
    "    # iterate over each class\n",
    "    # count (sum) entries of 1 per feature\n",
    "    counts[label] = X[y == label].sum(axis=0)\n",
    "print(\"Feature counts:\\n\", counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strengths, weaknesses and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_animal_tree()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Building decision trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_tree_progressive()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Controlling complexity of decision trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "6e5d7a76-9bba-42f7-b26e-907775d289b2"
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, stratify=cancer.target, random_state=42)\n",
    "tree = DecisionTreeClassifier(random_state=0)\n",
    "tree.fit(X_train, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(tree.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(tree.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = DecisionTreeClassifier(max_depth=4, random_state=0)\n",
    "tree.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(tree.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(tree.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analyzing Decision Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import export_graphviz\n",
    "export_graphviz(tree, out_file=\"tree.dot\", class_names=[\"malignant\", \"benign\"],\n",
    "                feature_names=cancer.feature_names, impurity=False, filled=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import graphviz\n",
    "\n",
    "with open(\"tree.dot\") as f:\n",
    "    dot_graph = f.read()\n",
    "display(graphviz.Source(dot_graph))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature Importance in trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "dc2f68ee-0df0-47ed-b500-7ec99d5a0a5d"
   },
   "outputs": [],
   "source": [
    "print(\"Feature importances:\")\n",
    "print(tree.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_feature_importances_cancer(model):\n",
    "    n_features = cancer.data.shape[1]\n",
    "    plt.barh(np.arange(n_features), model.feature_importances_, align='center')\n",
    "    plt.yticks(np.arange(n_features), cancer.feature_names)\n",
    "    plt.xlabel(\"Feature importance\")\n",
    "    plt.ylabel(\"Feature\")\n",
    "    plt.ylim(-1, n_features)\n",
    "\n",
    "plot_feature_importances_cancer(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "tree = mglearn.plots.plot_tree_not_monotone()\n",
    "display(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "ram_prices = pd.read_csv(os.path.join(mglearn.datasets.DATA_PATH, \"ram_price.csv\"))\n",
    "\n",
    "plt.semilogy(ram_prices.date, ram_prices.price)\n",
    "plt.xlabel(\"Year\")\n",
    "plt.ylabel(\"Price in $/Mbyte\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "# use historical data to forecast prices after the year 2000\n",
    "data_train = ram_prices[ram_prices.date < 2000]\n",
    "data_test = ram_prices[ram_prices.date >= 2000]\n",
    "\n",
    "# predict prices based on date\n",
    "X_train = data_train.date[:, np.newaxis]\n",
    "# we use a log-transform to get a simpler relationship of data to target\n",
    "y_train = np.log(data_train.price)\n",
    "\n",
    "tree = DecisionTreeRegressor(max_depth=3).fit(X_train, y_train)\n",
    "linear_reg = LinearRegression().fit(X_train, y_train)\n",
    "\n",
    "# predict on all data\n",
    "X_all = ram_prices.date[:, np.newaxis]\n",
    "\n",
    "pred_tree = tree.predict(X_all)\n",
    "pred_lr = linear_reg.predict(X_all)\n",
    "\n",
    "# undo log-transform\n",
    "price_tree = np.exp(pred_tree)\n",
    "price_lr = np.exp(pred_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.semilogy(data_train.date, data_train.price, label=\"Training data\")\n",
    "plt.semilogy(data_test.date, data_test.price, label=\"Test data\")\n",
    "plt.semilogy(ram_prices.date, price_tree, label=\"Tree prediction\")\n",
    "plt.semilogy(ram_prices.date, price_lr, label=\"Linear prediction\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strengths, weaknesses and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ensembles of Decision Trees\n",
    "##### Random forests\n",
    "###### Building random forests\n",
    "###### Analyzing random forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "b84dcdfe-994f-4a3d-842e-830153eefc59"
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y,\n",
    "                                                    random_state=42)\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators=5, random_state=2)\n",
    "forest.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "76ce4154-b441-475e-97e3-1b507964eb29"
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 3, figsize=(20, 10))\n",
    "for i, (ax, tree) in enumerate(zip(axes.ravel(), forest.estimators_)):\n",
    "    ax.set_title(\"Tree {}\".format(i))\n",
    "    mglearn.plots.plot_tree_partition(X_train, y_train, tree, ax=ax)\n",
    "    \n",
    "mglearn.plots.plot_2d_separator(forest, X_train, fill=True, ax=axes[-1, -1],\n",
    "                                alpha=.4)\n",
    "axes[-1, -1].set_title(\"Random Forest\")\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "forest = RandomForestClassifier(n_estimators=100, random_state=0)\n",
    "forest.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(forest.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(forest.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_feature_importances_cancer(forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Strengths, weaknesses, and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gradient Boosted Regression Trees (Gradient Boosting Machines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "gbrt = GradientBoostingClassifier(random_state=0)\n",
    "gbrt.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, max_depth=1)\n",
    "gbrt.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, learning_rate=0.01)\n",
    "gbrt.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, max_depth=1)\n",
    "gbrt.fit(X_train, y_train)\n",
    "\n",
    "plot_feature_importances_cancer(gbrt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Strengths, weaknesses and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Kernelized Support Vector Machines\n",
    "#### Linear Models and Non-linear Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_blobs(centers=4, random_state=8)\n",
    "y = y % 2\n",
    "\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "linear_svm = LinearSVC().fit(X, y)\n",
    "\n",
    "mglearn.plots.plot_2d_separator(linear_svm, X)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the squared first feature\n",
    "X_new = np.hstack([X, X[:, 1:] ** 2])\n",
    "\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D, axes3d\n",
    "figure = plt.figure()\n",
    "# visualize in 3D\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "# plot first all the points with y==0, then all with y == 1\n",
    "mask = y == 0\n",
    "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
    "           cmap=mglearn.cm2, s=60, edgecolor='k')\n",
    "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
    "           cmap=mglearn.cm2, s=60, edgecolor='k')\n",
    "ax.set_xlabel(\"feature0\")\n",
    "ax.set_ylabel(\"feature1\")\n",
    "ax.set_zlabel(\"feature1 ** 2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_svm_3d = LinearSVC().fit(X_new, y)\n",
    "coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_\n",
    "\n",
    "# show linear decision boundary\n",
    "figure = plt.figure()\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)\n",
    "yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)\n",
    "\n",
    "XX, YY = np.meshgrid(xx, yy)\n",
    "ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]\n",
    "ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)\n",
    "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
    "           cmap=mglearn.cm2, s=60, edgecolor='k')\n",
    "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
    "           cmap=mglearn.cm2, s=60, edgecolor='k')\n",
    "\n",
    "ax.set_xlabel(\"feature0\")\n",
    "ax.set_ylabel(\"feature1\")\n",
    "ax.set_zlabel(\"feature1 ** 2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ZZ = YY ** 2\n",
    "dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])\n",
    "plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],\n",
    "             cmap=mglearn.cm2, alpha=0.5)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The Kernel Trick\n",
    "#### Understanding SVMs\n",
    "\\begin{align*}\n",
    "\\end{align*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "X, y = mglearn.tools.make_handcrafted_dataset()                                                                  \n",
    "svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)\n",
    "mglearn.plots.plot_2d_separator(svm, X, eps=.5)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "# plot support vectors\n",
    "sv = svm.support_vectors_\n",
    "# class labels of support vectors are given by the sign of the dual coefficients\n",
    "sv_labels = svm.dual_coef_.ravel() > 0\n",
    "mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tuning SVM parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3, 3, figsize=(15, 10))\n",
    "\n",
    "for ax, C in zip(axes, [-1, 0, 3]):\n",
    "    for a, gamma in zip(ax, range(-1, 2)):\n",
    "        mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)\n",
    "        \n",
    "axes[0, 0].legend([\"class 0\", \"class 1\", \"sv class 0\", \"sv class 1\"],\n",
    "                  ncol=4, loc=(.9, 1.2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "19dca39b-9061-4fc6-9aab-f759854ec208"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "svc = SVC()\n",
    "svc.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.2f}\".format(svc.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.2f}\".format(svc.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.boxplot(X_train, manage_ticks=False)\n",
    "plt.yscale(\"symlog\")\n",
    "plt.xlabel(\"Feature index\")\n",
    "plt.ylabel(\"Feature magnitude\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Preprocessing data for SVMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the minimum value per feature on the training set\n",
    "min_on_training = X_train.min(axis=0)\n",
    "# Compute the range of each feature (max - min) on the training set\n",
    "range_on_training = (X_train - min_on_training).max(axis=0)\n",
    "\n",
    "# subtract the min, divide by range\n",
    "# afterward, min=0 and max=1 for each feature\n",
    "X_train_scaled = (X_train - min_on_training) / range_on_training\n",
    "print(\"Minimum for each feature\\n\", X_train_scaled.min(axis=0))\n",
    "print(\"Maximum for each feature\\n\", X_train_scaled.max(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use THE SAME transformation on the test set,\n",
    "# using min and range of the training set. See Chapter 3 (unsupervised learning) for details.\n",
    "X_test_scaled = (X_test - min_on_training) / range_on_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc = SVC()\n",
    "svc.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(\n",
    "        svc.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(svc.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc = SVC(C=1000)\n",
    "svc.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(\n",
    "    svc.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(svc.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strengths, weaknesses and parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Networks (Deep Learning)\n",
    "#### The Neural Network Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "display(mglearn.plots.plot_logistic_regression_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "display(mglearn.plots.plot_single_hidden_layer_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = np.linspace(-3, 3, 100)\n",
    "plt.plot(line, np.tanh(line), label=\"tanh\")\n",
    "plt.plot(line, np.maximum(line, 0), label=\"relu\")\n",
    "plt.legend(loc=\"best\")\n",
    "plt.xlabel(\"x\")\n",
    "plt.ylabel(\"relu(x), tanh(x)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_two_hidden_layer_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tuning Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y,\n",
    "                                                    random_state=42)\n",
    "\n",
    "mlp = MLPClassifier(solver='lbfgs', random_state=0).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(solver='lbfgs', random_state=0, hidden_layer_sizes=[10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using two hidden layers, with 10 units each\n",
    "mlp = MLPClassifier(solver='lbfgs', random_state=0,\n",
    "                    hidden_layer_sizes=[10, 10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using two hidden layers, with 10 units each, now with tanh nonlinearity.\n",
    "mlp = MLPClassifier(solver='lbfgs', activation='tanh',\n",
    "                    random_state=0, hidden_layer_sizes=[10, 10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for axx, n_hidden_nodes in zip(axes, [10, 100]):\n",
    "    for ax, alpha in zip(axx, [0.0001, 0.01, 0.1, 1]):\n",
    "        mlp = MLPClassifier(solver='lbfgs', random_state=0,\n",
    "                            hidden_layer_sizes=[n_hidden_nodes, n_hidden_nodes],\n",
    "                            alpha=alpha)\n",
    "        mlp.fit(X_train, y_train)\n",
    "        mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=ax)\n",
    "        mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, ax=ax)\n",
    "        ax.set_title(\"n_hidden=[{}, {}]\\nalpha={:.4f}\".format(\n",
    "                      n_hidden_nodes, n_hidden_nodes, alpha))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for i, ax in enumerate(axes.ravel()):\n",
    "    mlp = MLPClassifier(solver='lbfgs', random_state=i,\n",
    "                        hidden_layer_sizes=[100, 100])\n",
    "    mlp.fit(X_train, y_train)\n",
    "    mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=ax)\n",
    "    mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Cancer data per-feature maxima:\\n{}\".format(cancer.data.max(axis=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "mlp = MLPClassifier(random_state=42)\n",
    "mlp.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.2f}\".format(mlp.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.2f}\".format(mlp.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the mean value per feature on the training set\n",
    "mean_on_train = X_train.mean(axis=0)\n",
    "# compute the standard deviation of each feature on the training set\n",
    "std_on_train = X_train.std(axis=0)\n",
    "\n",
    "# subtract the mean, and scale by inverse standard deviation\n",
    "# afterward, mean=0 and std=1\n",
    "X_train_scaled = (X_train - mean_on_train) / std_on_train\n",
    "# use THE SAME transformation (using training mean and std) on the test set\n",
    "X_test_scaled = (X_test - mean_on_train) / std_on_train\n",
    "\n",
    "mlp = MLPClassifier(random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(\n",
    "    mlp.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(mlp.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(max_iter=1000, random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(\n",
    "    mlp.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(mlp.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(max_iter=1000, alpha=1, random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"Accuracy on training set: {:.3f}\".format(\n",
    "    mlp.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(mlp.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20, 5))\n",
    "plt.imshow(mlp.coefs_[0], interpolation='none', cmap='viridis')\n",
    "plt.yticks(range(30), cancer.feature_names)\n",
    "plt.xlabel(\"Columns in weight matrix\")\n",
    "plt.ylabel(\"Input feature\")\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strengths, weaknesses and parameters\n",
    "##### Estimating complexity in neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uncertainty estimates from classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.datasets import make_circles\n",
    "X, y = make_circles(noise=0.25, factor=0.5, random_state=1)\n",
    "\n",
    "# we rename the classes \"blue\" and \"red\" for illustration purposes:\n",
    "y_named = np.array([\"blue\", \"red\"])[y]\n",
    "\n",
    "# we can call train_test_split with arbitrarily many arrays;\n",
    "# all will be split in a consistent manner\n",
    "X_train, X_test, y_train_named, y_test_named, y_train, y_test = \\\n",
    "    train_test_split(X, y_named, y, random_state=0)\n",
    "\n",
    "# build the gradient boosting model\n",
    "gbrt = GradientBoostingClassifier(random_state=0)\n",
    "gbrt.fit(X_train, y_train_named)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The Decision Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_test.shape:\", X_test.shape)\n",
    "print(\"Decision function shape:\",\n",
    "      gbrt.decision_function(X_test).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show the first few entries of decision_function\n",
    "print(\"Decision function:\", gbrt.decision_function(X_test)[:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Thresholded decision function:\\n\",\n",
    "      gbrt.decision_function(X_test) > 0)\n",
    "print(\"Predictions:\\n\", gbrt.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make the boolean True/False into 0 and 1\n",
    "greater_zero = (gbrt.decision_function(X_test) > 0).astype(int)\n",
    "# use 0 and 1 as indices into classes_\n",
    "pred = gbrt.classes_[greater_zero]\n",
    "# pred is the same as the output of gbrt.predict\n",
    "print(\"pred is equal to predictions:\",\n",
    "      np.all(pred == gbrt.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_function = gbrt.decision_function(X_test)\n",
    "print(\"Decision function minimum: {:.2f} maximum: {:.2f}\".format(\n",
    "      np.min(decision_function), np.max(decision_function)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "mglearn.tools.plot_2d_separator(gbrt, X, ax=axes[0], alpha=.4,\n",
    "                                fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(gbrt, X, ax=axes[1],\n",
    "                                            alpha=.4, cm=mglearn.ReBl)\n",
    "\n",
    "for ax in axes:\n",
    "    # plot training and test points\n",
    "    mglearn.discrete_scatter(X_test[:, 0], X_test[:, 1], y_test,\n",
    "                             markers='^', ax=ax)\n",
    "    mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train,\n",
    "                             markers='o', ax=ax)\n",
    "    ax.set_xlabel(\"Feature 0\")\n",
    "    ax.set_ylabel(\"Feature 1\")\n",
    "cbar = plt.colorbar(scores_image, ax=axes.tolist())\n",
    "cbar.set_alpha(1)\n",
    "cbar.draw_all()\n",
    "axes[0].legend([\"Test class 0\", \"Test class 1\", \"Train class 0\",\n",
    "                \"Train class 1\"], ncol=4, loc=(.1, 1.1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predicting Probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Shape of probabilities:\", gbrt.predict_proba(X_test).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show the first few entries of predict_proba\n",
    "print(\"Predicted probabilities:\")\n",
    "print(gbrt.predict_proba(X_test[:6]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "    \n",
    "mglearn.tools.plot_2d_separator(\n",
    "    gbrt, X, ax=axes[0], alpha=.4, fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(\n",
    "    gbrt, X, ax=axes[1], alpha=.5, cm=mglearn.ReBl, function='predict_proba')\n",
    "\n",
    "for ax in axes:\n",
    "    # plot training and test points\n",
    "    mglearn.discrete_scatter(X_test[:, 0], X_test[:, 1], y_test,\n",
    "                             markers='^', ax=ax)\n",
    "    mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train,\n",
    "                             markers='o', ax=ax)\n",
    "    ax.set_xlabel(\"Feature 0\")\n",
    "    ax.set_ylabel(\"Feature 1\")\n",
    "# don't want a transparent colorbar\n",
    "cbar = plt.colorbar(scores_image, ax=axes.tolist())\n",
    "cbar.set_alpha(1)\n",
    "cbar.draw_all()\n",
    "axes[0].legend([\"Test class 0\", \"Test class 1\", \"Train class 0\",\n",
    "                \"Train class 1\"], ncol=4, loc=(.1, 1.1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![classifier_comparison](images/classifier_comparison.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Uncertainty in multiclass classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "\n",
    "iris = load_iris()\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    iris.data, iris.target, random_state=42)\n",
    "\n",
    "gbrt = GradientBoostingClassifier(learning_rate=0.01, random_state=0)\n",
    "gbrt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Decision function shape:\", gbrt.decision_function(X_test).shape)\n",
    "# plot the first few entries of the decision function\n",
    "print(\"Decision function:\")\n",
    "print(gbrt.decision_function(X_test)[:6, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Argmax of decision function:\")\n",
    "print(np.argmax(gbrt.decision_function(X_test), axis=1))\n",
    "print(\"Predictions:\")\n",
    "print(gbrt.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show the first few entries of predict_proba\n",
    "print(\"Predicted probabilities:\")\n",
    "print(gbrt.predict_proba(X_test)[:6])\n",
    "# show that sums across rows are one\n",
    "print(\"Sums:\", gbrt.predict_proba(X_test)[:6].sum(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Argmax of predicted probabilities:\")\n",
    "print(np.argmax(gbrt.predict_proba(X_test), axis=1))\n",
    "print(\"Predictions:\")\n",
    "print(gbrt.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "\n",
    "# represent each target by its class name in the iris dataset\n",
    "named_target = iris.target_names[y_train]\n",
    "logreg.fit(X_train, named_target)\n",
    "print(\"unique classes in training data:\", logreg.classes_)\n",
    "print(\"predictions:\", logreg.predict(X_test)[:10])\n",
    "argmax_dec_func = np.argmax(logreg.decision_function(X_test), axis=1)\n",
    "print(\"argmax of decision function:\", argmax_dec_func[:10])\n",
    "print(\"argmax combined with classes_:\",\n",
    "      logreg.classes_[argmax_dec_func][:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary and Outlook"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  },
  "latex_metadata": {
   "author": "Andreas C. M\\\"ller",
   "title": "Machine Learning with Python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}