{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Useful starting lines\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "from helpers import *\n",
    "\n",
    "height, weight, gender = load_data(sub_sample=False, add_outlier=False)\n",
    "x, mean_x, std_x = standardize(height)\n",
    "y, tx = build_model_data(x, weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000,), (10000, 2))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape, tx.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NB: throughout this laboratory the data has the following format: \n",
    "  * there are **N = 10000** data entries\n",
    "  * **y** represents the column vector containing weight information -- that which we wish to predict/the output (see also the first page of $\\texttt{exercise02.pdf}$). Its **shape** is **(N,)**.\n",
    "  * **tx** represents the matrix $\\tilde{X}$ formed by laterally concatenating a column vector of 1s to the column vector of height information -- the input data (see also the first page of $\\texttt{exercise02.pdf}$). Its **shape** is **(N,2)**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Computing the Cost Function\n",
    "Fill in the `compute_loss` function below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### SOLUTION\n",
    "def calculate_mse(e):\n",
    "    \"\"\"Calculate the mse for vector e.\"\"\"\n",
    "    return 1 / 2 * np.mean(e**2)\n",
    "\n",
    "\n",
    "def calculate_mae(e):\n",
    "    \"\"\"Calculate the mae for vector e.\"\"\"\n",
    "    return np.mean(np.abs(e))\n",
    "\n",
    "\n",
    "### TEMPLATE\n",
    "### END SOLUTION\n",
    "\n",
    "\n",
    "def compute_loss(y, tx, w):\n",
    "    \"\"\"Calculate the loss using either MSE or MAE.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        w: numpy array of shape=(2,). The vector of model parameters.\n",
    "\n",
    "    Returns:\n",
    "        the value of the loss (a scalar), corresponding to the input parameters w.\n",
    "    \"\"\"\n",
    "    ### SOLUTION\n",
    "    e = y - tx.dot(w)\n",
    "    return calculate_mse(e)\n",
    "    ### TEMPLATE\n",
    "    # # ***************************************************\n",
    "    # # INSERT YOUR CODE HERE\n",
    "    # # TODO: compute loss by MSE\n",
    "    # # ***************************************************\n",
    "    # raise NotImplementedError\n",
    "    ### END SOLUTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fill in the function `grid_search()` below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from costs import *\n",
    "\n",
    "\n",
    "def grid_search(y, tx, grid_w0, grid_w1):\n",
    "    \"\"\"Algorithm for grid search.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        grid_w0: numpy array of shape=(num_grid_pts_w0, ). A 1D array containing num_grid_pts_w0 values of parameter w0 to be tested in the grid search.\n",
    "        grid_w1: numpy array of shape=(num_grid_pts_w1, ). A 1D array containing num_grid_pts_w1 values of parameter w1 to be tested in the grid search.\n",
    "\n",
    "    Returns:\n",
    "        losses: numpy array of shape=(num_grid_pts_w0, num_grid_pts_w1). A 2D array containing the loss value for each combination of w0 and w1\n",
    "    \"\"\"\n",
    "\n",
    "    losses = np.zeros((len(grid_w0), len(grid_w1)))\n",
    "    ### SOLUTION\n",
    "    # compute loss for each combinationof w0 and w1.\n",
    "    for ind_row, row in enumerate(grid_w0):\n",
    "        for ind_col, col in enumerate(grid_w1):\n",
    "            w = np.array([row, col])\n",
    "            losses[ind_row, ind_col] = compute_loss(y, tx, w)\n",
    "    ### TEMPLATE\n",
    "    # # ***************************************************\n",
    "    # # INSERT YOUR CODE HERE\n",
    "    # # TODO: compute loss for each combination of w0 and w1.\n",
    "    # # ***************************************************\n",
    "    # raise NotImplementedError\n",
    "    ### END SOLUTION\n",
    "    return losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us play with the grid search demo now!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid Search: loss*=42.42448314678248, w0*=66.66666666666669, w1*=16.666666666666686, execution time=0.008 seconds\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAF5CAYAAAAmk6atAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAB0f0lEQVR4nO3deZxU5ZX/8c9hB0FEEAQkIy5kIrigRDCLwZBENEZB1DEqmITEhTbByWSSJv6UciFoNnVig+IKRGMYBCWKiDqiMREUBEU0KgraLIIIIosgy/n9ce61qpvq7uruqrq3qs779epXVd+6VfXc7qb7y7OcR1QV55xzzjkXf02iboBzzjnnnMuMBzfnnHPOuQLhwc0555xzrkB4cHPOOeecKxAe3JxzzjnnCoQHN+ecc865AhF5cBORe0RkvYi8lnIsISKrRWRJ8HF6ymNjRGS5iLwpIqdG02rnXL6ISCsReVFEXhGRZSJybXD8dyLyLxF5VURmisgBKc9J+3tCRE4QkaXBY/8jIhIcbykifw2OLxCRQ/N9nc45l4nIgxtwHzA4zfGbVfW44GM2gIgcBZwP9A6eM0FEmuatpc65KOwEvqmqxwLHAYNFZADwJNBHVY8B3gLGQJ2/JyYClwBHBh/h756RwCZVPQK4GbgpD9flnHP1FnlwU9XngI0Znn4W8KCq7lTVFcBy4MScNc45Fzk1W4NPmwcfqqpzVXV3cHw+cEhwP+3vCRHpCuyvqi+oVR6fAgxJec7k4P50YFDYG+ecc3ESeXCrxRXBEMg9ItIhONYdqEw5Z1VwzDlXxESkqYgsAdYDT6rqgmqn/Ah4PLhf0++J7sH96serPCcIg5uBjlm8BOecy4pmUTegBhOB6wENbv+A/WJO9z/gtHt2icgl2JAIRx111Ak9Nr7OhgOz07iPW++fnRdK8SEHZf01a/PJ1gPy+n5uX/u3/Tjv73kQH9b6+DuLPtmgqvX6Yewvopsb0aY3YRmwI+XQJFWdlHqOqu4Bjgvmsc0UkT6q+hqAiFwF7AbuD06v6fdEbb8/Mv7dEoVOnTrpoYcemtG527ZtY7/99sttg2KiVK61VK4TSuda67rORYsW1fi7OJbBTVXXhfdF5E7g0eDTVUCPlFMPAdbU8BqTgEkA/fr10zmnAL9qfNtmHfudxr9INbdzKb2y/qo1e/y5s/P4bq4mnwCnnTwjr+95GXfU+vhZMve9+r7mZuDuhjYI+BrsUNV+mZyrqh+LyDxsbtprInIxcAYwSJMbL9f0e2IVyeHU1OOpz1klIs2A9mQ+hSPnDj30UBYuXJjRufPmzWPgwIG5bVBMlMq1lsp1Qulca13XKSI1/i6O5VBpMBclNBQIV5zOAs4PVoD1xCYXv1jnC366KOttzJbbuTSv7+ehLV7y/f3I989bNojIQeGKURFpDXwL+JeIDMb+O3amqm5PeUra3xOquhbYIiIDgvlrI4BHUp5zcXD/HOD/UoKgc87FRuTBTUT+ArwAfFFEVonISOC3wZL9V4FTgP8EUNVlwDTgdWAOUBYModQtpr1t+eShLZ48vNWpK/BM8PvgJWyO26PAbUA74MmgbNDtUOfvicuBu7AFC++QnBd3N9BRRJYDPwfK83JlzjlXT5EPlarq99McrnHkRVXHAeNy16L8yecfUA9t8fb4c2fnfdi0UKjqq0DfNMePqOU5aX9PqOpCoE+a4zuAcxvXUuecy73Ie9wKRSH3tnloKwz5/D4VYK+bc845PLhFxv9wunQ8vDnnnKuNB7cMZLu3zYdInXPOOdcQHtyKmIe2wuS9bs4552riwa0Ohdrb5qGtsHl4c845l44Htzzy0Obqw7+PzjnnqvPgVotCXEnqf+yLS76+n97r5pxzhcGDW574H0bXUB7enHPOhSIvwBtX3tsWE4lGPl4kvECvc84VsMpK6NAB2rZt9Et5cMuDfPRkFFVoS2Tx3Pq8Vsx5eHPOuQK0fTuccQa0bw/PPgsijXo5D2455qEtA4mIXjuX75sjHt6cc66AqMLll8PSpTB7dqNDG3hwS6uQhkkLNrQlom4AVduQqOGcGPLw5pxzBeKOO2DKFEgkYPDgrLykL07IoVz3thVcaEukfMRNgni2qwYF9713zrkSUlkJt414ER09Gk4/Ha6+Omuv7T1u1RRSb1tBSETdgHpKUHhtds45Fyv3/e5DfjB1GJvad+PAqVOhSfb6yTy45UhJ97Ylom5AIyWq3caUD5k651wM7dnDfy++gKZNP+SjB/4JBx6Y1Zf3odIU2eptK9nQliD2YadeEsT+emL7s1BCROQeEVkvIq+lHPudiPxLRF4VkZkickDKY2NEZLmIvCkip0bSaOdc7lxzDa2ef4rmkyZw8OnHZ/3lPbgVmNj+oU5E3YAcSkTdABdz9wHVZx0/CfRR1WOAt4AxACJyFHA+0Dt4zgQRaZq/pjrncupvf4Pf/AZ+/GP40Y9y8hYe3AKF0NsWy9CWoDSCTYLYXmcsfy5KiKo+B2ysdmyuqu4OPp0PHBLcPwt4UFV3quoKYDlwYt4a65zLneXLYfhwOP54+NOfcvY2PsfNNVwi6gZEIFHtNiZ8vlus/Qj4a3C/OxbkQquCY/sQkUuASwC6dOnCvHnzMnqzrVu3ZnxuoSuVay2V64TCvdYmO3ZwfFkZLVVZ9N//zY7582s9vzHX6cEN721rkETUDYhYotqtc2mIyFXAbuD+8FCa0zTdc1V1EjAJoF+/fjpw4MCM3nPevHlkem6hK5VrLZXrhAK9VlW4+GJYsQJmz2ZABvXaGnOdPlSaJSUT2hJ4WEmViLoBSbH6OXGIyMXAGcCFqhqGs1VAj5TTDgHW5Lttzrksuv12mDoVxo7NWpHd2pR8cIt73bZY/TFORN2AmEoQm69NrH5eSpiIDAZ+BZypqttTHpoFnC8iLUWkJ3Ak8GIUbXTOZcGCBTB6NJx2WlaL7NbGh0qzIFe9bbH6I5yIugEFIFHt1pUEEfkLMBDoJCKrgLHYKtKWwJNiexPOV9XLVHWZiEwDXseGUMtUdU80LXfONcqHH8I550D37vDnP2e1yG5tSjq4xb23LRYSUTegACWI9OvmCxXyS1W/n+bw3bWcPw4Yl7sWOedybs8euOACC2//zH6R3dqU/FBpYxV1b1si6gYUsASRhzfnnHM5cs018NRTUFFh5T/yqGSDW5x722LxRzcRdQOKRCLqBjjnnMuqWbOsyO7IkfaRZyUb3LIh11tbRSYRdQOKTCKat43FfwCcc66AVFZCebndprV8OYwYAccfT+Wvbqv93BwpyeDmvW01SOChLVcS0bythzfnnMtcRQXcdBPceGOaALd9OwwbBk2bwkMPUXF3K266CSZMyG8bS3pxQmMUXW9bIuoGOOecc9EqKwMR2LzZApwIjB+PFdm9/HJYuhRmz4ZDD/383FGjLOBVVNjze/So820apeSCm/e2pZGI5m1LToJIvta+ytQ55zLTo4cFtcpKaN/eQhnAxvF3cOCUKWz+zwTtgyK74blgvXNVgl4OleRQaWPlorctktCWwENbviWieVsfMnXOucz16GGhraIC1v3tRfa/ZjSzOY2bWqQvsltWZuEtDHq55MGtVCWibkAJS0Tzth7enHOlovoigzoXHaRRUQF33/QhLS86B7p1Y+HoP3N5WfrYFPa+5XqYFEosuGVjmLQoetsS+X07l0Yi6gYUDhHpISLPiMgbIrJMREYHx48TkfkiskREForIiSnPGSMiy0XkTRE5NeX4CSKyNHjsfyTY1iDYguqvwfEFInJo3i/UOZc14SKDcOFA9c8zCXJDz9zDY+0vYP8d62k2czrX3HJgXoJZXUoquDkXK4n8v2WB9rrtBv5LVb8EDADKROQo4LfAtap6HHBN8DnBY+cDvYHBwAQRaRq81kTgEmyP0CODxwFGAptU9QjgZuCmPFyXcy5HwqHLIUPsdujQqkOZ1YNcOtt+cQ0nbn6KmYMq4IQT8tLuTJRMcIvrogTvbStxiagbEH+qulZVXw7ubwHeALoDCuwfnNYeWBPcPwt4UFV3quoKYDlwooh0BfZX1RdUVYEpwJCU50wO7k8HBoW9cc65whMOXc6caQHt4YerDmVWn5O2Tw/crFl884Xf8NIxIznxjvwX2a1Nya0qbYyCLwGSiLoBzjVOMITZF1gAXAk8ISK/x/4T+pXgtO7A/JSnrQqO7QruVz8ePqcSQFV3i8hmoCOwIRfX4ZzLj9SSHalSV4RCsgfu2Wdhxm+X0zUosvvlf9wGrfLb5rqURHD7uPX+dZ/kXFQS5DVUZ7s8SNsD4aun1n1ejf5CJxFZmHJkkqpOqn6aiLQFHgKuVNVPROQG4D9V9SEROQ/b2P1bQLqeMq3lOHU85pwrUNUDWk3Kyiy0vTI/KLLbpAlMnw6tMk9t+arlVjJDpXGU12HSRP7eyjVAIuoGRGqDqvZL+UgX2ppjoe1+VQ1T58VAeP9/gXBxwiog9dfmIdgw6qrgfvXjVZ4jIs2wodeNjb0w51xh6NEDpv1Veb7P5Ry8YSncfz/07Fmv18hk3lw2eHDLUMEPkzqXopAWKQRzze4G3lDVP6Y8tAb4RnD/m8Dbwf1ZwPnBStGe2CKEF1V1LbBFRAYErzkCeCTlORcH988B/i+YB+ecKxE9Zt/B8a9N4emvXENln9Pq/fx81XLz4BYR721z+0hE3YDY+iowHPhmUPpjiYicDvwE+IOIvAL8BlstiqouA6YBrwNzgDJV3RO81uXAXdiChXeAx4PjdwMdRWQ58HOgPC9X5pxrlIbUZ0vrxRdh9Gje7DmY7/zjmrS9ZnW9V75quZXEHLeSloi6Aa5eEuTte1YoW2Gp6vOkn4MGkHaNvqqOA8alOb4Q6JPm+A7g3EY00zkXgXB4cssWaNfOyn7MnFn7PLN95qJt2ADnnANdu9L24T9z+R1N2LzZzkt9jfC98rGtVW08uGUg28OkhTRM5ZxzzsVV9U3hn30W5s9PH67CwPbJJzBxYnDODXvYMfT7NF2zno8e+Qfdj+lIu3b2WosXwy23JINgTStU882DWzFLRN0A1yAJvNfNOecyUH1T+CFDrGbbqFH79qyFPWYjRsCAAXYuY8fS6vmnGMldtJp9AvoYbN0KfftaALzyyqpBMMqetpAHtzzz3jaXkQQevJ1zrg6p4SzcFD4MauXlVYc2U3vn5s+HN383i/4PjWPr+SPpfOhINm+2njiw1zr11KpBMC48uNWhYFeTJqJugGu0BHn5Pnqvm3Ou0KQb9lTdN6ht2UKV+Wph79xhe5dzwf0jWNXlePS62xh/pB0P90spL0/Ob+vfP7rrTMeDWzFKRN0A55xzLnfCYc9Ro6qW4Eidg9ajh4W5MNhVVATHO27nFy8MY/vuJnx93XQOHtGKadOSw6n5KqTbUJGXAxGRe0RkvYi8lnLsQBF5UkTeDm47pDw2RkSWi8ibItKYeu1558Okrt4S+Xkb/9l0zhWSsGZaeXmyBEfqLgk1lu1QZduIy9n76lLe/839dOjbk/nz4cYbk6eMH2+hMPVYnMShx+0+4DZsw+dQOfC0qt4oIuXB578SkaOA84HeQDfgKRHplVKjKasKcpg0EXUDXNYl8O+rc86lqG0rq9SyHWPG2KKFz+eo3XEH+z00hQRj2bn+NAYMsNWjhSTyHjdVfY59t5Y5C5gc3J8MDEk5/qCq7lTVFVgRzRMpAN6j4eLOf0adc3FWvQBuTQVxw964IUMsxIW36/5mRXY/HTiYz351DaNGWbArL7eVpuFrXXyxrTodMaLm945SHHrc0ukSbE+Dqq4Vkc7B8e7A/JTzVgXH9iEilxBUUj/oC5lvElvQElE3IEeeWbDvsVNiNls01xIU7/fXOecyEPakPfssTJsGV10FU6fCmjUwZUrVuWnjx1sv28SJMHcuvL94A+X7W5Hdjbf+mb0PNGHNGpgcdBFNnlx1kcP8+baaNFyYEJfiuxDf4FaTdNXT0+4nGGxUPQngiH7t673nYDaHSfPSk5HI/VvkVbqwlsnjxRzoEhTf99k55zJQWWkBrXNnC1UTJsDSpfbY3Ln2+PjxFr62bEkuRAA4pvceHvz4QtqvWc8HFf9g2CUdWbDAwt7atXbO8OEW9DZvth636oV241J8F+Ib3NaJSNegt60rsD44vgpIXeNxCLbRtCsWdQW2hjy/mMOcc84VqNpWb4ZBDCxIjR4NC4Jf72Hx3NWrLXitW2dBbutWe3zLluSQ5qhR8JtmCdqvmAt33sktfz+BBQugSxd7bqdOtuNVu3ZU2TEhXGUaqm1OXb7FNbjNAi4GbgxuH0k5/oCI/BFbnHAk8GIkLYyTRNQNyILGBrb6vnahhrkExfH9ds6VrHQ12MLiuYMG2TnhMChYkFqwwHYzOPpo2G8/G9qcOtXmob31VnIeG8DLLyd7337xxb/R/s0b+Me/j+T+l3/8eW/a6tX2/NNPh7ZtbXj0K1+Brl2TPXphzbe4lQaJPLiJyF+AgUAnEVkFjMUC2zQRGQm8T7D5s6ouE5FpwOvAbqAsFytKC26YtFDlMqzV570LLcQl8PDmnCtY6WqwhceOPtrC0hNP2Llduth+oeHuBanPHTUK5s2D11+HkSMtfAEsW2Yha8BB73DVm8P5137HM+hft7HzX9Yr162bBbHu3au+5uLF1gs3YEBySDROc9tCkQc3Vf1+DQ8NquH8ccC43LWowCSibkADRBnY0nlmQeGFN+ecK1Cp88XCXqzwWOfOFpDWr7fQNmlScpP3Hj2qPreiwkIbWFgDaN0aPv0U/vnUdp7dPQyaNGHSt6fzb6+34q23rDdu6lQbTlW19wp74U46yT6/5RZ7rfJyGDo0s7lt+eyZizy4FbOc97YlcvvyWRe3wJYqbFuhBLgEhff9d86VlJrCTLr5YuGxefOSx7p1g+nTkytHw56yIUPg3HPh17+2cLdunZ3fty/07AkzZii37h7FMbzKsGaP8vDDPenb18454QT4xjfgn/+EJUvsWDinraIiuZq0+vZZdclnz5wHt2oKsuhu3MU5sFXnvW/OOddgqWEtNcxU3wC+NoMHw333WaAKe9TCnrLly+GRR2D3brjsMhg3zubD9e0LBx1k5ya6TuIHayeTYCyLupzOqO/ZXLjU4dYlS/g8zIVz2qqvHK3PKtJ8rjr14JYjJd/bVkhhrbpCCW8J4v9z4JwrKalhrfqwZqYh7je/seFOgJ077XbXLgtaTz9toQ1sPtpPf2rnVlbCnDnwZV7kLvkZf287mOu2XoNWWo/cZZfZ/Dmw4c9nn7Xiu9On27EhQ/btCaxPz1k+V516cEvhvW1ZUMiBLVWhhDcXORG5BzgDWK+qfYJjBwJ/BQ4FVgLnqeqm4LExwEhgD/AzVX0igmY7lxPV56+FYaY+Ie7WW62HbOVK+Owz6NDBVo4CtGhR9f3CgLdmDXRkA9M5hzXalb+d92eOW9yExYvh+ectvC1Zkiz7MX++tW1+UNI/tdhu3EW+5VUxKtnetmIJbaFnFsT/mhJRN8Bh+y0PrnYs3G/5SODp4HOq7bc8GJggIk3z11TnciccJk1ddBAKQ1y4wKD6atLzzkvWXuvfH9q0sdDWrBkce6wdb9HCjqWzedMe/trk+3RmPT9uP52fJjryyCP2Hn372qIFsFWln3xix2+5Jbk6NQ6FdTPlwa3QJKJuQA3iHnAao5ivzTVaqey37FxdwhB244377uu5YIGV2ViwwMLbqFEW5Nasgd69refrrLNg2zZ77mGH2fN277ZhTYCmtfwXZ0KnsQza+xRXcBsvN+kH2Puo2hDqp5/a+7z6qtV3239/C4gVFfYRlxptmfDgFvBh0kYohWAT52tMRN0Al0aV/ZaB1P2WU7eprnG/ZecKTdiTtmVLMsCFLrvMQtvll1ugO/dcC1BTp8K779o5ixfDO+/Ycw8+GHr1suNhfbY9QdXW5s2rvu93eZRLN4zjie4/4m5+zKZNtmChshJeeCF5XosWNlyaWqetEIlqvbfxLDhH9Guvf1w4oNZzshXccjpMmsjdSzdYnANNLsR13lsiS6/zDVmkqv3q85R+HUUXntrwt5S/UO/3jBsRORR4NGWO28eqekDK45tUtYOIVAAvqOqfg+N3A7NV9aE0r3kJcAlAly5dTnjwwQczasvWrVtp27ZtI6+oMJTKtcb1OnftSu712bWr3a5fb8c/+shWeX7hC3Z85Uo71rGjBa8PPrBb1eRigyZNoFu3rXz0UVvatrXX+fhje0zEPvbuteft2mXH229YzUW3XMrmjt3460//xK5mLQEbYm3RArZvt2FSVXvunj02x+0LX9g3AOZTXd/TU045pcbfi744wTVcqYU2iG+9twTxDPalq9H7LavqJGASQL9+/XTgwIEZvfG8efPI9NxCVyrXGsfrDHvNwv1Dy8ttg/aJE+Hss20IdMwYq4+2dasNgb75Jhx/PJxzDvz85xbyevWy4LZ6ta0g/f3v5/Hf/z0Q1eQ+oqk6dIBNm+x+a7bzT77CdlowYPUTrCzvCVhoC8PggAE2x23ixKqvE86vq35N+Sqi25jvqQc3vLetQUoxtKXyVaeudr7fsitqFRXJ/UP79LHQFhbC/b//s56yRMKGP1MtW2bbWa0P/ivz1lvWWxeW/YDk0GhqaGvXzs5bufLzs7i39SiO+fRVzuBRVmKhLVzA0KmT7UN6ww129tat8OKL9r6bNtkQamVl1YAWx+2t0vE5boUgEXUDqin10BaK29chEXUDSlOw3/ILwBdFZFWwx/KNwLdF5G3g28HnqOoyINxveQ452m/ZuVwL57M98ojtaDBxIqxYYY+Fw5sHH2yrQ8F2NGhpo5isX59c5dmpk60arV7mo7otW2xYNlxVeimT+I9PJ3M9V/M4p39+Xpcudrthg7WrRw/7mDIF/vUveOUV64VbvNiK7qa7prjPf/Metywpmc3k4xZWouY9byXP91t2pSgs71FZacOifftaD9sTT1jIatfOtpXavt0CVPPmyV61Dh1skcIf/mA9dXPm1Pw+4d6jYK8LcKK8xK36M+ZwKtdxzT7njxpl565ebfXg2ra1/UjDPU+nTbPQVj2g5bOIbmOUfHCL/WrSRNQNSOGhLb04zXtLEK+fGedcUauosJWhYMOP4RDqiBHWQ9a+PRxzTNXVnZs2we9/X3NNtlDz5snQBtZj13bnBmbIMNZqVy7kfvbSFJHk8Orhh1tg3LYN3n47+dzFi63kSDgMmi6g5XOOW2OUfHBzzjnnXGZSww1YMdvhw62HbdQoe3zo0ORqU0jfo1ZXaANbOdqyZbKnbvfOPTzABXTcu57Rxz/Pxpc7AsnQ1qSJDYeGQ7ahvn2t2G64V2lNCmWOmwe3LMjZMGkiNy/bIN7bVjcfNnXOFbnUcBOuIh0xwoYmBw+21Zxr1yaHONu0seFSqDrsmalmzSyYffYZjCXBd3iS3/W6k/9dYZUymjSxMh9gt2uqrdHu29fm4fXoYQV3KyttHlu6XrXUbbni3Pvmwc25bIpDeEsQr9DvnCsaQ4fC3Lk2f+zDD+3Y3/++by+XiN22aWMLE3btsvCVXBWamW3b7Pa7PMrV3MDd/IhfvvXjzx8PQ1u4mnTnTptDd9BBttp1v/2s92zMGAtgtfWqpc5xKy+Pb++bBzdXN+9tq584hDfnnMuyykoYPdrmiy1enFzBGa4i7dDBVmy+/74tCFi6FC65BO69Nzl0mtpDlqnDeIepDOdl+nIFt9G+vfXi7dplQ7TNmiVru4Hd37TJ2hWWHWnf3gJYaq9abTI9LwolHdxivzAhDjy0OedcUWjs8F+48KB3b+vhOu00mzvWqpU9fsopNsfs9deTz5k0qWo9tvqGtjZs5yGGoQjDeIgdtGbH5uTj4UpTqDofDpKhLawzF9Zty6QHLc4rTL2OWyOVxPw2V38eeLNGRHqIyDMi8oaILBOR0dUe/4WIqIh0Sjk2RkSWi8ibInJqyvETRGRp8Nj/iNiAjoi0FJG/BscXBFtYOVdUwmHC6vXLqgvngS1YYD1Oo0bZ/U8+sflszZtbj9u991rPV9ib9tprVUMbJIc6G0apYBTH8CoX8efPi+w2qSG5hHXbWrWy8h9t2ti8u69/3ebi1XXdhaKke9xcHTx8FK4ExRT+dwP/paovi0g7YJGIPKmqr4tID6zA7fvhySJyFHA+0BvbneApEekVFLqdiO3/OR+YDQwGHgdGAptU9QgROR+4CfiP/F2ic7mX6fBfGPCefdZKaAA895ztetC3r23U3qmT7fspYkFqzx4LfNXVdzFCqkuYxA+YzLVcU6XI7jHH2PZZ1V97xQoLnOPHW4jbvh1efdV6/dq3j/+ig0x5j5tzueLBNytUda2qvhzc3wK8AXQPHr4Z+CWgKU85C3hQVXeq6gpgOXBisGfo/qr6gqoqMAUYkvKcycH96cCgsDfOuWIRDv/VFVjCHQRuucWGRSG5WvPoo20e24YNNhSpaqENGhfSquvHS/wP6YvsLllic9vAFh+A7Xk6ahQMGWJtv/765I4Oqdedaa9jnJVsj1us57clom4AHjpcPnUSkYUpn08KNljfRzCE2RdYICJnAqtV9ZVqGas71qMWWhUc2xXcr348fE4lgKruFpHNQEeg2hbXzhW/1PldX/yi9bRt2mSBbdQo+O1vG7bIIFMd2cB0zmEtySK71YXz11q3tuHY/v0tRI4cae0dNcpWvlYX50UHmSrZ4OZcXkS5wjRBfv4TcDDwq0Y8/y9sUNV+dZ0mIm2Bh4ArseHTq4DvpDs1zTGt5Xhtz3GuaFVWwlVX2erP229P1jmrqLCyHzNn2rw1sBWjffvC5MkwY4Yda9o02duWLbLXiux2YR1f43k20nGfc9q2tfB10EFw3HG2J6qq9a7VJc6LDjLlwa0RinZ/Uu9tczEjIs2x0Ha/qs4QkaOBnkDY23YI8LKInIj1pKUOBh0CrAmOH5LmOCnPWSUizYD2wMbcXZFz+VHbnK7U7aquvNK2pRo/3gLQ3Lm2AKFXL3t87147fvbZNr9tw4ZkrbZsOmnuZE7iSX7MnSwi+f+51B6+bdssqO3dayFywAAb1hWBrVst2JWXZ79tceHBzVXloS37vK5bowRzze4G3lDVPwKo6lKgc8o5K4F+qrpBRGYBD4jIH7HFCUcCL6rqHhHZIiIDgAXACOBPwUvMAi4GXgDOAf4vmAfnXEGrreBsWZmtCF261IJPZWVyT9Gjj4bDDoPHHrPPNwclOBYvTpb32L07u209g79x0lNTuZsfcTdWZDcMbKnDsuG/zG3bbB7b/PkWPKdNs3BaDAsQauOLE+ImEXUDXFFJRN2ArPgqMBz4pogsCT5Or+lkVV0GTANeB+YAZcGKUoDLgbuwBQvvYCtKwYJhRxFZDvwcKOL/r7tSEi40SDenq0cPG/p8+WX7vH9/m/g/YADccIPtiLBjR9XnpJs3lg2H8Q5TGMG67kdyBbd9frymeXQ9e9p1zZhhQ7jz58ONN9pjxbAAoTYl2eMW64UJUfLettzxXrcGU9XnST8HLfWcQ6t9Pg4Yl+a8hUCfNMd3AOc2qqHOxVBNc7qqbxYfbgzfrZv1vlVUQOfOtghgv/2S9dgy2Ry+vloHRXYBZo24lh3jW9d4bvv2yftvvWXBs2dP6wkMi/EWwwKE2pRkcMuGop3f5pxzruiFvVJbtsC8eRbaOnWyeWyjR1vB3abBYs7GFdGtizIhKLJ7Bo8yqGObGs/s0sXm2E2caEO34f6oYcmSsERIMSxAqI0PlTrjvW3OOVe0wt0QwiK5Q4fakOiWLcndDg48EMaOTW5r1blz8vlNmljPVrb9hDv5AZO5nqurFNmtrls3eOQR2yx++HALmaEvftGubcSIqtdYrDy4xUkiovf10JYfUX2dE9G8rXMuPsaPtx62s86yYDNzps0La9cu2WO1bp3NcQvvh1tZgc01C3u4sqUfL/EnflqlyG7TplVXqx5/vM1hu+466wlcswamTIFHH01ucn/wwXZ9M2cW99y2kA+VOueccyVi8WKbxL9liwWiU0+FDz6Ad95Jrhxt06bqxvC5EBbZ/YCDqxTZ3bMnuWoUbDHEunXWk7ZhA1x+uS2mmDnTjvfta+dXVhb/3LZQyQU3X5hQjfe25ZcvUnDORWDMmGRPlmqyftuwYfuW9ejeHd57LzcLEQCasIf7uZCD+YCv8o/Pi+w2b171vLDo75w5ybaEq0mHDrXr2bzZ5ryJWO9hsZYASVVywS0bfGGCKzgJfMjUuRLXrp0FnsmTLbDNnGmhrUkT+Ld/g5UrLdS9/XZu2zGWazmVufyESVWK7O7aVXWY9NBDk/PqPvnEegJfe81KgIR16SorbaXp5s0116srNh7c4iIRwXt6b1s0vNfNOZdD1Ut9VFTY3LCpU5M7InTtmqyRtnevPScfJadP5zGu4Xru5kfcFRTZTZXahu9/Hx54wFaS/uMfNjT61lvWC7d5s7U5XEEaBrhiHyYFD27OOedcUQm3rdqyxXrZbrrJeqvAerV697aN2A880ILSpk3Z3wUhnZ68y5+5iJfpGxTZrX3PrJtvtgUSrVrBpElwySXw1a/aooSJEy2ohb1rxV4CJJWvKi1V3ttWehJRN8A5l0uVldbj9OyzyWNlZda7tn17cqgxtHEj/Pu/21yyFi1y27ZWfMpDDEMRhvEQO0hfZLdJkEr69rWN7/v2tVIgY8daj1uvXnDxxVbKZMiQ3LY5rkoquMV2YUIiz+/noS16/j1wzmVRZSWce671RL3+ugWb8nLriZo50wLQgQfauRs3Jp+3YIH1uO3alcvWKRWUcSyvcBF/ZiXpC8J16GAbxHfoYCtIf/ELK/47Y0ZyK65Ro2yO3vz5VhakFPlQaT35wgTnnHNxU1GRLJzbooVtWxWuruzf30LPxIn2efX6bJDb+W0/5i5+xL1cyzW1Ftndts0WIWzaZB+p+vZNbiJf6jy4ORcVX6TgnMuSsIZZWB5jyhTrmVqxwuqeHX+8ndepk80RW7YsP+3qx0vcxhVViuyGWrSA1q2T9eM++yy5qrRFC9sjddOm5Jy2MLSNGVM6CxHSKamhUocP0ZW6RNQNcM7lQjg5P5z/tWyZhZ05c2yj+AULbK7bjTfCv/6VnzaFRXbX0rVKkV2wtvzv/8IFF8Bpp0HHjlaiJNxv9Iwz4PHH7bx16+Dhh/e91lLtfSuZHjef3+ZiyXvdnHMpUkt51BZMKiuTqygHD4bf/AZuvTW5lVW4irRFC5sz1qaN9b6VldnuBLnWhD38mYv2KbIbWrsWfvITC5WdO8NHH8GRR0LLlvb4wQfbtaxdm5zb5oz3uDnnnHMxUVFR936bqQsRJk60XqsFC2w7qE8+sZCTSNgwZMeONjz6xS/ac3futNsWLWy4MVfGci2DeYKf8qcqRXZDIhbawG779oWTTrJ5bsOH24bx4bX43LaqPLjVQ8EvTPBhUgfey+tcjJWV2WrQ2nqYwoUIffvax7ZtNg/ss88syO2/v/Vgffqp9VgtW2Yf4absYOeGc8uyLSyyew8/5E5+kvaccDFE66AqyEknWa/h9u22c8PMmclr8dBWVUkMlX7IQfSKuhHO1cSHS51zgUwKyZaVWXFdVZvT9vDDyUUJYX2zyZNtu6gVK+w5lZW5brlJLbJbRgW1Fdlt1gz+9CdbPLFli+2R2qaNrYjt1q00NoxvCO9xi1Ii6gY455wrND162CT+iRNh5Ej4299sAv+oUbbicuhQe2z79vy2KyyyC3AO02sssgs2l233bli+3K5l6lSr1wYWOiEZYMvL8xc8C0Hsg5uIrBSRpSKyREQWBscOFJEnReTt4LZD1O2MPR8mjbd8f38S+X0751x2lZUlV48uW2ahZ//94de/tuHRJk0szEntu0plkTKBUfRlCRfxZ1ZwGPvtV/PZ3/2utf+kk2wu2/DhNuy7fbuFznCOXyZz/kpN7INb4BRVPU5VwxmO5cDTqnok8HTweU4V/Pw251zeich/isgyEXlNRP4iIq38P54uG3r0sCHF3r1t4cHw4dbjFs4dCwvr5iu4/YQ7+SH3cS3XMJvvAhYkU7VubVtWDR9u8+3mz4fLLrOg1r07PPIIHHSQXUc4RJrJnL9SUyjBrbqzgKAzlcnAkOiaUgC8t60w+PepqIhId+BnQD9V7QM0Bc4ngv94uuI0ebL1tlVWwje+AVdeCWvWQPPmyXPCAJdL/XiJP/HTKkV2e/asurVWp062WOKtt2y/1Isvth62tWvtdtQoC6Nf+IL1soEFNijtmm3pFEJwU2CuiCwSkUuCY11UdS1AcNs5stY1VCLqBjjn8qAZ0FpEmgFtgDX4fzxdA4WbyI8YYcVqH3zQjm/fbjXRZsyAjz/et5ctl71uNRXZbd7c6rOBzWE7+WTrHQRYvNgWVAwYYJ+fdJLdlpcn90z1IdKaFcKq0q+q6hoR6Qw8KSIZ1XwOQt4lAK2+0CmX7XOuMCXw/0DkkKquFpHfA+8DnwJzVXWuiFT5j2fwu825Oo0fn9xvNJVI1b1GDzjAVpmGNdtytQ9pE/ZwPxemLbL71lvJ87ZssVA5fLh9vmsXrF4N55xjIW7EiGRQO/poOyfcwsuHSPcV++CmqmuC2/UiMhM4EVgnIl2DX3pdgfVpnjcJmATQvt8ROdw+1znn9hXMXTsL6Al8DPyviFxUj+d//p/PLl26MG/evIyet3Xr1ozPLXSFdK27diV3CUgdyszE1q1befLJefTpA7//PTRtarsfNGtm5TN27LC6bPn2lTn3cNJTc5l7zn/x/QFb+T7zajz3gAPsuo89Nnls5UoLb8uXw6BBFtr239+ude1a+NKXbHuud97J9ZXkX2N+dmMd3ERkP6CJqm4J7n8HuA6YBVwM3BjcPhJdK53LIq/pVky+BaxQ1Q8BRGQG8BUy+I8nVP3PZ79+/XTgwIEZvem8efPI9NxCV0jXWl5uPUrl5XXXaQuF218NGjSPp58eyE032fFhw+Ddd6FPH1i0CF5/PWfNrtHpPMZ/MZV7+CEjp/8Optc+HjtihG0a//TT1hvXqRNs2GD12mbMgP7Br7158+YxZ07yWuvz9SokjfnZjXVwA7oAM8UG6JsBD6jqHBF5CZgmIiOxYYhzI2xjvPmEd+ei8j4wQETaYEOlg4CFwDb8P54lpyFDf+Hw4aGHWm22LVvs+JYtNsS4eHFut62qSX2K7IYWLbKFFH372uennWY7JMyfb/Pd+qf8fzUsMAw+VJpOrIObqr4LHJvm+EfYL0HnnIslVV0gItOBl4HdwGKsB60t/h/PklPXjgjpNpcvK4Nnn7UtrR5+OLnactiw5PNytW1VTeoqshsO4/bubStEn3nGhnKPPx6+9z3b1eHhh5OBbMKEfcNZjx7Ja3X7inVwc865QqaqY4Gx1Q7vxP/j6aoJe9dEkgGvRw/bYP255+C88+xYZSU8/7zdD0NS/igVlNGXJXyXR1nBYZ8/0qmTtWXTJqvR9vjj1v7KymQ4CwNpau9aMQ6D5poHtwxkvfhuIrsv55xzrrBVH0pN7YHr3DkZcLZutUUOLVvaqtH27a3XrXnzZCmNXPkxd/Ej7uU6rv68yG5owwYr+wHwta8lQ1qPHnZN1XsTXcN5cHPOOeciljqUWlkJ554LCxbAlCnwm98ky4C0bGm3YamPcC7Yrl22M8Gnn+amfSewkNu4gif4Dtfu04lctS1dulQ9nq430TVcIRTgda605HNBSSJ/b+WcM5WVtW+cPn68hbaWLW1ngY0bbasoSAa2UOrOCLkKbQfyEdM5hw84mAt44PMiu9X16GGLD0aMsM/D6xw61LetyibvcXPOOefyqK4eqK1b7bZFCwtqqrBqFQwcaCtJd+yw402a2Gvkcp5bWGS3K2v5Os9XKbJb3aefWvsuvzy5K8LEid7Tlm3e41bMvBSIKwIi0kNEnhGRN4IN20cHx2vcrF1ExojIchF5U0ROTTl+gogsDR77HwlqDYlISxH5a3B8gYgcmvcLdSWjrMx6nzZvtl6p1B64BQvgiSfsvB07ks/Zvh3eeKPqjgh79+Z+ccI1XMdgnuCn/IlFTb6c9pyuXe02rMu2eLEFtq1bvactFzy4OefibjfwX6r6JWAAUCYiR1HDZu3BY+cDvYHBwAQRCcd2JmK7ERwZfAwOjo8ENqnqEcDNwE35uDBXmnr0sIn8EyfaisuwB+6qq2wHgfXrbVeEcLGBiK3aDGug5cvpPMZYruNefsCd/CTthvVNm8K3vmX3+/a1YrphO9u18w3ic8GDm3Mu1lR1raq+HNzfArwBdKfmzdrPAh5U1Z2qugJYDpwY7FKwv6q+oKoKTKn2nPC1pgODwt4453Jh6FAbThwyxHrgysth6VKr2da2Ldx+u9VCa93ahko3bLCdB/IlLLK7mOMYxQRqKrIbbiTft68F0f797XbAADj11Nrn8rmG8eDmnItaJxFZmPJxSU0nBkOYfYEFQJXN2oFws/buQOqfilXBse7B/erHqzxHVXcDm6GWyTzONUJlJYwebbsGXHklrFlj4eyKK2xF5te+ZoV3jz8evvEN63EDmDUrP+1LLbI7jIfYQWua1JAW2rWDqVNteLSiwgJbRYVdWyJhPYk33pifdpcKX5zgXBwV0J6lH7fen1nHDmjEK8zdoKr96jpLRNoCDwFXquontXSIpXtAazle23Ocy7qKCpvL1q1bMrzNn29zxdatgzlzqp4fDkXmuk6bSRbZPYO/sYLD6NDBNolfscLOaNPGgmb//hY6wfZNXboUliyxDe/Ly2H1agt0W7bY52Edt3S7RLjMeY+bcy72RKQ5FtruV9UZweF1wfAn1TZrXwWk/jk4BFgTHD8kzfEqzxGRZkB7YGP2r8SVitpKfoSLEwYMsOHQbt3sdu1a6NkTOnTY9zn5EhbZvZ7/x2OcAcApp9hK0VDPnvDmm7aK9K237NjJJ9vw7oABNlQ6fjyMG2dfA6ja8xbO6ZswIY8XVkS8xy3fElE3wLnCEsw1uxt4Q1X/mPLQLNJv1j4LeEBE/gh0wxYhvKiqe0Rki4gMwIZaRwB/qvZaLwDnAP8XzINzrkFqK/mxZg3MnGlBDapuvv7BB7mrx1aX1CK7iZQ/VjNmwD/+YffbtIG77072Gvbta71tW7bA5Mm2RVfqrgnjx++7qrT6LhGufjy4FSsvBeIylSDu/6H4KjAcWCoiS4Jjv8YC2z6btavqMhGZBryOrUgtU9WwaMLlwH1Aa+Dx4AMsGE4VkeVYT9v5Ob4mV+RqCyejR1to69LFJvC3bWtFay+/3IYWo1Bbkd3mzS2gzZkDgwcnQ1uvXhba2rZN7uzQvv2+QfXii+26wsK8qbtEuPrz4FaHrO9T6pyrF1V9npqWtNWwWbuqjgPGpTm+EOiT5vgOguDnXDbUFk5uvdXmtY0ZA//8J3zlKxbmeva0gNO5s5UEyZfUIrtfS1Nkd9cuW9EaFtWdOjX52FtvWTgNA2q6vVZnzrQ5fA8/XHWDedcwHtycc865Bqhrkn1Nj/fvDy+8YPO/brrJFiWsXWvz3Hr3ti2u8ikssvsTJrGQ9EV2Z8+2Idzeve3z1q3hxBOtTMm779oQ7+23J68zdajYh0azy4Obc3FVQCtLnStFdW1dVdfjYaA56SR7vFs3m0+WT6cxm7Fcxz38kLv48efHu3Sx+WzNm8OHH8KmTXb8i1+0YLl2LSxcaHXnli2zx6680gJp6rWNGuVDo9nmwc0555xrgLp6kup6PAw0s2bB8uW2pVV1uSwDfSgr+DMX8TJ9KaOC1BkJn3xipUnA5rJt2mS3XbpYaNtvPwttbdpYrbkPPoBbbtn32lz2eTkQ55xzrh7CUh+w75ZOqWVAUsNLeKyy0oLciBEwbJgNPZ5/vg05bt5sPVypcrW2OSyyKyjnMJ0dtK7yeLiytX1763ELj02bZu1+4AHrIdy+HY49Fl5+2eev5Yv3uDnnnHP1kG4ItLLS7r/wghWhTX0sPH/LFli0yFZk1qRVq3xsHm9Fdo9nMd/lUVZwGGD7ju7dWzUs7tiR3NQ+rEk3e7YtoJgxwxYc+Ny1/PLgVoy8FIhzzuVMuiHQiopkSYwBA6qurvzkE/tc1UJbulWjXbva6s1du3Id2pJFdq/jambz3c+Pp3vfnTttg/sDD4SOHeHFF63nbeJE+xq0a7fvc3xnhNzyoVLnnHOuHsIh0NRQMnSo1TobPtzmelVUJAPMxImw//5Wz6xvX9uDNNwdIRwaXbvWXm/z5ty2PbXI7rWMrfG8sF0dOsAhh1jZj5UrLdz17ZsMoul2QPCdEXLLe9zyKRF1AwrTfmznbsYxkqvYRpuom1OcEvjPp3ONMHOm1WA79VTbQWDiRBsaHTHCNow/6SSr1ZZaYLd37+SWUWDFbF9/PTk0mW2pRXYv5P4qRXZThfuJgg2dLllivW5r11pvYrg7QmWlzYHznRHyy3vcXOwNYiH/wdN8k4VRNyX/fNjbuYJQVmYLEFLDygsvWIibPx9+8hMbJm3Z0h5r0cK2vgo3jm/RAp5/PnfDpKlFds9hOh/RKe15vXolQ1uXLlV7AEeNSr+lVfXh0JqOu+zw4OZibyjzUGAoz0bdFOecS6tHDws2FRW2LVTXrta79txz1lsVzmn77LPkbVgbTcQ+X7ECdu/OTfvCIrs/5U81FtlNXUHaqpVtEj9ihAW4DRtsuNfDWPQ8uLmYU87geQT4Hs8Dvu+3cy5aqSU/UoVzu8aPt2HFbt2sOO2GDclzwhWbTZrseyxXTmM2V3M99/ID7uQnNZ7XvHkyTO7YYdfTtSs88si+vYk1fQ1c7vkcNxdrR7GCVth/UVuxky+xkjfoGXGrnHOlrKYdEcK5XUOGwJQpNsdtwQKbx9asWdXetL17079206bZHS4Ni+y+wrGMYgI1bfvbrJnVk7v9dmtnly62d2pYxqSiour5de0K4XLHg1stfIP56J3OP2mK/YZryl5O5x8e3Jxzkapp8n1qwd3Jk20z9vbt6/fa2QxtqUV2h/HQPkV2U+3eDXfeabf77We9bL/9rT32wQf7nu8LEKLjQ6XFpsgms5/HU7QOetxa8xnn8XTELXLOlbr6TL4PJ/eHixLy6Tau4HgWM5ypnxfZrU2rVjY0um2bFdZ99107vmLFvuf6AoToeHBzkZpOOcqAGj+O4Z0q5x/L8lrPn055RFeSQ0UWxp0rZgsWWMmME06wQruhcPVoLvceTTWSuxjJPVzP/+Mxzqjz/NatYdAgK2syYgTMmQNXXGHXEhYWdvHgQ6UuUuWM4jBWcySVtGXHPo+3ZFetn4e20oq3+ALleL+9cy436toRoLISzjzTVpAuXWr7eLZsaStGw9WkuV6IAFWL7CbqKNDYqpUtRPj3f7ctrHr1gjfftNptV19tQdR71eLFe9xcpJbzBfpxH2O5hG20ZHc9fyR304RttOQaLqEf97GcL+SopSUgEXUDnIu3mnYECDeOD0MbWGgTsWK6qWGtdc3TzLIiLLK7ji61FtkFC5W33WarQydOTK4cvfVWGzJdu9Z3P4gj73FzkdtLU/7IBczia0zjqhp736oLe9n+gxs8sDnnsqp671q45+jw4TZvLSyDUVEBb79tvVVg20GJwMsv79u71ry5haVPP81Nm1OL7H6N59MW2R0+PLnSdedOuPlmePxxeyxsb//+ds6ECb74II48uLnYCHvffsUUrubezxclpPMpLfgNF3MjF6PecZwdCeCZqBvhXDxUL3cR7jk6YIDthNC+fXKvzi5d7Dlt2tg5X/ta+tfctQs+/jh3bQ6L7F7K7TUW2X35ZQtm4VZby5ZZQNu82dq+dq31tg0dmp9hXVd/HtxcrOylKcs4nM9oXmtw+4zmvMbhHtqcczlRvdxFao22hx9OHt+yxVZfPvusDY9OmGDBZ9UqK7LbsqUdz7XTmM1YruNefsAkLqnxvGXLbKutnj2tB7BPHwttW7fa40uXWg26Z5+1gOp12uLHg5uLnaHMox21/6Zrx3aG8ix/4+Q8tco5V2pSe5xSa7T1758cSn33XVuBCRbYFi5MDqPu2ZOf0BYW2V3McbUW2W3Z0oZHN22yjwEDrLdw4kQLouXlyWBaPaC6+PDg5mLGtrhqkrK11W6a8BnNacEumgXFeJugKVtg5Wl9fVRO6R91C5wrObXtDBCuHl2ypOrWVWvXWs9Vtnc/qE1dRXbbtLHw2LOnzb9bvNiGaw84wHrUjjwyuSghXD3av3/VWxcvPs7kYuUoVlQZIt1KK17lCM7it7zKEWyl1eePtQ62wHIurkTkABGZLiL/EpE3ROQkETlQRJ4UkbeD2w5Rt9Ptq6xs3/05Q+PHW2gD27qqacrCzS1bLLTlq15bXUV2e/a0a2jd2hZQrFhhvW0HHGCPv/Za1dDm4s+DW7Ep8N4Z2+Jqzz5lPp6iP1/m3iplQ5oEW2A5F2O3AnNU9d+BY4E3gHLgaVU9Eng6+NzFTOrQ6KhR9rFggYW5deuqnptaaDfUqtW+x7KtriK7LVvC8cfbkO/rr9uxTp1sZenEickeuBtvzH1bXfZ4cKvFaSfPiLoJJec8nqI5e3iVIziOqdzMBZ8vQAjLhhzHVJZyOC3Y7VtgudgSkf2Bk4G7AVT1M1X9GDgLmBycNhkYEkX7XGbC1aQTJ8KVV9rwabgFVNjTtnHjvs+raRP5bDmeRbUW2W3WDE45xfZLXb/eCuu2agUbNsBTT9k5Awbkto0uN3yOm4uVD+jIf3MFt3B+jStGw7IhV/JXBrIozy10LmOHAR8C94rIscAiYDTQRVXXAqjqWhFJ018DInIJ2PLALl26MG/evIzedOvWrRmfW+iyca27dlmw6dzZVlmmHjvgADjqKLj3Xhv+3LkTfvxjC2Vbt9pt8+Y2LPpZzYvgG+2QQ7by+9/P+/zzVts2c9Etl/KZtued/xzFb/f7e5XzRayXTQS+9S3reTvppKqvuXgxfPe7cMwx1rsYlx+ZUvn5bcx1enBzsXImf8jovLD37Y9ckOMWOddgzYDjgZ+q6gIRuZV6DIuq6iRgEkC/fv104MCBGT1v3rx5ZHpuocvGtZaXWy9aeXlyaDQ8FtZsGzDAtoVassRWYYZDpU2bwkEHwQcfNKoJdfr97+fxi18MBKzI7qOcQSs28TWeZ+FYq9fWrp2tagWr0dasGezebZ936GDz2lq3tuK/3brZfLeZM/e99qiVys9vY67Tg5tzzuXGKmCVqi4IPp+OBbd1ItI16G3rCqyPrIVun3ptlZVW56xvX+tde+89C2+9elkASi3vsWdP7kNbdVdzPacxZ58iu1u22Ee7dvb57t0W1Hr0sFptM2bAYYfBN75hm8jPnGlFdlOv3RUGD27OOZcDqvqBiFSKyBdV9U1gEPB68HExcGNw+0iEzXRUrddWUWHzwgDuustKfHTunNxpIBQWsH333WTPFkDbtslittl2GrNJcC33cXGNRXa3bLHbsAzIt75l1xcuRPje95I9bV5ctzB5cHPOudz5KXC/iLQA3gV+iC0KmyYiI4H3gXMjbF/Jq16vrawsGX5GjLDHH300eX5YxBZsHlxqaIPchbawyO4Sjq21yC7YcO6kSfDCC8mtrEaNglNPtds1a2xnhCFDctNWl1sNDm4i8itVvSmbjXHOVVPg5V1KnaouAfqleWhQnptS8iork71LY8Yk65ZVHyrt0cPCWmj0aJsfBtbL9vHHFtzClaX50GzXTh5iGE3YyzAe4lPafN7W1q2tNzCcv9a7N5wcbCjzzDPwk5/YHL0RI5I7PowebaVNHn7Yi+wWooyDm4hMS/0UOA6ILLiJyGCsRlJT4C5VjX8lmkTw4VzcJKJuQO1E5B7gDGC9qvZJOf5T4ApgN/CYqv4yOD4GGAnsAX6mqk8Ex08A7gNaA7OB0aqqItISmAKcAHwE/IeqrszP1bl8CMt6gG0QH4a41HptkNzKqqzMeqY2boQWLWzV6Jo1yd62kIgNS27blru2f3PmrRzNYs7gb7zL4Z8f37DBwtphh9lCiRYtrG7bxIk2p23dOli50m6nTLEh0k8+sdA2YIDPbStU9elx+0RVfxx+IiITc9CejIhIU6AC+DY2AfglEZmlqq9H1SbnXE7dB9yGhSsAROQUrCbaMaq6MyyrISJHAecDvYFuwFMi0ktV9wATsRIb87HgNhh4HAt5m1T1CBE5H/tP6X/k6dpcHqQOgdYWWMaPt+CzZg387W/WwxYKQ1vqik3V3Ia2kdzF0S8+zg1ctU+R3U8/tduVK5N1477xDVsl+tZbFt6++lU4+GAbGl22zHreqm9x5QpLnQV4RSSs/zyu2kNXZb85GTsRWK6q76rqZ8CD2C9w51wRUtXngOplTi8HblTVncE54erMs4AHVXWnqq4AlgMnBis491fVF1RVsRA4JOU5W4Ptp6YDg0TytWmRy4dwCLSiIrPA8tprVUMb2KpSyN8+pGGR3feOPIGxXFvjeXv3WnHdXr1sHpsq/PKXFtBuucVWmi5bZueqVl2M4QpPJj1uL4nIXOx/qp9T1TS1ovOmO1CZ8vkqwEfqQ6f0h2cW1H2ec4WtF/B1ERkH7AB+oaovYb8f5qectyo4tiu4X/04we2/gJeAl4HPgI7AhlxegIufMWNsKHXIEEgkbJeB3btt7lhYvy0fwedAPuIhhrGezjx20dXsHWvbNDRvbitXN21KDuG2b2+LEN56y3oM58+vumI0tbdR1VeUFrpMgtuxwHeBm0WkCRbgHgv+xxqVdP8TrtKe1Krjrb7QKR9tcq4kfchB3M6ljXiFuZ1EZGHKgUlB8dm6NAM6AAOAL2MrNQ+j5t8Ptf3eEOB3wJXAd4DTgBdF5EHgblV9J5MrcYUldT5b2AsXznlbsABeecVCW5cuVq/to4+gSZPcb2fVhD3cz4V0ZS1f5+/8x37J4nG7dsEZZ1iP4LZtFtbCHR969oS334azz646HJy64KKy0oKez28rXJnsVdoeWAZcCzwE/BbI43qatFYBqZ3dhwBrUk9Q1Umq2k9V+7U4qH1eG+ecq5cN4b/V4COT0Ab2e2CGmheBvUAnav79sCq4X/14+Fo9gv+QfoiFws+wYDhdRH7bwGtzMRaWApkwwQJNebndgq28XLsW9tvPeto++sgCXPfutb9mNlzDdQzmCUZzKy9xYpXHwuHaxYsttHXtaosU+vZNtnP+/JqHg8Ng6vPbClcmwe0jYCpwHjacMAm4LpeNysBLwJEi0jOoj3Q+MCviNjnn8uth4JsAItILaIENbc4CzheRliLSEzgSeDHYH3SLiAwI5q+NIFn8dhbwOxFZBNwLLAaOVtXLsZWmw/J3WS6XUgNaWZndHzIEzj3XQtx559ljv/61bQ01dqz1ZLVsaSU3KivrfItGOY3ZjOU6JjOCO4Ke7HD+Glhv24cf2v2ePW2laHm5rRLdvt1WuB5zjPWo5bqtLhqZDJX2w4pIHg3cBcxU1Rx3FNdOVXeLyBXAE1g5kHtUdVmUbXLO5Y6I/AUYCHQSkVXAWOAe4B4ReQ3rHbs46DFbFpQveh0rE1IWrCgFW9BwH1YO5PHgA+BuoAxog21BdZGq7gJQ1b0iUnU5nytY1Qvujh9vwWfBAutRmz8fbrzRJvSvWWMrM8OabStX5rZtqUV2y2QiqNCpk+2TOmBAcgi0X1AZcPt2q8MW1mcL57qlK3viikedwU1VXwZ+KCIHAj8BnhOR2ar6m5y3rvZ2zcaW8zvnipyqfr+Ghy6q4fxx7LsSHlVdCPRJc3wHcFQt7/9GZi11cRcW3B0yJDnP6+KLbfL+s8/acOMLL9h5XbrAAQfYsGKue69a8WmVIrvb1Irs7tpljy9aZO2tqIDf/AauuSYZ0CA5BFpZadcHPo+tWNUZ3ERkHtAW+5+oYPNIzgEiDW7OFT3fNcG5rOvRwwLNuedaLxvYfLG+fa1kxn772edXX20hbs4cG36sLnXrq2y4jSs4Piiyu7b14RDUaNu82W6XLYPLL7e2lZfD6tU1X1/qzg+u+GQyVPoD4GNgc8QrSZ1zzrlGC1eNduhgOw7Mn287DgwYYPd797aernbtLCBt22a9WKl/AbMZ2kZyFyO5J1lk91MLkn2CvuHWre22T5/kfqOudNW5OEFVV6rqxx7anHPOFZrqq0VTbdpkKzJbtoTHH7fFCMOH2+T/t96CAw+sukNCLoRFdue1+DavDL2Ws8+24Nizp9VrGzcO/u3fLMhB1R0Pars2V7wavMl8qTjt5Bk8/tzZUTfDOedcA1RfjAA2p23+/GQdtHDT+BUrrLzG+vW2knPjRjjuuOSQajZ16gR7NmzkIYaxji5cdegD/HNmUwYMsGHRcKeD9u2tp23xYvvo3j15HemuzRU/D27FyndPcJlKRN0A53InXIwwapQFsMsus8D29tvWu9aihRWzBRs2XbvWzt+xw0KdSMMXJzRtWvP2WO3b7uFPG6zI7td4noVvdaJvX9uiasoU2LrVevk2b7YFEiNGwNKltqgi3bW50pFJHTfnnHOuIKUWnB09GpYssdAGNoftrrtsGLJjx2TISh0WVW1YaOvQofY9TS/78HpOYw6/bPk/LOTLAJx0kpX2qKiAyZNt6HbiRHv/cNHEww+nvzZXOjy45Vsi6gY451xpuvVWK/EBFtaOPx4GDbJA1LGj9ZClqv55fWzalH41KsBgHufn267jPi7mriaXfN6ecL5aaoHgAQOSiyPKyzPrXfO5b8XNh0qdc86VhP794aWXbIurUaPs823boFkzGxYF69nats3u19ZjVpfmzfcNfiLwBV3J/VzIus7H8J8fT2D7p0KvXvBIsIdHWKYknLc2bRo895zt6JBpz5rPfStu3uPmnHOuYDSkNyl8zoIFFmrClZm33249cIcEO9h26QI//GF22tmihRX1DTVrBi10BzNkGC2b7+UPX3mIjz+zLrmNG5P11xYssF62sGetRw9bkFCf4dBwKy+f+1acvMfNuTjy4rvOpZXam3TqqenPqay088rK7POwF2vuXBsW3bLFgs2cObaKdMkSq5W2bl3V3Qgy1bWrLWpI9emnVT/fvRsmcgXH68ucsetvvP364Z8/Fm5hlbrYoDHz1sK5b644eXBzzjlXMFLDzTvvpD8nDHfPPmtzx8JerG7dLLht3WrnhCGtQwebkwbwpS8lV5mmk27HhA8+qPp5uJo0tWjvj7ibH3M3v291FY/tOIMOHybPP+wwu/XA5TLhQ6XOOecKRmq4Wb06/ZBpOKk/rNXWty8ceaTNXwMrbHvkkTZ8WV041y3Uq5eFtXS6drUFCCNGQJOUv6apq1M7doS+vEwFZczl27x2zrV07ZoMiuGiBOcy5cHNOedcQamstOHPDz6whQapx8MQNG2a9cq9+qr1sk2dauU/ysvt4+qrbfiyTRs46CB7TsuW8Nln+75ft25226KF9baFiw727oXt223Xhb17qz6ndWsrsvuVL37E31oMY2+nzrz4s/tp064pa9daYBs1ynr9Kir2DaC+MtTVxINbMfN5Us65IhRO4t9vv6oT8MMh0gkTrGeuXTubvxaGpBEjkkOXt99ugezkk62XrUmT9PuPvvWW7agAtlIUrEetbVubE9emje20UN2nn8LGDXu4/J8XcdCuNVzbZzp/eeog1q1LBrZ27axe20032arR1JCWei3OpfI5bs7FTT4DdyJ/b+VctoTz3A4/vOok/rIyW3iwebOFoKFDbZ7bLbdY6Y/ycgtDW7ZYgBs0KLmdVfUes1RNmtjjYZkQgBNOgFdegY8/Th7r3Rvefz+5mvRqrMjuZTqRO+adCMDrr1s7Zs60towYYUOu8+dbSAuHgTOZy+dKkwe3DGR9v9IE/gfTOecaKJznNm/evsfbtbNAtHix9bTNn2+7DfTvnwx2zz6b3Au0Jl26WI8apA91zz6bvN+kCRxxBHzxi8nX/W6Tx7lm73XM6TKCF7teCktsEUT37rZtVbduFsw2b7YVqaklQFKvEeoObqmraH0XheLnQ6XFzodLnXMlZOjQZA/Wtm0WiIYMSYYb1WS46tTJbnv2tPupoWfz5qqvu99+NgfuC19IHuva1cLX3r1Vh1QPZQX3y4Us5WjmnDmRR2YJ5eVw/vm2YvXhh+29wqA2apTNyWto6PJh1dLiPW7OxYkHbecaZeZM68Hq3Rseeww++siCzdKlNt9txAjbXH7xYlsxevDB8O67VtOtf3/45BPbYP7YY5M7GKgmh0mbNrWFB59+Crt22WNNm1o4/O//hjv/tIP/mnEObZrsZe5FM/ivq9p83ntWWQnt2ycDW1iSpLy8cT1lvtl8afEeN+dKVSLqBjiXPeEqzK98xXrZwEIbwMsvW2gDC1pvvWU9XzNm2LHnnrPbF1+0nradO2116YABycUMHTrY7Zo1Ftq6dEkWzt2zx0qTdOsGly69gi99+jJ3fWMqHx2QLLIL+24Kn60dDnyz+dLiPW5RSeB/OF1V3tvmXIOE5UEWLEjuYjB8uJXvOPro5PBo797Jnre+fa3w7fTpVtID7Lz27W0LrIkTLYjdeKM9NmIE/PjHyeK8X/pSchVop042NPvipXcz7NW7eaTPVSQWfo/1s2vfL9QL7rqG8OBWCk7pD88siLoVzjmXE2F5kG7drEesa1frzeof/F9o1ix46ikLSnPmWO/Zzp3wxhtWyiMc+gRo1crqsoW9V+FrX3ZZ1XIdL76YDHynnw5f2r6IM2eWsePr3+amndeyfr21x4cvXbZ5cMtQ1leWOueca7DKyuTOCeEcryFD4MorkytJAUaPtpWka9cmS3ds2mQfhx1mIe/YYy3QhfuV3nhjsihuRQU88URyqBXsvO3brdfupJPg15dtpPXXz2Htns5M7vsAN1/QlCuvtDIkPnzpss2Dm3NxkO9h0kR+3865bKuosB0Pwtpn4ZDjtGl27Igj4Gtfs90RevWCzp2td61TJysZsmKFBbB337WVqOXlFgSnTq36HmGttTVr7Pm9e8Pxxyd3YejRfS+ccRH66Rr+MuLv/OAXnejRA154IfNr8XIerj58cUKUEnl8L58/5ZwrImVltiK0+lBkOG8s3NIKbMh02DC7v2GD9ZR17QoXX2z3t2yx3jqwz0eMSL5HeTnccAMsXGjv1by5hbv99w9C1vXXw+OPI7feStnkExsUvLych6sP73FzzjlXcHr0sGK2qUEptefq+uvhZz+zLa3GjbMes/nzbTXonDk21PmHP9jQ6OLFNgdu7Vp7nSlTbH5a9V6w1C20Nm+GD6fO4aBrr7Wkd+mlDb4WL+fh6qMkgttBfBh1E5yrmQ+TFjURaQosBFar6hkiciDwV+BQYCVwnqpuiq6FhWvXLusRKyuzz8OVpVu2wKJFFs6OO86CV0WFBbSuXe1427bw1a9aSZDWrS20tWxpixY++CD5WqmrQsOAtXkzzJ64kt/fe4EtW5040R5oIF9d6urDh0rr4bSTZ0TdhMbx4VLnojAaeCPl83LgaVU9Eng6+NzVU2WlbQUVDjGOH29Bq29fK+uxYEFy14TycpvHNmqULSbo3ds+/+Uv7ZxwRWm4yfyKFcnnp9uGasx/7uD5g8+hVYu9rLltBuXXtamy4tS5XCqJHjeAy7iD22l4V7ZzOeFhuqiJyCHAd4FxwM+Dw2cBA4P7k4F5wK/y3bZCUVmZ7I0aM6ZqmY6DDkqGq9Qeq8GDrXftlluSm7mL2FDnhAn2nKlTbah12jRbRbp1qwW+du1s5PPhh+11081Z6/Hbn8IHi+CRR/ifxw7//PW918zlQ8kEt9hK4ENXzhWvW4BfAu1SjnVR1bUAqrpWRDpH0bBCEW4LBVYcN3XY8plnrIcNLNQtXmzz2MaPT5YECTeW37zZFiOEZUNSg1lFxb7v27+m/1Pdcw/cdRf8+tdw5pmU9fX5aS6/PLiVGi/GW9oSUTegdIjIGcB6VV0kIgMb8PxLgEsAunTpwrx58zJ63tatWzM+txAMGgRHHWX3u3aF1Etr23Yrhx8+j+ees96z666zkh0HHGA12zp3tuHUr3zF5q29+Sb06WP7kZ56qj32zjvp33fXLnutzp1tJSlAq9ff4stXXsHHfU9g6Te/+Xlj6nqtxiq272ltSuVaG3OdHtyci4oPkxa7rwJnisjpQCtgfxH5M7BORLoGvW1dgfXpnqyqk4BJAP369dOBAwdm9Kbz5s0j03MLTeqqUYD335/Hu+8O5Lzzaq9/VllpQ6TvvJPc1D3c9L2m+mnl5TbEGp7Lxo1sHPJDVu86mIrjHkeePChvddeK+XtaXalca2Ou04NbPfkOCs65TKjqGGAMQNDj9gtVvUhEfgdcDNwY3D4SVRsLTVjvTMTmox10ULKeWhjChg6FyZPt/HBOXLiooLLShlvDYc3U16s+P61KiY69e+Gii+iwfTUPjvg7W1odxESf1+Yi4sEtDhLkvxivD5e6AiIi9wDh0GOf4NjvgO8BnwHvAD9U1Y+Dx8YAI4E9wM9U9Yng+AnAfUBrYDYwWlVVRFoCU4ATgI+A/1DVlTm6nBuBaSIyEngfODdH71N0qtc7e+45OO88ux+GsClTkvXYUufEwb5lN2qrn1bl3GuDIrsTJjDq8v77BEDn8qmkyoFcxh1RN8E5E8UwaSL/b5lF9wGDqx17EuijqscAb5Hs3ToKOB/oHTxnQlBLDWAiNm/syOAjfM2RwCZVPQK4Gbgpm41X1XmqekZw/yNVHaSqRwa3G7P5XsUsDFNhL1pqAd6yMlstunatLVgYNWrfYFVZaUOfYemO1Ner0eOPw7XXwvDhttN8QDW71+ZcpkoquLkUPr/KFRBVfQ7YWO3YXFUNNjViPnBIcP8s4EFV3amqK4DlwInBfLL9VfUFVVWsh21IynOCATamA4NEGlFR1eVdjx5W2qO8HB55xHrgqgeyem8ttWIFXHihFdm9/fbPi+z6FlUuSj5U6ly+eWjOhR9huxEAdMeCXGhVcGxXcL/68fA5lQCqultENgMdgQ05bLNrpNSdE1LnstWkXltL7dgB55xj89tmzIA2bRr2Os5lmQe3uEiQ/6Esn+vmsuCTrQc0dsFOJxFZmPL5pGBFZUZE5CpgN3B/eCjNaVrL8dqe42Js/fqqiwtqWyUK9dxa6oor4OWXYdYsOPzwhr+Oc1nmQ6UNUPBbX7nSk4i6AbXaoKr9Uj7qE9ouxhYtXBgMf4L1pKX+2T4EWBMcPyTN8SrPEZFmQHuqDc26/Ko+Hy2dzp3tnOqrRBs9hHn33fbx61/D977XyBdzLrtKLrj5AoVqfNguv/zrnTUiMhjbKupMVd2e8tAs4HwRaSkiPbFFCC8GuxVsEZEBwfy1ESRLcczCSnMAnAP8X0oQdBHINISlfpfKyqoGudrUGAwXLbIX+ta3rKKvczHjQ6XOudgTkb9g+3t2EpFVwFhsFWlL4MlgHcF8Vb1MVZeJyDTgdWwItUxV9wQvdTnJciCPBx8AdwNTRWQ51tN2fj6uy9Usk3lk1YdK6zOEmbaG28aNNq+tc2d44AFo2rTW13AuCh7c4iRBNENaPtctP6LqbUtE87bZpKrfT3P47lrOH4dt7F79+EKgT5rjO/B6arGSSQirPlRaH/sEw6DILqtXw9//btV9nYshD27OOecKUvPmDV8ksE8wvN6K7DJhQi07zDsXvZKb45YtRbdAwedeOedK1Zw5aYvsOhdHHtycywcPxs5Fos7VqStXwgUX7FNk17m4Ksng5itLa+Dhovgkom6Ac/WTSRmQ+qh1dWpqkd2HHqpSZNe5uPI5bnGTwP/YOudKVtrVno1Q6+rUn/7Uyn888ggccUTj38y5PIhlj5uIJERktYgsCT5OT3lsjIgsF5E3ReTUKNtZlLzXLfv8a+pcxmqrxdaQ3rgaN5K/5x646y4YMwbOPLNRbXYun2IZ3AI3q+pxwcdsABE5Cquv1BsYDEwQkcgK7RTdAgXnnItYjUGLLO6M8PLLlgwHDbLVpM4VkEIbKj0LeFBVdwIrgmKZJwIvRNusIuN13bInyt62RHRv7VwuZGVz940bYdgwq9P2l794kV1XcOLc43aFiLwqIveISIfgWHcgtZN8VXBsHyJyiYgsFJGFn3z42T6Px3qBQiLqBuDDe8652KmtNy4je/dayY/Vq2H6dC+y6wpSZMFNRJ4SkdfSfJwFTAQOB44D1gJ/CJ+W5qXS7ieoqpPCTav3P6hFLi6h+Hl4axz/+jkXLzfcALNnw623epFdV7AiC26q+i1V7ZPm4xFVXaeqe1R1L3AnNhwK1sOW+n+tQ4A1+W57qpzNc0vk5mXrzcNHw0T9dUtE+/bO1Ue2S4CkNWcOJBJeZNcVvFgOlYpI15RPhwKvBfdnAeeLSEsR6QkcCbzY0PeJ9XCpK1xRhzbnCkzWFh3U5L334MILvciuKwpxXZzwWxE5DhsGXQlcCqCqy0RkGvA6sBsoU9U9UTWyZPhihczFIbQlom6Ac/WTlUUHNdmxwxYj7NnjRXZdUYhlj5uqDlfVo1X1GFU9U1XXpjw2TlUPV9UvqurjUbYzVPTDpRCPQBJ3/jVyrkEaveigNj/7mRXZnTLFi+y6ohDL4JZPsR8uTUTdgBQeTGoWl69NIuoGOBcj99wDd94Jv/61F9l1RaPkg1u2lEwx3rgElDjxr4lz8bN4sY29futbcN11UbfGuazx4FYIElE3oBoPKuaU/vH6WiSiboBzMbFxI5x9ttVpe+ABL7LriooHNwpguDSO4hRYolDq1+9cXHmRXVfkPLhlUU6HSxO5e+kGK9XwEsfrTkTdAOdiwovsuiLnwc25+ohjaHPOGS+y60qAB7dAtoZLvdetiJXStTpXaFauhAsu8CK7ruh5cHONVwqBJs7XmIi6Ac5FbMcOOOccm9/mRXZdkfPgVmgSUTegBnEONo1VzNfmXDHwIruuhHhwy4Gc13RL5PblG6zYAk7cyn2kk4i6Ac5F7N57rcjumDFeZNeVBA9uKbwsSBYUQtjJRDFcg3NFru3bb1uR3UGD4Prro26Oc3nhwS1HSrbXLVTIAa5Q2p2IugHORWjTJnqPHQudOsFf/uJFdl3JaBZ1A1yRC0PQMwuibUemCiW0OVfKgiK7LT/8EJ5/3ovsupLiPW7VZHO4tOR73VIVQg9c3NuXKhF1A1xdRKSHiDwjIm+IyDIRGR0cP1BEnhSRt4PbDlG3teCMGwePPcbysjIvsutKjgc3l19xDXBxbJMrdLuB/1LVLwEDgDIROQooB55W1SOBp4PPXaaeeALGjoWLLmLNWWdF3Rrn8s6DW6FLRN2ABgoDXBwCUxzaUB+JqBvgMqGqa1X15eD+FuANoDtwFjA5OG0yMCSSBhai996zIrt9+sAdd3iRXVeSfI5bGpdxB7dzaVZe67STZ/D4c2dn5bWKVj7nwRVaSHNFQUQOBfoCC4AuqroWLNyJSOco21YwwiK7u3d7kV1X0jy4FYMExdELk+0AV4whLRF1A1x9iUhb4CHgSlX9RDLsJRKRS4BLALp06cK8efMyet7WrVszPreQ9Pr97+m2cCFLr7+ej1avhtWri/ZaqyuV64TSudbGXKcHtzzIS69bguL5o96QAFeMIc19TkT+E/gxoMBS4IdAG+CvwKHASuA8Vd0UnD8GGAnsAX6mqk8Ex08A7gNaA7OB0aqqOWx3cyy03a+q4WqldSLSNeht6wqsT/dcVZ0ETALo16+fDhw4MKP3nDdvHpmeWzDuvRceewzKyzn6//2/zw8X5bWmUSrXCaVzrY25Tp/jVgMvxhsDNc2BS50fF5d5cvmQiLoB0RCR7sDPgH6q2gdoCpxPDZP8gwUA5wO9gcHABBEJi3xNxHqxjgw+Buew3QLcDbyhqn9MeWgWcHFw/2LgkVy1oSgsXuxFdp1L4T1uxSRBcf5xL5VgVptE1A2IXDOgtYjswnra1gBjgIHB45OBecCvsMn/D6rqTmCFiCwHThSRlcD+qvoCgIhMwRYGPJ6jNn8VGA4sFZElwbFfAzcC00RkJPA+cG6O3r/wbdoEw4Yli+w28z9Zzvm/gjzxRQou7k47eUbOEkwdOonIwpTPJwXDhACo6moR+T0Wcj4F5qrqXBGpaZJ/d2B+yuutCo7tCu5XP54Tqvo8UNOEtkG5et+isXcvXHQRrFoFzz3nRXadC3hwq0U2V5fmTQLvnSk2iagbUIc1NLaNG1S1X00PBgVqzwJ6Ah8D/ysiF9XyeunCktZy3MXRuHEwezZUVMCAAVG3xrnY8DlueZTznRRc8Unk521i/rP5LWCFqn6oqruAGcBXCCb5A1Sb5L8K6JHy/EOweLkquF/9uIublCK7XH551K1xLlY8uBWjRNQNcC6r3gcGiEibYML/IKyYbU2T/GcB54tISxHpiS1CeDEYVt0iIgOC1xmBLwyIn5Urvciuc7Xw4FYHX13qIpOIugHxoKoLgOnAy1gpkCZYmYwbgW+LyNvAt4PPUdVlwDTgdWAOUKaqe4KXuxy4C1gOvEPuFia4hgiL7O7ZAzNmeJFd59LwOW55lrdFCgn8D38hS+TvrWI+TAqAqo4FxlY7vJMaJvmr6jhgXJrjC4E+WW+gy46f/QwWLYJHHoEjjoi6Nc7Fkve4FbNE1A1wzrkM3Xsv3HknjBkDZ54ZdWuciy0PbhnI9nBpIfRwuAgl8vdW/rPoYsGL7DqXMQ9uxS4RdQNcvSSiboBzeVa9yG7TpnU/x7kS5nPcnCtR3tvmIpdaZPfvf/ciu85lwHvcMlTQw6WJ/L2Va4RE1A1wLs/CIru33AL9fWs75zLhwS1CHt7c5xL5fTvvbXOR8yK7zjVISQS3Az79JCuvU/A13RJRN8Cllcjv23loc5F77z0vsutcA5VEcAM485W5UTchLf8jWuISUTfAuTwLi+zu3g0PPeRFdp2rp5IJbi6QiLoB7nOJ/L+l/0fBRW70aFi4EKZMgSOPjLo1zhWckgpu2eh1y8Vwad7/mCby+3YujUT+39JDm4vcvffCpElQXg5nnRV1a5wrSCUV3FyKBB7gopKIugHORSAssvvNb3qRXecaoeSCm/e6VZOI5m1LViKat/XeNhepsMhux45WZLeZlxB1rqFKLrjFmYe3IpeI5m09tLlIpRbZnT4dOneOukXOFTQPbg1U8KVBqktE3YAil4i6Ac5FJCyye/PNMGBA1K1xruCVZHCLa2kQiLh3JIEHjCLjvW0uUqlFdkeNiro1zhWFkgxu2VJ0vW6hRNQNKDKJaN7WQ5uLlBfZdS4nSja4ea9bHRJRN6BIJKJugHMR8CK7zuVMyQa3bCnaXjfw0NFYiejeOhbh35UuL7LrXM5EGtxE5FwRWSYie0WkX7XHxojIchF5U0ROTTl+gogsDR77H5GG9797r1sGEniAq68E/jVzpeu++7zIrnM5FHWP22vA2cBzqQdF5CjgfKA3MBiYICJNg4cnApcARwYfg/PW2hrkqtctNuENPIhkIkEsvk6x+rlxpWXJErj8ci+y61wORRrcVPUNVX0zzUNnAQ+q6k5VXQEsB04Uka7A/qr6gqoqMAUY0pg2xLnXDWL2RzgRdQNiLBF1A0ysfl5cadm0Cc4+24vsOpdjUfe41aQ7UJny+argWPfgfvXjkSvquW6pElE3IGYS+NfEub17YfhwL7LrXB7kPLiJyFMi8lqaj9omP6Sbt6a1HE/3vpeIyEIRWfjhpoa0PD5i14uSwMMKxO5rELufE1c6xo2Dxx7zIrvO5UHOg5uqfktV+6T5eKSWp60CeqR8fgiwJjh+SJrj6d53kqr2U9V+B3WovY3ZGi7NZa9bLP8oJ6JuQEQSxO7aY/nz4UrD3LlWZPfCC73IrnN5ENeh0lnA+SLSUkR6YosQXlTVtcAWERkQrCYdAdQWAPPOw1sRS1Ba1+tcXd57D77/fejd24vsOpcnUZcDGSoiq4CTgMdE5AkAVV0GTANeB+YAZaq6J3ja5cBd2IKFd4DHs9GWuC9SCHl4i0gi6gbULJY/E674pRbZnTED9tsv6hY5VxIiXfajqjOBmTU8Ng4Yl+b4QqBPjpvWKJdxB7dzadTNyK9EtdtikYi6AbXz0OYiExbZnTnTi+w6l0dxHSqNhPe6ZUGC2IedjCQojutwLhfCIru/+hUMGRJ1a5wrKR7cciTX5UFiHd4gGXwSkbaiYRJRNyAzsf8ZcMUpLLJ7yilwww1Rt8a5kuPBrZpC6XWDAvrDnaAwQlyC+LfRuSilFtl98EEvsutcBDy45VDJFOWtjwTxDEiJqBtQPwUT2l1aIjI42Id5uYiUR92ejKQW2f3f//Uiu85FxINbGtnsdSv5IdPaJIguxCWIb4isQ0F/zx3BvssVwGnAUcD3g/2Z4y0ssvvHP8JJJ0XdGudKlvdzF4HTTp7B48+dHXUzGidRw/1svWaRKOXQFgSehcBqVT1DRA4E/gocCqwEzlPVTcG5Y4CRwB7gZ6r6RHD8BOA+oDUwGxgd7HucTycCy1X13aBND2L7M7+e53Zk7oknkkV2y8qibo1zJc2DWx7kozxIUYS3UKKG+3WdW+RKObQFRgNvAPsHn5cDT6vqjcFwYznwq6D36nygN9ANeEpEegW1ICcClwDzseA2mCzVgqyHdHsx969+kohcgrWVLl26MG/evIxefOvWrRmfm4mWH3xAv0svZeehh/LyhRey99lns/bajZXta42rUrlOKJ1rbcx1enCrwZmvzGXWsd+Juhn1UlThLZSIugHxUOqhTUQOAb6L1Xb8eXD4LGBgcH8yMA/4VXD8QVXdCawQkeXAiSKyEthfVV8IXnMKMIT8B7eM9lxW1UnAJIB+/frpwIEDM3rxefPmkem5ddqxA77+dQCaP/EEJ8esXltWrzXGSuU6oXSutTHX6XPc8sQXKriGyldoi/BntJOILEz5uCTNObcAvwT2phzrEmyDR3AbzpZP16PVPfhYleZ4vtW0F3P8hEV2J0/2IrvOxYT3uNXCe92cy8CWbfDMgsa8wgZV7VfTgyJyBrBeVReJyMAMXq+mHq2Merry4CXgyGAf5tXYsO4FEbSjdl5k17lY8h63PMpXj0apD6sVkxLobcvEV4Ezg6HOB4FvisifgXUi0hUguF0fnF9Tj9aq4H7143mlqruBK4AnsDl704L9mePDi+w6F1se3OqQ7YK8Ht5cpjy0GVUdo6qHqOqhWO/U/6nqRcAs4OLgtIuBR4L7s4DzRaRl0Kt1JPBiMJy6RUQGiIgAI1Kek1eqOltVe6nq4cG+zPGxaRMMG+ZFdp2LKQ9uRczDW+Hy711GbgS+LSJvA98OPifovZqGldeYA5QFK0oBLgfuApYD75D/hQnxFhbZraz0IrvOxZQHtwwUaq8beAAoRPn8nsW9t606VZ2nqmcE9z9S1UGqemRwuzHlvHFBb9YXVfXxlOMLVbVP8NgVEdRwi7ff/MaK7N58sxfZdS6mPLhFxMObq+60k2d4aHPRefJJuOYaK7I7alTUrXHO1cCDW4YKafP5dDy8xVu+vz8e2lwV770H3/8+9O4Nd9wBkm4BrnMuDkojuH2QnZcp5CFT8PAWVx7aXKR27oRzz4Vdu2DGDNhvv6hb5JyrRWkEN4Cbom5Aeh7eSpuHNhe50aPhpZe8yK5zBaJ0gluW5GLI1MNbafLvg4vc5Mk2NOpFdp0rGKUV3GLa6xYFDw3RyfcihJD3trkqliyByy7zIrvOFZjSCm6QlfBWDL1uEF2AKGVRfb09tLkqvMiucwWr9IJbjEX1x9UDXH54aHOxsHcvjBjhRXadK1ClGdxi2usG0f6R9fCWOx7aXGyMHw+PPgp//KMX2XWuAJVmcMuSQq/tlo73vmWffz1dbMydC1dfDRdcAGVlUbfGOdcApRvcYrxQIQ69JB42Gi/qEByHnyMXI++9Z4Gtd2+YNMmL7DpXoEo3uGVJMQ6ZhqIOHoUs6q9bHH5+XIykFtl96CEvsutcASvt4JalXrdiDm/gAa4+4vC1isvPjYuR1CK7vXpF3RrnXCOUdnCDWA+ZQrz+CEcdSOIsDoEN4vXz4mLCi+w6V1S8eE+WnPnKXGYd+52cvPZl3MHtXJqT166vMJw8/tzZEbckenEIaqk8tLl9eJFd54qO97hB7IdMIX5/lOMWWvIpLr1rztXKi+w6V5Q8uGVZqYW3Ugowcb7euP1suIiFRXbff9+L7DpXZDy4hWI+1y0Uxz/QcQ402RD364vjz4SLWFhk9+abvciuc0XGg1uqAhgyhfj+oY5zuGmIuAc2iO/PgotOh5de8iK7zhUxn/SQI7lcrADxWrCQqnrQKcRFDHEPayEPbW4f773HUTfc4EV2nStiHtyquwn4VXZeqlTDW6pCCnKFEtjAQ5urwYIFdutFdp0rWh7cClwhhLdUcQxyhRTYnKvVeecxf7/9+LoX2XWuaHlwS6eAet2g8MJbqnwEuWIKZt7T5uqyx3vanCtqHtzywMNb5uob5IoplNXFQ5tzzjkPbjXJYq9bvhRLeEtVSsGsNh7anHPOgZcDqV0Wa7vlukRIyP/AF598fU/z9TPqnHOu4Ty45ZGHN1dfHtqcc86l8uBWlyzvqODhzWXiMu7w0Oacc24fHtyKmIe3wpTP75uHNuecKywe3DJRoL1u4OGt0Hhoc845VxsPbpny8OZyzL9Pzjnn6hJpcBORc0VkmYjsFZF+KccPFZFPRWRJ8HF7ymMniMhSEVkuIv8jUrib8Xl4c6F8f38KrbdNRAaLyJvBv/vyqNvjnHNRibrH7TXgbOC5NI+9o6rHBR+XpRyfCFwCHBl8DM7kjf7xl8Y2laz3uuWbh7d48tBWOxFpClQApwFHAd8XkaOibZVzzkUj0uCmqm+o6puZni8iXYH9VfUFVVVgCjCkzif2PKHBbcy1fP8R9fAWH/lcORoqtNAWOBFYrqrvqupnwIPAWRG3yTnnIhF1j1tteorIYhF5VkS+HhzrDqxKOWdVcCx/ctDr5uGt9ETxPSjQ0Ab2b7wy5fP8/7t3zrmYyPmWVyLyFHBwmoeuUtVHanjaWuALqvqRiJwAPCwivYF089m0hve9BBtSBVj2NYC/sKNeja9Jw4ddOwEb0j+U1z+qnWBuDe3Iq1q+HnmV93Y8HpN2pPHF+j/lX0/AgE6NeM9WIrIw5fNJqjop5fOM/90Xq0WLFm0QkfcyPD0OP0f5UirXWirXCaVzrXVd57/V9EDOg5uqfqsBz9kJ7AzuLxKRd4Be2P+0D0k59RBgTQ2vMQn4/Je/iCxU1X7pzs2XOLTB2+HtqKsN9X2OqmY0z7QRVgE9Uj6v8d99sVLVgzI9Nw4/R/lSKtdaKtcJpXOtjbnOWA6VishBwYRkROQwbBHCu6q6FtgiIgOC1aQjgJp67ZxzxeEl4EgR6SkiLYDzgVkRt8k55yIRdTmQoSKyCjgJeExEnggeOhl4VUReAaYDl6nqxuCxy4G7gOXAO9Q46uScKwaquhu4AngCeAOYpqrLom2Vc85FI+dDpbVR1ZnAzDTHHwIequE5C4E+DXi7SXWfknNxaAN4O6rzdiTFoQ37UNXZwOyo21EgYvk9zJFSudZSuU4onWtt8HWKVdVwzjnnnHNxF8s5bs4555xzbl9FF9xq2kYreGxMsGXOmyJyasrxnG6jJSIJEVmdsoXX6XW1KVei2jpIRFYGX+Ml4cpFETlQRJ4UkbeD2w45eN97RGS9iLyWcqzG983V96OGduT950JEeojIMyLyRvDvZHRwPO9fE9c46X6mqj1+oYi8Gnz8U0SOzXcbs6Gu60w578siskdEzslX27Ipk+sUkYHB74plIvJsPtuXTRn87LYXkb+JyCvBtf4w323Mhpp+31Y7R4LcsTz4t3p8nS+sqkX1AXwJq0U1D+iXcvwo4BWgJdATW9jQNHjsRWyBhGCLHU7LcpsSwC/SHK+xTTn62jQN3uMwoEXw3kfl6fuyEuhU7dhvgfLgfjlwUw7e92TgeOC1ut43l9+PGtqR958LoCtwfHC/HfBW8H55/5r4R/Z/pqo9/hWgQ3D/NGBB1G3OxXUG5zQF/g+bB3lO1G3O0ffzAOB1rMYpQOeo25zDa/11yu+gg4CNQIuo292A60z7+7baOadjuUOAAZn8Oy26HjeteRuts4AHVXWnqq7AVqWeKA3dRis70rYph+8Xt62DzgImB/cnk4Ovu6o+h/2jz+R9c/b9qKEdNcllO9aq6svB/S3YKs3uRPA1cY1T18+Uqv5TVTcFn86nag3MgpHhv52fYgva1ue+RbmRwXVeAMxQ1feD84v5WhVoF4x+tQ3O3Z2PtmVTLb9vU50FTFEzHzggyCU1KrrgVouats3J1zZaVwTdoPekDEPleyufKLcOUmCuiCwS29UCoItabT6C2855aktN7xvF1yeynwsRORToCywgXl8Tl30jKdLSSSLSHRgK3B51W3KsF9BBROYFv0dHRN2gHLoNGz1bAywFRqvq3mib1DjVft+mqvfv2IIMbiLylIi8luajtt6jmrbNycp2OnW0aSJwOHActp3XH+poU65EuXXQV1X1eGzIpkxETs7T+9ZHvr8+kf1ciEhbrIfiSlX9pLZTc90Wl1sicgoW3H4VdVty5BbgV6q6J+qG5Fgz4ATgu8CpwNUi0ivaJuXMqcASoBv2+/E2Edk/ygY1Rh2/b+v9OzbSOm4NpQ3YRouat83JeButbLRJRO4EHq2jTbkS2dZBqromuF0vIjOx4bZ1ItJVVdcGXcP56vqv6X3z+vVR1XXh/Xz+XIhIc+yXyP2qOiM4HIuvicsuETkGK1h+mqp+FHV7cqQf8KCNqtEJOF1Edqvqw5G2KvtWARtUdRuwTUSeA47F5k0Vmx8CNwbTl5aLyArg37H56AWlht+3qer9O7Yge9waaBZwvoi0FJGe2DZaL2oettGqNl49FAhX0qRtUzbfu5pItg4Skf1EpF14H/gO9jWYBVwcnHYx+du+rKb3zev3I4qfi+Bn/G7gDVX9Y8pDsfiauOwRkS8AM4DhqlqMf9wBUNWeqnqoqh6K7bQzqghDG9i/ya+LSDMRaQP0x+ZMFaP3gUEAItIFW3D4bqQtaoBaft+mmgWMCFaXDgA2h9NWalKQPW61EZGhwJ+wlSiPicgSVT1VVZeJyDRsVc5uoCyla/1y4D6gNTYPJNtzQX4rIsdh3Z8rgUsB6mhT1qnqbhEJtw5qCtyj+dk6qAswM/gfcTPgAVWdIyIvAdNEZCT2D/XcbL+xiPwFGAh0EttebSxwY7r3zeX3o4Z2DIzg5+KrwHBgqYgsCY79mgi+Jq5xaviZag6gqrcD1wAdgQnBv73dWoCbd2dwnUWhrutU1TdEZA7wKrAXuEtVay2RElcZfE+vB+4TkaXYUOKvVHVDRM1tjJp+334BPr/W2djK0uXAdqy3sVa+c4JzzjnnXIEopaFS55xzzrmC5sHNOeecc65AeHBzzjnnnCsQHtycc8455wqEBzfnnHPOuQLhwc0555xzrkB4cHPOOeecKxAe3FzOBTs1PBvcP15EVEQ6ikjTYD/XNlG30Tnn4kJEviwir4pIq2DnmWUi0ifqdrl4KLqdE1wsfQy0C+7/FJgPdMCqSj+pqtsjapdzzsWOqr4kIrOAG7Adff5cqLskuOzz4ObyYTPQRkQ6Al2Bf2DB7RLg58H+pROAz4B5qnp/ZC11zrl4uA7bX3oH8LOI2+JixIdKXc6p6t7g7k+wDXe3AMcATYPNr88GpqvqT4Azo2mlc87FyoFAW2y0olXEbXEx4sHN5cteLJTNBD4BfgGEG0QfAlQG930Dc+ecg0nA1cD9wE0Rt8XFiAc3ly+fAY+r6m4suO0HPBo8tgoLb+A/k865EiciI4DdqvoAcCPwZRH5ZsTNcjEhqhp1G1yJC+a43YbN5Xje57g555xz6Xlwc84555wrED4s5ZxzzjlXIDy4Oeecc84VCA9uzjnnnHMFwoObc84551yB8ODmnHPOOVcgPLg555xzzhUID27OOeeccwXCg5tzzjnnXIHw4Oacc845VyD+P6+2PbkdDRXYAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 720x432 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from grid_search import generate_w, get_best_parameters\n",
    "from plots import grid_visualization\n",
    "\n",
    "# Generate the grid of parameters to be swept\n",
    "grid_w0, grid_w1 = generate_w(num_intervals=10)\n",
    "\n",
    "# Start the grid search\n",
    "start_time = datetime.datetime.now()\n",
    "grid_losses = grid_search(y, tx, grid_w0, grid_w1)\n",
    "\n",
    "# Select the best combinaison\n",
    "loss_star, w0_star, w1_star = get_best_parameters(grid_w0, grid_w1, grid_losses)\n",
    "end_time = datetime.datetime.now()\n",
    "execution_time = (end_time - start_time).total_seconds()\n",
    "\n",
    "# Print the results\n",
    "print(\n",
    "    \"Grid Search: loss*={l}, w0*={w0}, w1*={w1}, execution time={t:.3f} seconds\".format(\n",
    "        l=loss_star, w0=w0_star, w1=w1_star, t=execution_time\n",
    "    )\n",
    ")\n",
    "\n",
    "# Plot the results\n",
    "fig = grid_visualization(grid_losses, grid_w0, grid_w1, mean_x, std_x, height, weight)\n",
    "fig.set_size_inches(10.0, 6.0)\n",
    "fig.savefig(\"grid_plot\")  # Optional saving"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Gradient Descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, please fill in the functions `compute_gradient` below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gradient(y, tx, w):\n",
    "    \"\"\"Computes the gradient at w.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        w: numpy array of shape=(2, ). The vector of model parameters.\n",
    "\n",
    "    Returns:\n",
    "        An numpy array of shape (2, ) (same shape as w), containing the gradient of the loss at w.\n",
    "    \"\"\"\n",
    "    ### SOLUTION\n",
    "    err = y - tx.dot(w)\n",
    "    grad = -tx.T.dot(err) / len(err)\n",
    "    return grad, err\n",
    "\n",
    "    ### TEMPLATE\n",
    "    # # ***************************************************\n",
    "    # # INSERT YOUR CODE HERE\n",
    "    # # TODO: compute gradient vector\n",
    "    # # ***************************************************\n",
    "    # raise NotImplementedError\n",
    "    ### END SOLUTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please fill in the functions `gradient_descent` below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(y, tx, initial_w, max_iters, gamma):\n",
    "    \"\"\"The Gradient Descent (GD) algorithm.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        initial_w: numpy array of shape=(2, ). The initial guess (or the initialization) for the model parameters\n",
    "        max_iters: a scalar denoting the total number of iterations of GD\n",
    "        gamma: a scalar denoting the stepsize\n",
    "\n",
    "    Returns:\n",
    "        losses: a list of length max_iters containing the loss value (scalar) for each iteration of GD\n",
    "        ws: a list of length max_iters + 1 containing the model parameters as numpy arrays of shape (2, ),\n",
    "            for each iteration of GD (as well as the final weights)\n",
    "    \"\"\"\n",
    "    # Define parameters to store w and loss\n",
    "    ws = [initial_w]\n",
    "    losses = []\n",
    "    w = initial_w\n",
    "    for n_iter in range(max_iters):\n",
    "        ### SOLUTION\n",
    "        # compute loss, gradient\n",
    "        grad, err = compute_gradient(y, tx, w)\n",
    "        loss = calculate_mse(err)\n",
    "        # update w by gradient descent\n",
    "        w = w - gamma * grad\n",
    "\n",
    "        ### TEMPLATE\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: compute gradient and loss\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: update w by gradient\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        ### END SOLUTION\n",
    "\n",
    "        # store w and loss\n",
    "        ws.append(w)\n",
    "        losses.append(loss)\n",
    "        print(\n",
    "            \"GD iter. {bi}/{ti}: loss={l}, w0={w0}, w1={w1}\".format(\n",
    "                bi=n_iter, ti=max_iters - 1, l=loss, w0=w[0], w1=w[1]\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return losses, ws"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test your gradient descent function through gradient descent demo shown below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GD iter. 0/49: loss=2792.2367127591674, w0=51.305745401473644, w1=9.435798704492269\n",
      "GD iter. 1/49: loss=265.3024621089598, w0=66.69746902191571, w1=12.266538315840005\n",
      "GD iter. 2/49: loss=37.87837955044126, w0=71.31498610804834, w1=13.115760199244333\n",
      "GD iter. 3/49: loss=17.410212120174467, w0=72.70024123388814, w1=13.370526764265632\n",
      "GD iter. 4/49: loss=15.568077051450455, w0=73.11581777164007, w1=13.446956733772023\n",
      "GD iter. 5/49: loss=15.402284895265295, w0=73.24049073296565, w1=13.469885724623941\n",
      "GD iter. 6/49: loss=15.38736360120863, w0=73.27789262136334, w1=13.476764421879516\n",
      "GD iter. 7/49: loss=15.38602068474353, w0=73.28911318788263, w1=13.478828031056189\n",
      "GD iter. 8/49: loss=15.385899822261674, w0=73.29247935783842, w1=13.47944711380919\n",
      "GD iter. 9/49: loss=15.385888944638305, w0=73.29348920882515, w1=13.47963283863509\n",
      "GD iter. 10/49: loss=15.3858879656522, w0=73.29379216412117, w1=13.479688556082861\n",
      "GD iter. 11/49: loss=15.385887877543452, w0=73.29388305070998, w1=13.479705271317192\n",
      "GD iter. 12/49: loss=15.385887869613665, w0=73.29391031668663, w1=13.479710285887492\n",
      "GD iter. 13/49: loss=15.385887868899983, w0=73.29391849647962, w1=13.479711790258582\n",
      "GD iter. 14/49: loss=15.38588786883575, w0=73.29392095041752, w1=13.479712241569908\n",
      "GD iter. 15/49: loss=15.385887868829974, w0=73.29392168659889, w1=13.479712376963306\n",
      "GD iter. 16/49: loss=15.38588786882945, w0=73.2939219074533, w1=13.479712417581325\n",
      "GD iter. 17/49: loss=15.385887868829403, w0=73.29392197370962, w1=13.479712429766732\n",
      "GD iter. 18/49: loss=15.3858878688294, w0=73.29392199358652, w1=13.479712433422353\n",
      "GD iter. 19/49: loss=15.385887868829403, w0=73.2939219995496, w1=13.47971243451904\n",
      "GD iter. 20/49: loss=15.385887868829398, w0=73.29392200133852, w1=13.479712434848047\n",
      "GD iter. 21/49: loss=15.3858878688294, w0=73.29392200187519, w1=13.479712434946748\n",
      "GD iter. 22/49: loss=15.3858878688294, w0=73.29392200203618, w1=13.479712434976358\n",
      "GD iter. 23/49: loss=15.3858878688294, w0=73.29392200208449, w1=13.479712434985242\n",
      "GD iter. 24/49: loss=15.3858878688294, w0=73.29392200209898, w1=13.479712434987906\n",
      "GD iter. 25/49: loss=15.385887868829398, w0=73.29392200210333, w1=13.479712434988706\n",
      "GD iter. 26/49: loss=15.3858878688294, w0=73.29392200210464, w1=13.479712434988945\n",
      "GD iter. 27/49: loss=15.3858878688294, w0=73.29392200210502, w1=13.479712434989018\n",
      "GD iter. 28/49: loss=15.3858878688294, w0=73.29392200210513, w1=13.47971243498904\n",
      "GD iter. 29/49: loss=15.3858878688294, w0=73.29392200210518, w1=13.479712434989047\n",
      "GD iter. 30/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 31/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 32/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 33/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 34/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 35/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 36/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 37/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 38/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 39/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 40/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 41/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 42/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 43/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 44/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 45/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 46/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 47/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 48/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD iter. 49/49: loss=15.3858878688294, w0=73.29392200210519, w1=13.479712434989048\n",
      "GD: execution time=0.015 seconds\n"
     ]
    }
   ],
   "source": [
    "# from gradient_descent import *\n",
    "from plots import gradient_descent_visualization\n",
    "\n",
    "# Define the parameters of the algorithm.\n",
    "max_iters = 50\n",
    "gamma = 0.7\n",
    "\n",
    "# Initialization\n",
    "w_initial = np.array([0, 0])\n",
    "\n",
    "# Start gradient descent.\n",
    "start_time = datetime.datetime.now()\n",
    "gd_losses, gd_ws = gradient_descent(y, tx, w_initial, max_iters, gamma)\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "# Print result\n",
    "exection_time = (end_time - start_time).total_seconds()\n",
    "print(\"GD: execution time={t:.3f} seconds\".format(t=exection_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d842832a8aa413fbe6d022c839283fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=1, description='n_iter', max=51, min=1), Output()), _dom_classes=('widge…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_figure(n_iter)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Time Visualization\n",
    "from ipywidgets import IntSlider, interact\n",
    "\n",
    "\n",
    "def plot_figure(n_iter):\n",
    "    fig = gradient_descent_visualization(\n",
    "        gd_losses,\n",
    "        gd_ws,\n",
    "        grid_losses,\n",
    "        grid_w0,\n",
    "        grid_w1,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        height,\n",
    "        weight,\n",
    "        n_iter,\n",
    "    )\n",
    "    fig.set_size_inches(10.0, 6.0)\n",
    "\n",
    "\n",
    "interact(plot_figure, n_iter=IntSlider(min=1, max=len(gd_ws)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 4. Stochastic gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_stoch_gradient(y, tx, w):\n",
    "    \"\"\"Compute a stochastic gradient at w from a data sample batch of size B, where B < N, and their corresponding labels.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(B, )\n",
    "        tx: numpy array of shape=(B,2)\n",
    "        w: numpy array of shape=(2, ). The vector of model parameters.\n",
    "\n",
    "    Returns:\n",
    "        A numpy array of shape (2, ) (same shape as w), containing the stochastic gradient of the loss at w.\n",
    "    \"\"\"\n",
    "\n",
    "    ### SOLUTION\n",
    "    err = y - tx.dot(w)\n",
    "    grad = -tx.T.dot(err) / len(err)\n",
    "    return grad, err\n",
    "\n",
    "    ### TEMPLATE\n",
    "    # # ***************************************************\n",
    "    # # INSERT YOUR CODE HERE\n",
    "    # # TODO: implement stochastic gradient computation. It's the same as the usual gradient.\n",
    "    # # ***************************************************\n",
    "    # raise NotImplementedError\n",
    "    ### END SOLUTION\n",
    "\n",
    "\n",
    "def stochastic_gradient_descent(y, tx, initial_w, batch_size, max_iters, gamma):\n",
    "    \"\"\"The Stochastic Gradient Descent algorithm (SGD).\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        initial_w: numpy array of shape=(2, ). The initial guess (or the initialization) for the model parameters\n",
    "        batch_size: a scalar denoting the number of data points in a mini-batch used for computing the stochastic gradient\n",
    "        max_iters: a scalar denoting the total number of iterations of SGD\n",
    "        gamma: a scalar denoting the stepsize\n",
    "\n",
    "    Returns:\n",
    "        losses: a list of length max_iters containing the loss value (scalar) for each iteration of SGD\n",
    "        ws: a list of length max_iters containing the model parameters as numpy arrays of shape (2, ), for each iteration of SGD\n",
    "    \"\"\"\n",
    "\n",
    "    # Define parameters to store w and loss\n",
    "    ws = [initial_w]\n",
    "    losses = []\n",
    "    w = initial_w\n",
    "\n",
    "    for n_iter in range(max_iters):\n",
    "        ### SOLUTION\n",
    "        for y_batch, tx_batch in batch_iter(\n",
    "            y, tx, batch_size=batch_size, num_batches=1\n",
    "        ):\n",
    "            # compute a stochastic gradient and loss\n",
    "            grad, _ = compute_stoch_gradient(y_batch, tx_batch, w)\n",
    "            # update w through the stochastic gradient update\n",
    "            w = w - gamma * grad\n",
    "            # calculate loss\n",
    "            loss = compute_loss(y, tx, w)\n",
    "            # store w and loss\n",
    "            ws.append(w)\n",
    "            losses.append(loss)\n",
    "\n",
    "        ### TEMPLATE\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: implement stochastic gradient descent.\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        ### END SOLUTION\n",
    "\n",
    "        print(\n",
    "            \"SGD iter. {bi}/{ti}: loss={l}, w0={w0}, w1={w1}\".format(\n",
    "                bi=n_iter, ti=max_iters - 1, l=loss, w0=w[0], w1=w[1]\n",
    "            )\n",
    "        )\n",
    "    return losses, ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD iter. 0/49: loss=2511.025623058236, w0=5.580780474540902, w1=-6.674938017124126\n",
      "SGD iter. 1/49: loss=2275.65113180722, w0=10.67949568849968, w1=-11.01445228428339\n",
      "SGD iter. 2/49: loss=1553.8441438273528, w0=19.935201781860602, w1=-1.6782388901310554\n",
      "SGD iter. 3/49: loss=989.6758821668608, w0=29.249909332336173, w1=16.430125379186666\n",
      "SGD iter. 4/49: loss=790.4401283335372, w0=34.846438645274816, w1=5.000354885288253\n",
      "SGD iter. 5/49: loss=723.5807130915333, w0=37.8192839591523, w1=0.9123059337697814\n",
      "SGD iter. 6/49: loss=462.355128385035, w0=43.67293720391913, w1=9.41329624649552\n",
      "SGD iter. 7/49: loss=412.3385262875832, w0=45.787639441283794, w1=7.371546055316614\n",
      "SGD iter. 8/49: loss=342.8719966611879, w0=48.39288694302858, w1=7.571187189942297\n",
      "SGD iter. 9/49: loss=248.0108558307405, w0=51.8726396903842, w1=10.954123385802362\n",
      "SGD iter. 10/49: loss=226.2214191254766, w0=53.47830313862976, w1=8.093404648501965\n",
      "SGD iter. 11/49: loss=203.30978311817813, w0=55.15529420924709, w1=6.635885121938737\n",
      "SGD iter. 12/49: loss=186.17848676062766, w0=56.41552034744507, w1=5.949456438838368\n",
      "SGD iter. 13/49: loss=126.23379308386446, w0=59.022435595765394, w1=9.234658114520506\n",
      "SGD iter. 14/49: loss=88.39643748213528, w0=61.43599637337616, w1=11.153621505953133\n",
      "SGD iter. 15/49: loss=83.55575662043049, w0=61.89137811133208, w1=10.965407297939233\n",
      "SGD iter. 16/49: loss=75.23997328417481, w0=62.579848450795495, w1=11.262326864589317\n",
      "SGD iter. 17/49: loss=58.55973763903879, w0=64.1861380435786, w1=11.636896409113312\n",
      "SGD iter. 18/49: loss=47.19329511819909, w0=65.45246767713714, w1=12.021491406413745\n",
      "SGD iter. 19/49: loss=40.23544670765998, w0=66.34647439731486, w1=12.283013033988803\n",
      "SGD iter. 20/49: loss=32.60874673253512, w0=67.44605050508333, w1=13.977825556409066\n",
      "SGD iter. 21/49: loss=29.826734470382593, w0=67.93670877909285, w1=13.906279764127335\n",
      "SGD iter. 22/49: loss=25.83799183838366, w0=68.96654211444653, w1=14.955514334634053\n",
      "SGD iter. 23/49: loss=25.85558851086333, w0=68.9619278482557, w1=14.95389964666933\n",
      "SGD iter. 24/49: loss=24.955276207101157, w0=69.37676596706419, w1=15.42770249494453\n",
      "SGD iter. 25/49: loss=23.804389854025917, w0=69.59001549238944, w1=15.245521183885236\n",
      "SGD iter. 26/49: loss=22.07684193257946, w0=71.09626187031263, w1=16.404126576766785\n",
      "SGD iter. 27/49: loss=22.06545854964137, w0=71.11232600061005, w1=16.412250627047996\n",
      "SGD iter. 28/49: loss=23.022109279859706, w0=70.56435326279887, w1=16.27647794526379\n",
      "SGD iter. 29/49: loss=21.03526207428459, w0=70.86303894754286, w1=15.801254161073697\n",
      "SGD iter. 30/49: loss=20.995590275616447, w0=70.880106263175, w1=15.80197391726596\n",
      "SGD iter. 31/49: loss=19.37686936664767, w0=71.60963004906719, w1=15.747998927601474\n",
      "SGD iter. 32/49: loss=19.385853574977727, w0=71.60430376323447, w1=15.747998488195753\n",
      "SGD iter. 33/49: loss=19.071530702040572, w0=71.9541553385511, w1=15.8411338191667\n",
      "SGD iter. 34/49: loss=18.963286515467438, w0=71.89575237454841, w1=15.760045521591485\n",
      "SGD iter. 35/49: loss=19.16691582053634, w0=71.84414442553752, w1=15.816419708460728\n",
      "SGD iter. 36/49: loss=19.108722394351854, w0=72.05342467710236, w1=15.910110640573006\n",
      "SGD iter. 37/49: loss=19.71583685725685, w0=71.87231452942505, w1=16.05632458478476\n",
      "SGD iter. 38/49: loss=21.477766845374397, w0=72.24051810883225, w1=16.807489198945337\n",
      "SGD iter. 39/49: loss=19.678801460870485, w0=72.88630417436846, w1=16.381380040107416\n",
      "SGD iter. 40/49: loss=19.2519292185589, w0=72.50828908336885, w1=16.147082603611473\n",
      "SGD iter. 41/49: loss=19.018492497548735, w0=72.5862760225, w1=16.08056737615499\n",
      "SGD iter. 42/49: loss=18.82771134469247, w0=72.33014531470742, w1=15.919954523093162\n",
      "SGD iter. 43/49: loss=20.467601703579728, w0=71.97212484071166, w1=16.380794980114686\n",
      "SGD iter. 44/49: loss=20.034043499986076, w0=72.16920413621553, w1=16.313670971111613\n",
      "SGD iter. 45/49: loss=17.295598423026867, w0=73.43124109654585, w1=15.429216108926836\n",
      "SGD iter. 46/49: loss=16.24344040301425, w0=72.92032359484709, w1=14.734913334631775\n",
      "SGD iter. 47/49: loss=16.501919059776057, w0=72.85784568970737, w1=14.908663040034176\n",
      "SGD iter. 48/49: loss=15.696072875818203, w0=73.39505122061404, w1=14.260828876463297\n",
      "SGD iter. 49/49: loss=15.512520837470493, w0=73.09907402420723, w1=13.943716963836843\n",
      "SGD: execution time=0.032 seconds\n"
     ]
    }
   ],
   "source": [
    "# from stochastic_gradient_descent import *\n",
    "\n",
    "# Define the parameters of the algorithm.\n",
    "max_iters = 50\n",
    "gamma = 0.1\n",
    "batch_size = 1\n",
    "\n",
    "# Initialization\n",
    "w_initial = np.array([0, 0])\n",
    "\n",
    "# Start SGD.\n",
    "start_time = datetime.datetime.now()\n",
    "sgd_losses, sgd_ws = stochastic_gradient_descent(\n",
    "    y, tx, w_initial, batch_size, max_iters, gamma\n",
    ")\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "# Print result\n",
    "exection_time = (end_time - start_time).total_seconds()\n",
    "print(\"SGD: execution time={t:.3f} seconds\".format(t=exection_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdbdc510bb46462490ddd4eee30882dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=1, description='n_iter', max=51, min=1), Output()), _dom_classes=('widge…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_figure(n_iter)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Time Visualization\n",
    "from ipywidgets import IntSlider, interact\n",
    "\n",
    "\n",
    "def plot_figure(n_iter):\n",
    "    fig = gradient_descent_visualization(\n",
    "        sgd_losses,\n",
    "        sgd_ws,\n",
    "        grid_losses,\n",
    "        grid_w0,\n",
    "        grid_w1,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        height,\n",
    "        weight,\n",
    "        n_iter,\n",
    "    )\n",
    "    fig.set_size_inches(10.0, 6.0)\n",
    "\n",
    "\n",
    "interact(plot_figure, n_iter=IntSlider(min=1, max=len(sgd_ws)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Effect of Outliers and MAE Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "from helpers import *\n",
    "\n",
    "### SOLUTION\n",
    "height, weight, gender = load_data(sub_sample=True, add_outlier=True)\n",
    "### TEMPLATE\n",
    "## ***************************************************\n",
    "## INSERT YOUR CODE HERE\n",
    "## TODO: reload the data by subsampling first, then by subsampling and adding outliers\n",
    "## ***************************************************\n",
    "# raise NotImplementedError\n",
    "### END SOLUTION\n",
    "\n",
    "x, mean_x, std_x = standardize(height)\n",
    "y, tx = build_model_data(x, weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((202,), (202, 2))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape, tx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GD iter. 0/49: loss=2869.8351145358524, w0=51.84746409844842, w1=7.7244264061924195\n",
      "GD iter. 1/49: loss=318.2821247015965, w0=67.40170332798297, w1=10.041754328050114\n",
      "GD iter. 2/49: loss=88.6423556165128, w0=72.06797509684336, w1=10.736952704607411\n",
      "GD iter. 3/49: loss=67.9747763988552, w0=73.46785662750146, w1=10.945512217574597\n",
      "GD iter. 4/49: loss=66.11469426926604, w0=73.88782108669889, w1=11.00808007146475\n",
      "GD iter. 5/49: loss=65.94728687760302, w0=74.01381042445813, w1=11.026850427631798\n",
      "GD iter. 6/49: loss=65.93222021235334, w0=74.0516072257859, w1=11.032481534481914\n",
      "GD iter. 7/49: loss=65.93086421248087, w0=74.06294626618423, w1=11.034170866536945\n",
      "GD iter. 8/49: loss=65.93074217249236, w0=74.06634797830372, w1=11.034677666153454\n",
      "GD iter. 9/49: loss=65.93073118889338, w0=74.06736849193958, w1=11.034829706038408\n",
      "GD iter. 10/49: loss=65.93073020036948, w0=74.06767464603033, w1=11.034875318003895\n",
      "GD iter. 11/49: loss=65.93073011140233, w0=74.06776649225755, w1=11.034889001593541\n",
      "GD iter. 12/49: loss=65.93073010339529, w0=74.06779404612573, w1=11.034893106670431\n",
      "GD iter. 13/49: loss=65.93073010267466, w0=74.06780231228618, w1=11.034894338193501\n",
      "GD iter. 14/49: loss=65.93073010260979, w0=74.06780479213431, w1=11.034894707650421\n",
      "GD iter. 15/49: loss=65.93073010260395, w0=74.06780553608874, w1=11.034894818487496\n",
      "GD iter. 16/49: loss=65.93073010260342, w0=74.06780575927507, w1=11.03489485173862\n",
      "GD iter. 17/49: loss=65.93073010260338, w0=74.06780582623098, w1=11.034894861713957\n",
      "GD iter. 18/49: loss=65.93073010260338, w0=74.06780584631775, w1=11.034894864706557\n",
      "GD iter. 19/49: loss=65.93073010260336, w0=74.06780585234378, w1=11.034894865604338\n",
      "GD iter. 20/49: loss=65.93073010260338, w0=74.06780585415159, w1=11.034894865873675\n",
      "GD iter. 21/49: loss=65.93073010260336, w0=74.06780585469393, w1=11.034894865954474\n",
      "GD iter. 22/49: loss=65.93073010260338, w0=74.06780585485663, w1=11.034894865978712\n",
      "GD iter. 23/49: loss=65.93073010260336, w0=74.06780585490544, w1=11.034894865985985\n",
      "GD iter. 24/49: loss=65.93073010260338, w0=74.0678058549201, w1=11.034894865988166\n",
      "GD iter. 25/49: loss=65.93073010260336, w0=74.06780585492449, w1=11.034894865988822\n",
      "GD iter. 26/49: loss=65.93073010260336, w0=74.06780585492581, w1=11.034894865989015\n",
      "GD iter. 27/49: loss=65.93073010260336, w0=74.06780585492619, w1=11.034894865989076\n",
      "GD iter. 28/49: loss=65.93073010260338, w0=74.06780585492632, w1=11.034894865989099\n",
      "GD iter. 29/49: loss=65.93073010260339, w0=74.06780585492635, w1=11.0348948659891\n",
      "GD iter. 30/49: loss=65.93073010260338, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 31/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 32/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 33/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 34/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 35/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 36/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 37/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 38/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 39/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 40/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 41/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 42/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 43/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 44/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 45/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 46/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 47/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 48/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD iter. 49/49: loss=65.93073010260339, w0=74.06780585492636, w1=11.0348948659891\n",
      "GD: execution time=0.004 seconds\n"
     ]
    }
   ],
   "source": [
    "from plots import gradient_descent_visualization\n",
    "\n",
    "# Define the parameters of the algorithm.\n",
    "max_iters = 50\n",
    "gamma = 0.7\n",
    "\n",
    "# Initialization\n",
    "w_initial = np.array([0, 0])\n",
    "\n",
    "# Start gradient descent.\n",
    "start_time = datetime.datetime.now()\n",
    "\n",
    "### SOLUTION\n",
    "gd_losses, gd_ws = gradient_descent(y, tx, w_initial, max_iters, gamma)\n",
    "### TEMPLATE\n",
    "# # ***************************************************\n",
    "# # INSERT YOUR CODE HERE\n",
    "# # TODO: fit the model to the subsampled data / subsampled data with outliers and visualize the cloud of points\n",
    "# #       and the model fit\n",
    "# # ***************************************************\n",
    "# raise NotImplementedError\n",
    "### END SOLUTION\n",
    "\n",
    "\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "# Print result\n",
    "exection_time = (end_time - start_time).total_seconds()\n",
    "print(\"GD: execution time={t:.3f} seconds\".format(t=exection_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac56884593f1450b8e978866badd19e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=1, description='n_iter', max=51, min=1), Output()), _dom_classes=('widge…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_figure(n_iter)>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Time Visualization\n",
    "from ipywidgets import IntSlider, interact\n",
    "\n",
    "\n",
    "def plot_figure(n_iter):\n",
    "    fig = gradient_descent_visualization(\n",
    "        gd_losses,\n",
    "        gd_ws,\n",
    "        grid_losses,\n",
    "        grid_w0,\n",
    "        grid_w1,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        height,\n",
    "        weight,\n",
    "        n_iter,\n",
    "    )\n",
    "    fig.set_size_inches(10.0, 6.0)\n",
    "\n",
    "\n",
    "interact(plot_figure, n_iter=IntSlider(min=1, max=len(gd_ws)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 6. Subgradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_subgradient_mae(y, tx, w):\n",
    "    \"\"\"Compute a subgradient of the MAE at w.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        w: numpy array of shape=(2, ). The vector of model parameters.\n",
    "\n",
    "    Returns:\n",
    "        A numpy array of shape (2, ) (same shape as w), containing the subgradient of the MAE at w.\n",
    "    \"\"\"\n",
    "    ### SOLUTION\n",
    "    err = y - tx.dot(w)\n",
    "    grad = -np.dot(tx.T, np.sign(err)) / len(err)\n",
    "    return grad, err\n",
    "    ### TEMPLATE\n",
    "    # # ***************************************************\n",
    "    # # INSERT YOUR CODE HERE\n",
    "    # # TODO: compute subgradient gradient vector for MAE\n",
    "    # # ***************************************************\n",
    "    # raise NotImplementedError\n",
    "    ### END SOLUTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subgradient_descent(y, tx, initial_w, max_iters, gamma):\n",
    "    \"\"\"The SubGradient Descent (SubGD) algorithm.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(N, )\n",
    "        tx: numpy array of shape=(N,2)\n",
    "        initial_w: numpy array of shape=(2, ). The initial guess (or the initialization) for the model parameters\n",
    "        max_iters: a scalar denoting the total number of iterations of GD\n",
    "        gamma: a scalar denoting the stepsize\n",
    "\n",
    "    Returns:\n",
    "        losses: a list of length max_iters containing the loss value (scalar) for each iteration of SubGD\n",
    "        ws: a list of length max_iters containing the model parameters as numpy arrays of shape (2, ), for each iteration of SubGD\n",
    "    \"\"\"\n",
    "    # Define parameters to store w and loss\n",
    "    ws = [initial_w]\n",
    "    losses = []\n",
    "    w = initial_w\n",
    "    for n_iter in range(max_iters):\n",
    "        ### SOLUTION\n",
    "        # compute loss, gradient\n",
    "        grad, err = compute_subgradient_mae(y, tx, w)\n",
    "        loss = calculate_mae(err)\n",
    "        # gradient w by descent update\n",
    "        w = w - gamma * grad\n",
    "        # store w and loss\n",
    "\n",
    "        ### TEMPLATE\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: compute subgradient and loss\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: update w by subgradient\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        ### END SOLUTION\n",
    "\n",
    "        ws.append(w)\n",
    "        losses.append(loss)\n",
    "        print(\n",
    "            \"SubGD iter. {bi}/{ti}: loss={l}, w0={w0}, w1={w1}\".format(\n",
    "                bi=n_iter, ti=max_iters - 1, l=loss, w0=w[0], w1=w[1]\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return losses, ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SubGD iter. 0/499: loss=74.06780585492638, w0=0.7, w1=6.109524327590712e-16\n",
      "SubGD iter. 1/499: loss=73.36780585492637, w0=1.4, w1=1.2219048655181425e-15\n",
      "SubGD iter. 2/499: loss=72.66780585492637, w0=2.0999999999999996, w1=1.832857298277214e-15\n",
      "SubGD iter. 3/499: loss=71.96780585492638, w0=2.8, w1=2.443809731036285e-15\n",
      "SubGD iter. 4/499: loss=71.26780585492638, w0=3.5, w1=3.054762163795356e-15\n",
      "SubGD iter. 5/499: loss=70.56780585492638, w0=4.2, w1=3.665714596554428e-15\n",
      "SubGD iter. 6/499: loss=69.86780585492637, w0=4.9, w1=4.276667029313499e-15\n",
      "SubGD iter. 7/499: loss=69.16780585492639, w0=5.6000000000000005, w1=4.887619462072571e-15\n",
      "SubGD iter. 8/499: loss=68.46780585492637, w0=6.300000000000001, w1=5.498571894831642e-15\n",
      "SubGD iter. 9/499: loss=67.76780585492638, w0=7.000000000000001, w1=6.109524327590714e-15\n",
      "SubGD iter. 10/499: loss=67.06780585492638, w0=7.700000000000001, w1=6.720476760349785e-15\n",
      "SubGD iter. 11/499: loss=66.36780585492637, w0=8.4, w1=7.331429193108857e-15\n",
      "SubGD iter. 12/499: loss=65.66780585492639, w0=9.1, w1=7.942381625867928e-15\n",
      "SubGD iter. 13/499: loss=64.96780585492638, w0=9.799999999999999, w1=8.553334058627e-15\n",
      "SubGD iter. 14/499: loss=64.26780585492638, w0=10.499999999999998, w1=9.164286491386072e-15\n",
      "SubGD iter. 15/499: loss=63.567805854926384, w0=11.199999999999998, w1=9.775238924145143e-15\n",
      "SubGD iter. 16/499: loss=62.867805854926374, w0=11.899999999999997, w1=1.0386191356904215e-14\n",
      "SubGD iter. 17/499: loss=62.167805854926385, w0=12.599999999999996, w1=1.0997143789663286e-14\n",
      "SubGD iter. 18/499: loss=61.46780585492638, w0=13.299999999999995, w1=1.1608096222422358e-14\n",
      "SubGD iter. 19/499: loss=60.76780585492638, w0=13.999999999999995, w1=1.2219048655181429e-14\n",
      "SubGD iter. 20/499: loss=60.067805854926384, w0=14.699999999999994, w1=1.28300010879405e-14\n",
      "SubGD iter. 21/499: loss=59.36780585492639, w0=15.399999999999993, w1=1.3440953520699572e-14\n",
      "SubGD iter. 22/499: loss=58.667805854926385, w0=16.099999999999994, w1=1.4051905953458644e-14\n",
      "SubGD iter. 23/499: loss=57.96780585492638, w0=16.799999999999994, w1=1.4662858386217714e-14\n",
      "SubGD iter. 24/499: loss=57.26780585492638, w0=17.499999999999993, w1=1.5273810818976784e-14\n",
      "SubGD iter. 25/499: loss=56.567805854926384, w0=18.199999999999992, w1=1.5884763251735854e-14\n",
      "SubGD iter. 26/499: loss=55.867805854926395, w0=18.89999999999999, w1=1.6495715684494924e-14\n",
      "SubGD iter. 27/499: loss=55.167805854926385, w0=19.59999999999999, w1=1.7106668117253994e-14\n",
      "SubGD iter. 28/499: loss=54.46780585492638, w0=20.29999999999999, w1=1.7717620550013064e-14\n",
      "SubGD iter. 29/499: loss=53.767805854926394, w0=20.99999999999999, w1=1.8328572982772134e-14\n",
      "SubGD iter. 30/499: loss=53.06780585492639, w0=21.69999999999999, w1=1.8939525415531204e-14\n",
      "SubGD iter. 31/499: loss=52.367805854926395, w0=22.399999999999988, w1=1.9550477848290273e-14\n",
      "SubGD iter. 32/499: loss=51.667805854926385, w0=23.099999999999987, w1=2.0161430281049343e-14\n",
      "SubGD iter. 33/499: loss=50.96780585492639, w0=23.799999999999986, w1=2.0772382713808413e-14\n",
      "SubGD iter. 34/499: loss=50.267805854926394, w0=24.499999999999986, w1=2.1383335146567483e-14\n",
      "SubGD iter. 35/499: loss=49.56780585492639, w0=25.199999999999985, w1=2.1994287579326553e-14\n",
      "SubGD iter. 36/499: loss=48.867805854926395, w0=25.899999999999984, w1=2.2605240012085623e-14\n",
      "SubGD iter. 37/499: loss=48.1678058549264, w0=26.599999999999984, w1=2.3216192444844693e-14\n",
      "SubGD iter. 38/499: loss=47.4678058549264, w0=27.299999999999983, w1=2.3827144877603763e-14\n",
      "SubGD iter. 39/499: loss=46.7678058549264, w0=27.999999999999982, w1=2.4438097310362833e-14\n",
      "SubGD iter. 40/499: loss=46.067805854926405, w0=28.69999999999998, w1=2.5049049743121903e-14\n",
      "SubGD iter. 41/499: loss=45.367805854926395, w0=29.39999999999998, w1=2.5660002175880973e-14\n",
      "SubGD iter. 42/499: loss=44.6678058549264, w0=30.09999999999998, w1=2.6270954608640043e-14\n",
      "SubGD iter. 43/499: loss=43.9678058549264, w0=30.79999999999998, w1=2.6881907041399113e-14\n",
      "SubGD iter. 44/499: loss=43.2678058549264, w0=31.49999999999998, w1=2.7492859474158183e-14\n",
      "SubGD iter. 45/499: loss=42.567805854926405, w0=32.19999999999998, w1=2.8103811906917253e-14\n",
      "SubGD iter. 46/499: loss=41.867805854926395, w0=32.899999999999984, w1=2.871476433967632e-14\n",
      "SubGD iter. 47/499: loss=41.167805854926385, w0=33.59999999999999, w1=2.9325716772435396e-14\n",
      "SubGD iter. 48/499: loss=40.46780585492639, w0=34.29999999999999, w1=2.993666920519447e-14\n",
      "SubGD iter. 49/499: loss=39.767805854926394, w0=34.99999999999999, w1=3.054762163795354e-14\n",
      "SubGD iter. 50/499: loss=39.067805854926384, w0=35.699999999999996, w1=3.1158574070712615e-14\n",
      "SubGD iter. 51/499: loss=38.36780585492638, w0=36.4, w1=3.176952650347169e-14\n",
      "SubGD iter. 52/499: loss=37.66780585492638, w0=37.1, w1=3.238047893623076e-14\n",
      "SubGD iter. 53/499: loss=36.96780585492638, w0=37.800000000000004, w1=3.2991431368989835e-14\n",
      "SubGD iter. 54/499: loss=36.26780585492637, w0=38.50000000000001, w1=3.360238380174891e-14\n",
      "SubGD iter. 55/499: loss=35.56780585492637, w0=39.20000000000001, w1=3.421333623450798e-14\n",
      "SubGD iter. 56/499: loss=34.86780585492637, w0=39.90000000000001, w1=3.4824288667267054e-14\n",
      "SubGD iter. 57/499: loss=34.16780585492637, w0=40.600000000000016, w1=3.543524110002613e-14\n",
      "SubGD iter. 58/499: loss=33.46780585492636, w0=41.30000000000002, w1=3.60461935327852e-14\n",
      "SubGD iter. 59/499: loss=32.767805854926365, w0=42.00000000000002, w1=3.6657145965544273e-14\n",
      "SubGD iter. 60/499: loss=32.067805854926355, w0=42.700000000000024, w1=3.7268098398303347e-14\n",
      "SubGD iter. 61/499: loss=31.36780585492636, w0=43.40000000000003, w1=3.787905083106242e-14\n",
      "SubGD iter. 62/499: loss=30.66780585492635, w0=44.10000000000003, w1=3.849000326382149e-14\n",
      "SubGD iter. 63/499: loss=29.967805854926347, w0=44.80000000000003, w1=3.9100955696580566e-14\n",
      "SubGD iter. 64/499: loss=29.267805854926348, w0=45.500000000000036, w1=3.971190812933964e-14\n",
      "SubGD iter. 65/499: loss=28.567805854926345, w0=46.20000000000004, w1=4.032286056209871e-14\n",
      "SubGD iter. 66/499: loss=27.867805854926342, w0=46.90000000000004, w1=4.0933812994857785e-14\n",
      "SubGD iter. 67/499: loss=27.17327020966892, w0=47.59306930693074, w1=0.011147845678271063\n",
      "SubGD iter. 68/499: loss=26.490451563751197, w0=48.279207920792125, w1=0.03308574108989941\n",
      "SubGD iter. 69/499: loss=25.81721232277017, w0=48.96534653465351, w1=0.05502363650152776\n",
      "SubGD iter. 70/499: loss=25.15503943465645, w0=49.63069306930698, w1=0.10538326388307814\n",
      "SubGD iter. 71/499: loss=24.524103413894778, w0=50.28910891089114, w1=0.16746568532793435\n",
      "SubGD iter. 72/499: loss=23.899295346035593, w0=50.947524752475296, w1=0.22954810677279056\n",
      "SubGD iter. 73/499: loss=23.284392925657144, w0=51.59207920792084, w1=0.31242512932747524\n",
      "SubGD iter. 74/499: loss=22.686876444181845, w0=52.22277227722777, w1=0.4119501328839991\n",
      "SubGD iter. 75/499: loss=22.10626756964055, w0=52.84653465346539, w1=0.5208167847923756\n",
      "SubGD iter. 76/499: loss=21.537818828008433, w0=53.4564356435644, w1=0.6457900912635992\n",
      "SubGD iter. 77/499: loss=20.986339874628463, w0=54.0594059405941, w1=0.7796904498577214\n",
      "SubGD iter. 78/499: loss=20.445560936620446, w0=54.655445544554496, w1=0.9197570104995693\n",
      "SubGD iter. 79/499: loss=19.91191015895785, w0=55.24455445544559, w1=1.0670920297849913\n",
      "SubGD iter. 80/499: loss=19.389644090563234, w0=55.819801980198065, w1=1.2261255948210765\n",
      "SubGD iter. 81/499: loss=18.887989064395885, w0=56.36732673267331, w1=1.410709342622213\n",
      "SubGD iter. 82/499: loss=18.415960501854236, w0=56.900990099009945, w1=1.605853732220269\n",
      "SubGD iter. 83/499: loss=17.954898543040386, w0=57.42772277227727, w1=1.808762802293962\n",
      "SubGD iter. 84/499: loss=17.505757656579824, w0=57.933663366336674, w1=2.0285064197514697\n",
      "SubGD iter. 85/499: loss=17.07495742693161, w0=58.43267326732677, w1=2.2494370848672776\n",
      "SubGD iter. 86/499: loss=16.652967297509903, w0=58.91089108910895, w1=2.4837982986028337\n",
      "SubGD iter. 87/499: loss=16.24854073149673, w0=59.382178217821824, w1=2.7260245553531504\n",
      "SubGD iter. 88/499: loss=15.849105212654159, w0=59.83960396039608, w1=2.978742333469136\n",
      "SubGD iter. 89/499: loss=15.46691979123133, w0=60.262376237623805, w1=3.251528669355438\n",
      "SubGD iter. 90/499: loss=15.108294621512215, w0=60.67821782178222, w1=3.5270865794242794\n",
      "SubGD iter. 91/499: loss=14.754896345922832, w0=61.087128712871326, w1=3.806459183951815\n",
      "SubGD iter. 92/499: loss=14.40452896162028, w0=61.49603960396043, w1=4.085831788479351\n",
      "SubGD iter. 93/499: loss=14.055787028127279, w0=61.891089108910926, w1=4.373839384328607\n",
      "SubGD iter. 94/499: loss=13.714620911605635, w0=62.27920792079211, w1=4.666037469532047\n",
      "SubGD iter. 95/499: loss=13.381236307284155, w0=62.65346534653469, w1=4.959829093241769\n",
      "SubGD iter. 96/499: loss=13.058821615166238, w0=63.02079207920796, w1=5.25705719205664\n",
      "SubGD iter. 97/499: loss=12.74025172433924, w0=63.38118811881192, w1=5.560434316352406\n",
      "SubGD iter. 98/499: loss=12.42321888875611, w0=63.74158415841588, w1=5.863811440648173\n",
      "SubGD iter. 99/499: loss=12.107561731901173, w0=64.08811881188123, w1=6.172402175278548\n",
      "SubGD iter. 100/499: loss=11.800622097398135, w0=64.42772277227726, w1=6.486369310516498\n",
      "SubGD iter. 101/499: loss=11.495041794646427, w0=64.7673267326733, w1=6.800336445754448\n",
      "SubGD iter. 102/499: loss=11.189461491894715, w0=65.10693069306933, w1=7.114303580992399\n",
      "SubGD iter. 103/499: loss=10.883881189143004, w0=65.44653465346536, w1=7.428270716230349\n",
      "SubGD iter. 104/499: loss=10.584593408313202, w0=65.76534653465349, w1=7.747893210218626\n",
      "SubGD iter. 105/499: loss=10.295816534318941, w0=66.070297029703, w1=8.073669686866905\n",
      "SubGD iter. 106/499: loss=10.01135208122136, w0=66.37524752475251, w1=8.399446163515185\n",
      "SubGD iter. 107/499: loss=9.72808432666813, w0=66.6663366336634, w1=8.73297028041739\n",
      "SubGD iter. 108/499: loss=9.44812546112251, w0=66.9574257425743, w1=9.066494397319596\n",
      "SubGD iter. 109/499: loss=9.17104110409667, w0=67.23465346534658, w1=9.39863031947029\n",
      "SubGD iter. 110/499: loss=8.903656131158964, w0=67.51188118811886, w1=9.730766241620982\n",
      "SubGD iter. 111/499: loss=8.636271158221257, w0=67.78910891089114, w1=10.062902163771675\n",
      "SubGD iter. 112/499: loss=8.376151920302375, w0=68.06633663366343, w1=10.363999289979422\n",
      "SubGD iter. 113/499: loss=8.140540838751496, w0=68.32970297029709, w1=10.660466909273612\n",
      "SubGD iter. 114/499: loss=7.918544501597273, w0=68.59306930693076, w1=10.943174379960814\n",
      "SubGD iter. 115/499: loss=7.705279728377, w0=68.85643564356442, w1=11.225881850648015\n",
      "SubGD iter. 116/499: loss=7.493695831178641, w0=69.11287128712878, w1=11.504395843582206\n",
      "SubGD iter. 117/499: loss=7.289992405743416, w0=69.35544554455453, w1=11.78820189306775\n",
      "SubGD iter. 118/499: loss=7.097234035781543, w0=69.58415841584166, w1=12.060911465190971\n",
      "SubGD iter. 119/499: loss=6.919905294668923, w0=69.80594059405948, w1=12.324245668386048\n",
      "SubGD iter. 120/499: loss=6.750573527315454, w0=70.0277227722773, w1=12.587579871581125\n",
      "SubGD iter. 121/499: loss=6.584744810805664, w0=70.25643564356443, w1=12.824765405096484\n",
      "SubGD iter. 122/499: loss=6.430343276347806, w0=70.47821782178225, w1=13.065616959310148\n",
      "SubGD iter. 123/499: loss=6.278071481890353, w0=70.69306930693077, w1=13.302953389983912\n",
      "SubGD iter. 124/499: loss=6.133663329263324, w0=70.89405940594067, w1=13.525403099312918\n",
      "SubGD iter. 125/499: loss=6.00584079834303, w0=71.08811881188126, w1=13.742945617944212\n",
      "SubGD iter. 126/499: loss=5.885021825223219, w0=71.27524752475254, w1=13.953548196006844\n",
      "SubGD iter. 127/499: loss=5.771635252269658, w0=71.46237623762383, w1=14.164150774069476\n",
      "SubGD iter. 128/499: loss=5.667162061790257, w0=71.62178217821788, w1=14.349779559473173\n",
      "SubGD iter. 129/499: loss=5.586726765993146, w0=71.75346534653471, w1=14.51689010761231\n",
      "SubGD iter. 130/499: loss=5.523847812160388, w0=71.87128712871292, w1=14.670791185324186\n",
      "SubGD iter. 131/499: loss=5.480093708591872, w0=71.95445544554461, w1=14.780276456654521\n",
      "SubGD iter. 132/499: loss=5.4530880035020255, w0=72.0376237623763, w1=14.889761727984856\n",
      "SubGD iter. 133/499: loss=5.427392630862905, w0=72.10693069306937, w1=14.985916181776727\n",
      "SubGD iter. 134/499: loss=5.407322445682752, w0=72.17623762376245, w1=15.082070635568597\n",
      "SubGD iter. 135/499: loss=5.387252260502599, w0=72.24554455445552, w1=15.178225089360467\n",
      "SubGD iter. 136/499: loss=5.3704607803386955, w0=72.30099009900998, w1=15.25972348971591\n",
      "SubGD iter. 137/499: loss=5.357406523334741, w0=72.34950495049513, w1=15.335091856448138\n",
      "SubGD iter. 138/499: loss=5.345929264022584, w0=72.39801980198028, w1=15.410460223180365\n",
      "SubGD iter. 139/499: loss=5.335714659517473, w0=72.43267326732682, w1=15.469961786755725\n",
      "SubGD iter. 140/499: loss=5.330043910465361, w0=72.46039603960405, w1=15.51864528583281\n",
      "SubGD iter. 141/499: loss=5.325676428273225, w0=72.48811881188128, w1=15.561592159086487\n",
      "SubGD iter. 142/499: loss=5.322176726526591, w0=72.5019801980199, w1=15.597828332032526\n",
      "SubGD iter. 143/499: loss=5.320111309643114, w0=72.52277227722782, w1=15.624722856626713\n",
      "SubGD iter. 144/499: loss=5.318478284898438, w0=72.55049504950505, w1=15.642690329098\n",
      "SubGD iter. 145/499: loss=5.3172400485651465, w0=72.56435643564366, w1=15.664356578291091\n",
      "SubGD iter. 146/499: loss=5.316406547951545, w0=72.58514851485158, w1=15.677095775361284\n",
      "SubGD iter. 147/499: loss=5.315557122666144, w0=72.6059405940595, w1=15.689834972431477\n",
      "SubGD iter. 148/499: loss=5.31470769738074, w0=72.62673267326743, w1=15.70257416950167\n",
      "SubGD iter. 149/499: loss=5.313876880922167, w0=72.64059405940604, w1=15.72424041869476\n",
      "SubGD iter. 150/499: loss=5.3130522468713846, w0=72.66138613861396, w1=15.736979615764954\n",
      "SubGD iter. 151/499: loss=5.312377839024387, w0=72.66831683168327, w1=15.74811029423128\n",
      "SubGD iter. 152/499: loss=5.312132229725043, w0=72.67524752475258, w1=15.759240972697606\n",
      "SubGD iter. 153/499: loss=5.311886620425697, w0=72.68217821782189, w1=15.770371651163932\n",
      "SubGD iter. 154/499: loss=5.311683566098433, w0=72.68217821782189, w1=15.774323911906686\n",
      "SubGD iter. 155/499: loss=5.311661251291322, w0=72.68217821782189, w1=15.77827617264944\n",
      "SubGD iter. 156/499: loss=5.311638936484209, w0=72.68217821782189, w1=15.782228433392193\n",
      "SubGD iter. 157/499: loss=5.311616621677096, w0=72.68217821782189, w1=15.786180694134947\n",
      "SubGD iter. 158/499: loss=5.311594306869985, w0=72.68217821782189, w1=15.7901329548777\n",
      "SubGD iter. 159/499: loss=5.311571992062872, w0=72.68217821782189, w1=15.794085215620454\n",
      "SubGD iter. 160/499: loss=5.311549677255758, w0=72.68217821782189, w1=15.798037476363207\n",
      "SubGD iter. 161/499: loss=5.311527362448647, w0=72.68217821782189, w1=15.801989737105961\n",
      "SubGD iter. 162/499: loss=5.311505047641534, w0=72.68217821782189, w1=15.805941997848715\n",
      "SubGD iter. 163/499: loss=5.311482732834422, w0=72.68217821782189, w1=15.809894258591468\n",
      "SubGD iter. 164/499: loss=5.311460418027309, w0=72.68217821782189, w1=15.813846519334222\n",
      "SubGD iter. 165/499: loss=5.311438103220198, w0=72.68217821782189, w1=15.817798780076975\n",
      "SubGD iter. 166/499: loss=5.311415788413085, w0=72.68217821782189, w1=15.821751040819729\n",
      "SubGD iter. 167/499: loss=5.311393473605972, w0=72.68217821782189, w1=15.825703301562482\n",
      "SubGD iter. 168/499: loss=5.31137115879886, w0=72.68217821782189, w1=15.829655562305236\n",
      "SubGD iter. 169/499: loss=5.311348843991747, w0=72.68217821782189, w1=15.83360782304799\n",
      "SubGD iter. 170/499: loss=5.311326529184636, w0=72.68217821782189, w1=15.837560083790743\n",
      "SubGD iter. 171/499: loss=5.311304214377523, w0=72.68217821782189, w1=15.841512344533497\n",
      "SubGD iter. 172/499: loss=5.311281899570409, w0=72.68217821782189, w1=15.84546460527625\n",
      "SubGD iter. 173/499: loss=5.311259584763298, w0=72.68217821782189, w1=15.849416866019004\n",
      "SubGD iter. 174/499: loss=5.311237269956185, w0=72.68217821782189, w1=15.853369126761757\n",
      "SubGD iter. 175/499: loss=5.311214955149072, w0=72.68217821782189, w1=15.857321387504511\n",
      "SubGD iter. 176/499: loss=5.31119264034196, w0=72.68217821782189, w1=15.861273648247264\n",
      "SubGD iter. 177/499: loss=5.311170325534848, w0=72.68217821782189, w1=15.865225908990018\n",
      "SubGD iter. 178/499: loss=5.311148010727736, w0=72.68217821782189, w1=15.869178169732772\n",
      "SubGD iter. 179/499: loss=5.311125695920624, w0=72.68217821782189, w1=15.873130430475525\n",
      "SubGD iter. 180/499: loss=5.31110338111351, w0=72.68217821782189, w1=15.877082691218279\n",
      "SubGD iter. 181/499: loss=5.311081066306399, w0=72.68217821782189, w1=15.881034951961032\n",
      "SubGD iter. 182/499: loss=5.311058751499285, w0=72.68217821782189, w1=15.884987212703786\n",
      "SubGD iter. 183/499: loss=5.3110364366921745, w0=72.68217821782189, w1=15.88893947344654\n",
      "SubGD iter. 184/499: loss=5.311014121885061, w0=72.68217821782189, w1=15.892891734189293\n",
      "SubGD iter. 185/499: loss=5.310991807077948, w0=72.68217821782189, w1=15.896843994932047\n",
      "SubGD iter. 186/499: loss=5.310969492270837, w0=72.68217821782189, w1=15.9007962556748\n",
      "SubGD iter. 187/499: loss=5.310947177463723, w0=72.68217821782189, w1=15.904748516417554\n",
      "SubGD iter. 188/499: loss=5.310924862656612, w0=72.68217821782189, w1=15.908700777160307\n",
      "SubGD iter. 189/499: loss=5.310902547849499, w0=72.68217821782189, w1=15.91265303790306\n",
      "SubGD iter. 190/499: loss=5.310913706061381, w0=72.67524752475258, w1=15.910526938117323\n",
      "SubGD iter. 191/499: loss=5.310892237186269, w0=72.67524752475258, w1=15.914479198860077\n",
      "SubGD iter. 192/499: loss=5.3108699223791564, w0=72.67524752475258, w1=15.91843145960283\n",
      "SubGD iter. 193/499: loss=5.310862636053835, w0=72.66831683168327, w1=15.916305359817093\n",
      "SubGD iter. 194/499: loss=5.310859611715927, w0=72.66831683168327, w1=15.920257620559847\n",
      "SubGD iter. 195/499: loss=5.310837296908816, w0=72.66831683168327, w1=15.9242098813026\n",
      "SubGD iter. 196/499: loss=5.310814982101703, w0=72.66831683168327, w1=15.928162142045354\n",
      "SubGD iter. 197/499: loss=5.310823570190169, w0=72.66138613861396, w1=15.926036042259616\n",
      "SubGD iter. 198/499: loss=5.310804671438473, w0=72.66138613861396, w1=15.92998830300237\n",
      "SubGD iter. 199/499: loss=5.3107823566313614, w0=72.66138613861396, w1=15.933940563745123\n",
      "SubGD iter. 200/499: loss=5.310772500182623, w0=72.65445544554466, w1=15.931814463959386\n",
      "SubGD iter. 201/499: loss=5.3107720459681325, w0=72.65445544554466, w1=15.93576672470214\n",
      "SubGD iter. 202/499: loss=5.310749731161021, w0=72.65445544554466, w1=15.939718985444893\n",
      "SubGD iter. 203/499: loss=5.310727416353908, w0=72.65445544554466, w1=15.943671246187646\n",
      "SubGD iter. 204/499: loss=5.310733434318958, w0=72.64752475247535, w1=15.941545146401909\n",
      "SubGD iter. 205/499: loss=5.310717105690679, w0=72.64752475247535, w1=15.945497407144662\n",
      "SubGD iter. 206/499: loss=5.3106947908835656, w0=72.64752475247535, w1=15.949449667887416\n",
      "SubGD iter. 207/499: loss=5.310682364311412, w0=72.64059405940604, w1=15.947323568101679\n",
      "SubGD iter. 208/499: loss=5.310684480220337, w0=72.64059405940604, w1=15.951275828844432\n",
      "SubGD iter. 209/499: loss=5.310662165413225, w0=72.64059405940604, w1=15.955228089587186\n",
      "SubGD iter. 210/499: loss=5.310639850606112, w0=72.64059405940604, w1=15.95918035032994\n",
      "SubGD iter. 211/499: loss=5.310643298447746, w0=72.63366336633673, w1=15.957054250544202\n",
      "SubGD iter. 212/499: loss=5.310629539942882, w0=72.63366336633673, w1=15.961006511286955\n",
      "SubGD iter. 213/499: loss=5.3106072251357705, w0=72.63366336633673, w1=15.964958772029709\n",
      "SubGD iter. 214/499: loss=5.3105922284402, w0=72.62673267326743, w1=15.962832672243971\n",
      "SubGD iter. 215/499: loss=5.310633183099026, w0=72.63366336633673, w1=15.967301372051375\n",
      "SubGD iter. 216/499: loss=5.3105993435850625, w0=72.62673267326743, w1=15.965175272265638\n",
      "SubGD iter. 217/499: loss=5.31061822827579, w0=72.63366336633673, w1=15.969643972073042\n",
      "SubGD iter. 218/499: loss=5.310606458729927, w0=72.62673267326743, w1=15.967517872287305\n",
      "SubGD iter. 219/499: loss=5.3106032734525535, w0=72.63366336633673, w1=15.971986572094709\n",
      "SubGD iter. 220/499: loss=5.310613573874789, w0=72.62673267326743, w1=15.969860472308971\n",
      "SubGD iter. 221/499: loss=5.310588318629318, w0=72.63366336633673, w1=15.974329172116375\n",
      "SubGD iter. 222/499: loss=5.310620689019651, w0=72.62673267326743, w1=15.972203072330638\n",
      "SubGD iter. 223/499: loss=5.3105749661498844, w0=72.62673267326743, w1=15.970593411609551\n",
      "SubGD iter. 224/499: loss=5.31058363964973, w0=72.63366336633673, w1=15.975062111416955\n",
      "SubGD iter. 225/499: loss=5.310622915165495, w0=72.62673267326743, w1=15.972936011631218\n",
      "SubGD iter. 226/499: loss=5.310576651555033, w0=72.62673267326743, w1=15.97132635091013\n",
      "SubGD iter. 227/499: loss=5.310578960670142, w0=72.63366336633673, w1=15.975795050717535\n",
      "SubGD iter. 228/499: loss=5.310625141311338, w0=72.62673267326743, w1=15.973668950931797\n",
      "SubGD iter. 229/499: loss=5.31057833696018, w0=72.62673267326743, w1=15.97205929021071\n",
      "SubGD iter. 230/499: loss=5.310574635520699, w0=72.62673267326743, w1=15.970449629489623\n",
      "SubGD iter. 231/499: loss=5.310584557534202, w0=72.63366336633673, w1=15.974918329297028\n",
      "SubGD iter. 232/499: loss=5.31062247845816, w0=72.62673267326743, w1=15.97279222951129\n",
      "SubGD iter. 233/499: loss=5.310576320925847, w0=72.62673267326743, w1=15.971182568790203\n",
      "SubGD iter. 234/499: loss=5.3105798785546146, w0=72.63366336633673, w1=15.975651268597607\n",
      "SubGD iter. 235/499: loss=5.310624704604003, w0=72.62673267326743, w1=15.97352516881187\n",
      "SubGD iter. 236/499: loss=5.310578006330994, w0=72.62673267326743, w1=15.971915508090783\n",
      "SubGD iter. 237/499: loss=5.310575199575027, w0=72.63366336633673, w1=15.976384207898187\n",
      "SubGD iter. 238/499: loss=5.310626930749847, w0=72.62673267326743, w1=15.97425810811245\n",
      "SubGD iter. 239/499: loss=5.31057969173614, w0=72.62673267326743, w1=15.972648447391363\n",
      "SubGD iter. 240/499: loss=5.310575990296659, w0=72.62673267326743, w1=15.971038786670276\n",
      "SubGD iter. 241/499: loss=5.310580796439088, w0=72.63366336633673, w1=15.97550748647768\n",
      "SubGD iter. 242/499: loss=5.310624267896667, w0=72.62673267326743, w1=15.973381386691942\n",
      "SubGD iter. 243/499: loss=5.310577675701808, w0=72.62673267326743, w1=15.971771725970855\n",
      "SubGD iter. 244/499: loss=5.310576117459501, w0=72.63366336633673, w1=15.97624042577826\n",
      "SubGD iter. 245/499: loss=5.31062649404251, w0=72.62673267326743, w1=15.974114325992522\n",
      "SubGD iter. 246/499: loss=5.310579361106954, w0=72.62673267326743, w1=15.972504665271435\n",
      "SubGD iter. 247/499: loss=5.310575659667472, w0=72.62673267326743, w1=15.970895004550348\n",
      "SubGD iter. 248/499: loss=5.310581714323563, w0=72.63366336633673, w1=15.975363704357752\n",
      "SubGD iter. 249/499: loss=5.310623831189332, w0=72.62673267326743, w1=15.973237604572015\n",
      "SubGD iter. 250/499: loss=5.310577345072619, w0=72.62673267326743, w1=15.971627943850928\n",
      "SubGD iter. 251/499: loss=5.310577035343975, w0=72.63366336633673, w1=15.976096643658332\n",
      "SubGD iter. 252/499: loss=5.310626057335176, w0=72.62673267326743, w1=15.973970543872595\n",
      "SubGD iter. 253/499: loss=5.310579030477766, w0=72.62673267326743, w1=15.972360883151508\n",
      "SubGD iter. 254/499: loss=5.3105753290382856, w0=72.62673267326743, w1=15.97075122243042\n",
      "SubGD iter. 255/499: loss=5.310582632208036, w0=72.63366336633673, w1=15.975219922237825\n",
      "SubGD iter. 256/499: loss=5.310623394481999, w0=72.62673267326743, w1=15.973093822452087\n",
      "SubGD iter. 257/499: loss=5.3105770144434326, w0=72.62673267326743, w1=15.971484161731\n",
      "SubGD iter. 258/499: loss=5.3105779532284485, w0=72.63366336633673, w1=15.975952861538405\n",
      "SubGD iter. 259/499: loss=5.3106256206278415, w0=72.62673267326743, w1=15.973826761752667\n",
      "SubGD iter. 260/499: loss=5.31057869984858, w0=72.62673267326743, w1=15.97221710103158\n",
      "SubGD iter. 261/499: loss=5.310574998409098, w0=72.62673267326743, w1=15.970607440310493\n",
      "SubGD iter. 262/499: loss=5.3105835500925105, w0=72.63366336633673, w1=15.975076140117897\n",
      "SubGD iter. 263/499: loss=5.3106229577746635, w0=72.62673267326743, w1=15.97295004033216\n",
      "SubGD iter. 264/499: loss=5.310576683814244, w0=72.62673267326743, w1=15.971340379611073\n",
      "SubGD iter. 265/499: loss=5.310578871112923, w0=72.63366336633673, w1=15.975809079418477\n",
      "SubGD iter. 266/499: loss=5.310625183920505, w0=72.62673267326743, w1=15.97368297963274\n",
      "SubGD iter. 267/499: loss=5.310578369219393, w0=72.62673267326743, w1=15.972073318911653\n",
      "SubGD iter. 268/499: loss=5.310574667779911, w0=72.62673267326743, w1=15.970463658190566\n",
      "SubGD iter. 269/499: loss=5.310584467976984, w0=72.63366336633673, w1=15.97493235799797\n",
      "SubGD iter. 270/499: loss=5.3106225210673275, w0=72.62673267326743, w1=15.972806258212232\n",
      "SubGD iter. 271/499: loss=5.310576353185058, w0=72.62673267326743, w1=15.971196597491145\n",
      "SubGD iter. 272/499: loss=5.310579788997396, w0=72.63366336633673, w1=15.97566529729855\n",
      "SubGD iter. 273/499: loss=5.310624747213171, w0=72.62673267326743, w1=15.973539197512812\n",
      "SubGD iter. 274/499: loss=5.310578038590206, w0=72.62673267326743, w1=15.971929536791725\n",
      "SubGD iter. 275/499: loss=5.310575110017808, w0=72.63366336633673, w1=15.97639823659913\n",
      "SubGD iter. 276/499: loss=5.310626973359014, w0=72.62673267326743, w1=15.974272136813392\n",
      "SubGD iter. 277/499: loss=5.310579723995353, w0=72.62673267326743, w1=15.972662476092305\n",
      "SubGD iter. 278/499: loss=5.310576022555871, w0=72.62673267326743, w1=15.971052815371218\n",
      "SubGD iter. 279/499: loss=5.31058070688187, w0=72.63366336633673, w1=15.975521515178622\n",
      "SubGD iter. 280/499: loss=5.310624310505837, w0=72.62673267326743, w1=15.973395415392885\n",
      "SubGD iter. 281/499: loss=5.310577707961019, w0=72.62673267326743, w1=15.971785754671798\n",
      "SubGD iter. 282/499: loss=5.310576027902282, w0=72.63366336633673, w1=15.976254454479202\n",
      "SubGD iter. 283/499: loss=5.31062653665168, w0=72.62673267326743, w1=15.974128354693464\n",
      "SubGD iter. 284/499: loss=5.310579393366166, w0=72.62673267326743, w1=15.972518693972377\n",
      "SubGD iter. 285/499: loss=5.310575691926685, w0=72.62673267326743, w1=15.97090903325129\n",
      "SubGD iter. 286/499: loss=5.3105816247663435, w0=72.63366336633673, w1=15.975377733058695\n",
      "SubGD iter. 287/499: loss=5.310623873798502, w0=72.62673267326743, w1=15.973251633272957\n",
      "SubGD iter. 288/499: loss=5.310577377331832, w0=72.62673267326743, w1=15.97164197255187\n",
      "SubGD iter. 289/499: loss=5.310576945786757, w0=72.63366336633673, w1=15.976110672359274\n",
      "SubGD iter. 290/499: loss=5.3106260999443435, w0=72.62673267326743, w1=15.973984572573537\n",
      "SubGD iter. 291/499: loss=5.310579062736979, w0=72.62673267326743, w1=15.97237491185245\n",
      "SubGD iter. 292/499: loss=5.310575361297499, w0=72.62673267326743, w1=15.970765251131363\n",
      "SubGD iter. 293/499: loss=5.310582542650818, w0=72.63366336633673, w1=15.975233950938767\n",
      "SubGD iter. 294/499: loss=5.310623437091167, w0=72.62673267326743, w1=15.97310785115303\n",
      "SubGD iter. 295/499: loss=5.310577046702645, w0=72.62673267326743, w1=15.971498190431943\n",
      "SubGD iter. 296/499: loss=5.31057786367123, w0=72.63366336633673, w1=15.975966890239347\n",
      "SubGD iter. 297/499: loss=5.310625663237008, w0=72.62673267326743, w1=15.97384079045361\n",
      "SubGD iter. 298/499: loss=5.310578732107793, w0=72.62673267326743, w1=15.972231129732522\n",
      "SubGD iter. 299/499: loss=5.310575030668312, w0=72.62673267326743, w1=15.970621469011435\n",
      "SubGD iter. 300/499: loss=5.31058346053529, w0=72.63366336633673, w1=15.97509016881884\n",
      "SubGD iter. 301/499: loss=5.310623000383831, w0=72.62673267326743, w1=15.972964069033102\n",
      "SubGD iter. 302/499: loss=5.3105767160734585, w0=72.62673267326743, w1=15.971354408312015\n",
      "SubGD iter. 303/499: loss=5.310578781555702, w0=72.63366336633673, w1=15.97582310811942\n",
      "SubGD iter. 304/499: loss=5.310625226529674, w0=72.62673267326743, w1=15.973697008333682\n",
      "SubGD iter. 305/499: loss=5.3105784014786055, w0=72.62673267326743, w1=15.972087347612595\n",
      "SubGD iter. 306/499: loss=5.310574700039124, w0=72.62673267326743, w1=15.970477686891508\n",
      "SubGD iter. 307/499: loss=5.310584378419764, w0=72.63366336633673, w1=15.974946386698912\n",
      "SubGD iter. 308/499: loss=5.310622563676499, w0=72.62673267326743, w1=15.972820286913175\n",
      "SubGD iter. 309/499: loss=5.3105763854442705, w0=72.62673267326743, w1=15.971210626192088\n",
      "SubGD iter. 310/499: loss=5.310579699440178, w0=72.63366336633673, w1=15.975679325999492\n",
      "SubGD iter. 311/499: loss=5.31062478982234, w0=72.62673267326743, w1=15.973553226213754\n",
      "SubGD iter. 312/499: loss=5.3105780708494175, w0=72.62673267326743, w1=15.971943565492667\n",
      "SubGD iter. 313/499: loss=5.310575020460591, w0=72.63366336633673, w1=15.976412265300072\n",
      "SubGD iter. 314/499: loss=5.3106270159681825, w0=72.62673267326743, w1=15.974286165514334\n",
      "SubGD iter. 315/499: loss=5.310579756254566, w0=72.62673267326743, w1=15.972676504793247\n",
      "SubGD iter. 316/499: loss=5.310576054815085, w0=72.62673267326743, w1=15.97106684407216\n",
      "SubGD iter. 317/499: loss=5.310580617324651, w0=72.63366336633673, w1=15.975535543879564\n",
      "SubGD iter. 318/499: loss=5.3106243531150055, w0=72.62673267326743, w1=15.973409444093827\n",
      "SubGD iter. 319/499: loss=5.310577740220231, w0=72.62673267326743, w1=15.97179978337274\n",
      "SubGD iter. 320/499: loss=5.310575938345063, w0=72.63366336633673, w1=15.976268483180144\n",
      "SubGD iter. 321/499: loss=5.310626579260847, w0=72.62673267326743, w1=15.974142383394407\n",
      "SubGD iter. 322/499: loss=5.31057942562538, w0=72.62673267326743, w1=15.97253272267332\n",
      "SubGD iter. 323/499: loss=5.310575724185897, w0=72.62673267326743, w1=15.970923061952233\n",
      "SubGD iter. 324/499: loss=5.310581535209124, w0=72.63366336633673, w1=15.975391761759637\n",
      "SubGD iter. 325/499: loss=5.310623916407671, w0=72.62673267326743, w1=15.9732656619739\n",
      "SubGD iter. 326/499: loss=5.310577409591045, w0=72.62673267326743, w1=15.971656001252812\n",
      "SubGD iter. 327/499: loss=5.310576856229536, w0=72.63366336633673, w1=15.976124701060217\n",
      "SubGD iter. 328/499: loss=5.310626142553512, w0=72.62673267326743, w1=15.973998601274479\n",
      "SubGD iter. 329/499: loss=5.310579094996192, w0=72.62673267326743, w1=15.972388940553392\n",
      "SubGD iter. 330/499: loss=5.31057539355671, w0=72.62673267326743, w1=15.970779279832305\n",
      "SubGD iter. 331/499: loss=5.310582453093599, w0=72.63366336633673, w1=15.97524797963971\n",
      "SubGD iter. 332/499: loss=5.310623479700335, w0=72.62673267326743, w1=15.973121879853972\n",
      "SubGD iter. 333/499: loss=5.310577078961858, w0=72.62673267326743, w1=15.971512219132885\n",
      "SubGD iter. 334/499: loss=5.3105777741140106, w0=72.63366336633673, w1=15.975980918940289\n",
      "SubGD iter. 335/499: loss=5.310625705846178, w0=72.62673267326743, w1=15.973854819154552\n",
      "SubGD iter. 336/499: loss=5.310578764367005, w0=72.62673267326743, w1=15.972245158433465\n",
      "SubGD iter. 337/499: loss=5.310575062927524, w0=72.62673267326743, w1=15.970635497712378\n",
      "SubGD iter. 338/499: loss=5.3105833709780725, w0=72.63366336633673, w1=15.975104197519782\n",
      "SubGD iter. 339/499: loss=5.310623042993, w0=72.62673267326743, w1=15.972978097734044\n",
      "SubGD iter. 340/499: loss=5.31057674833267, w0=72.62673267326743, w1=15.971368437012957\n",
      "SubGD iter. 341/499: loss=5.310578691998486, w0=72.63366336633673, w1=15.975837136820362\n",
      "SubGD iter. 342/499: loss=5.310625269138844, w0=72.62673267326743, w1=15.973711037034624\n",
      "SubGD iter. 343/499: loss=5.310578433737819, w0=72.62673267326743, w1=15.972101376313537\n",
      "SubGD iter. 344/499: loss=5.3105747322983365, w0=72.62673267326743, w1=15.97049171559245\n",
      "SubGD iter. 345/499: loss=5.310584288862546, w0=72.63366336633673, w1=15.974960415399854\n",
      "SubGD iter. 346/499: loss=5.3106226062856665, w0=72.62673267326743, w1=15.972834315614117\n",
      "SubGD iter. 347/499: loss=5.310576417703483, w0=72.62673267326743, w1=15.97122465489303\n",
      "SubGD iter. 348/499: loss=5.310579609882959, w0=72.63366336633673, w1=15.975693354700434\n",
      "SubGD iter. 349/499: loss=5.310624832431509, w0=72.62673267326743, w1=15.973567254914697\n",
      "SubGD iter. 350/499: loss=5.3105781031086305, w0=72.62673267326743, w1=15.97195759419361\n",
      "SubGD iter. 351/499: loss=5.310574930903372, w0=72.63366336633673, w1=15.976426294001014\n",
      "SubGD iter. 352/499: loss=5.310627058577351, w0=72.62673267326743, w1=15.974300194215276\n",
      "SubGD iter. 353/499: loss=5.3105797885137775, w0=72.62673267326743, w1=15.97269053349419\n",
      "SubGD iter. 354/499: loss=5.310576087074297, w0=72.62673267326743, w1=15.971080872773102\n",
      "SubGD iter. 355/499: loss=5.310580527767432, w0=72.63366336633673, w1=15.975549572580507\n",
      "SubGD iter. 356/499: loss=5.310624395724173, w0=72.62673267326743, w1=15.973423472794769\n",
      "SubGD iter. 357/499: loss=5.310577772479444, w0=72.62673267326743, w1=15.971813812073682\n",
      "SubGD iter. 358/499: loss=5.3105758487878445, w0=72.63366336633673, w1=15.976282511881086\n",
      "SubGD iter. 359/499: loss=5.310626621870017, w0=72.62673267326743, w1=15.974156412095349\n",
      "SubGD iter. 360/499: loss=5.31057945788459, w0=72.62673267326743, w1=15.972546751374262\n",
      "SubGD iter. 361/499: loss=5.31057575644511, w0=72.62673267326743, w1=15.970937090653175\n",
      "SubGD iter. 362/499: loss=5.310581445651906, w0=72.63366336633673, w1=15.975405790460579\n",
      "SubGD iter. 363/499: loss=5.310623959016839, w0=72.62673267326743, w1=15.973279690674842\n",
      "SubGD iter. 364/499: loss=5.310577441850257, w0=72.62673267326743, w1=15.971670029953755\n",
      "SubGD iter. 365/499: loss=5.310576766672319, w0=72.63366336633673, w1=15.976138729761159\n",
      "SubGD iter. 366/499: loss=5.310626185162682, w0=72.62673267326743, w1=15.974012629975421\n",
      "SubGD iter. 367/499: loss=5.310579127255404, w0=72.62673267326743, w1=15.972402969254334\n",
      "SubGD iter. 368/499: loss=5.310575425815924, w0=72.62673267326743, w1=15.970793308533247\n",
      "SubGD iter. 369/499: loss=5.310582363536381, w0=72.63366336633673, w1=15.975262008340652\n",
      "SubGD iter. 370/499: loss=5.310623522309504, w0=72.62673267326743, w1=15.973135908554914\n",
      "SubGD iter. 371/499: loss=5.31057711122107, w0=72.62673267326743, w1=15.971526247833827\n",
      "SubGD iter. 372/499: loss=5.310577684556793, w0=72.63366336633673, w1=15.975994947641231\n",
      "SubGD iter. 373/499: loss=5.310625748455347, w0=72.62673267326743, w1=15.973868847855494\n",
      "SubGD iter. 374/499: loss=5.310578796626218, w0=72.62673267326743, w1=15.972259187134407\n",
      "SubGD iter. 375/499: loss=5.310575095186736, w0=72.62673267326743, w1=15.97064952641332\n",
      "SubGD iter. 376/499: loss=5.310583281420854, w0=72.63366336633673, w1=15.975118226220724\n",
      "SubGD iter. 377/499: loss=5.3106230856021694, w0=72.62673267326743, w1=15.972992126434987\n",
      "SubGD iter. 378/499: loss=5.310576780591885, w0=72.62673267326743, w1=15.9713824657139\n",
      "SubGD iter. 379/499: loss=5.310578602441266, w0=72.63366336633673, w1=15.975851165521304\n",
      "SubGD iter. 380/499: loss=5.310625311748012, w0=72.62673267326743, w1=15.973725065735566\n",
      "SubGD iter. 381/499: loss=5.310578465997031, w0=72.62673267326743, w1=15.97211540501448\n",
      "SubGD iter. 382/499: loss=5.31057476455755, w0=72.62673267326743, w1=15.970505744293392\n",
      "SubGD iter. 383/499: loss=5.310584199305326, w0=72.63366336633673, w1=15.974974444100797\n",
      "SubGD iter. 384/499: loss=5.310622648894835, w0=72.62673267326743, w1=15.972848344315059\n",
      "SubGD iter. 385/499: loss=5.310576449962697, w0=72.62673267326743, w1=15.971238683593972\n",
      "SubGD iter. 386/499: loss=5.31057952032574, w0=72.63366336633673, w1=15.975707383401376\n",
      "SubGD iter. 387/499: loss=5.310624875040677, w0=72.62673267326743, w1=15.973581283615639\n",
      "SubGD iter. 388/499: loss=5.3105781353678445, w0=72.62673267326743, w1=15.971971622894552\n",
      "SubGD iter. 389/499: loss=5.310574841346153, w0=72.63366336633673, w1=15.976440322701956\n",
      "SubGD iter. 390/499: loss=5.31062710118652, w0=72.62673267326743, w1=15.974314222916218\n",
      "SubGD iter. 391/499: loss=5.3105798207729915, w0=72.62673267326743, w1=15.972704562195132\n",
      "SubGD iter. 392/499: loss=5.3105761193335095, w0=72.62673267326743, w1=15.971094901474045\n",
      "SubGD iter. 393/499: loss=5.310580438210215, w0=72.63366336633673, w1=15.975563601281449\n",
      "SubGD iter. 394/499: loss=5.310624438333343, w0=72.62673267326743, w1=15.973437501495711\n",
      "SubGD iter. 395/499: loss=5.3105778047386565, w0=72.62673267326743, w1=15.971827840774624\n",
      "SubGD iter. 396/499: loss=5.310575759230627, w0=72.63366336633673, w1=15.976296540582029\n",
      "SubGD iter. 397/499: loss=5.310626664479185, w0=72.62673267326743, w1=15.974170440796291\n",
      "SubGD iter. 398/499: loss=5.3105794901438035, w0=72.62673267326743, w1=15.972560780075204\n",
      "SubGD iter. 399/499: loss=5.310575788704323, w0=72.62673267326743, w1=15.970951119354117\n",
      "SubGD iter. 400/499: loss=5.310581356094687, w0=72.63366336633673, w1=15.975419819161521\n",
      "SubGD iter. 401/499: loss=5.310624001626008, w0=72.62673267326743, w1=15.973293719375784\n",
      "SubGD iter. 402/499: loss=5.31057747410947, w0=72.62673267326743, w1=15.971684058654697\n",
      "SubGD iter. 403/499: loss=5.310576677115099, w0=72.63366336633673, w1=15.976152758462101\n",
      "SubGD iter. 404/499: loss=5.31062622777185, w0=72.62673267326743, w1=15.974026658676364\n",
      "SubGD iter. 405/499: loss=5.310579159514617, w0=72.62673267326743, w1=15.972416997955277\n",
      "SubGD iter. 406/499: loss=5.310575458075135, w0=72.62673267326743, w1=15.97080733723419\n",
      "SubGD iter. 407/499: loss=5.310582273979162, w0=72.63366336633673, w1=15.975276037041594\n",
      "SubGD iter. 408/499: loss=5.310623564918673, w0=72.62673267326743, w1=15.973149937255856\n",
      "SubGD iter. 409/499: loss=5.310577143480284, w0=72.62673267326743, w1=15.97154027653477\n",
      "SubGD iter. 410/499: loss=5.3105775949995735, w0=72.63366336633673, w1=15.976008976342174\n",
      "SubGD iter. 411/499: loss=5.310625791064514, w0=72.62673267326743, w1=15.973882876556436\n",
      "SubGD iter. 412/499: loss=5.31057882888543, w0=72.62673267326743, w1=15.972273215835349\n",
      "SubGD iter. 413/499: loss=5.310575127445949, w0=72.62673267326743, w1=15.970663555114262\n",
      "SubGD iter. 414/499: loss=5.310583191863635, w0=72.63366336633673, w1=15.975132254921666\n",
      "SubGD iter. 415/499: loss=5.310623128211338, w0=72.62673267326743, w1=15.973006155135929\n",
      "SubGD iter. 416/499: loss=5.310576812851096, w0=72.62673267326743, w1=15.971396494414842\n",
      "SubGD iter. 417/499: loss=5.310578512884048, w0=72.63366336633673, w1=15.975865194222246\n",
      "SubGD iter. 418/499: loss=5.31062535435718, w0=72.62673267326743, w1=15.973739094436509\n",
      "SubGD iter. 419/499: loss=5.310578498256244, w0=72.62673267326743, w1=15.972129433715422\n",
      "SubGD iter. 420/499: loss=5.310574796816762, w0=72.62673267326743, w1=15.970519772994335\n",
      "SubGD iter. 421/499: loss=5.31058410974811, w0=72.63366336633673, w1=15.974988472801739\n",
      "SubGD iter. 422/499: loss=5.310622691504003, w0=72.62673267326743, w1=15.972862373016001\n",
      "SubGD iter. 423/499: loss=5.310576482221909, w0=72.62673267326743, w1=15.971252712294914\n",
      "SubGD iter. 424/499: loss=5.310579430768521, w0=72.63366336633673, w1=15.975721412102319\n",
      "SubGD iter. 425/499: loss=5.310624917649847, w0=72.62673267326743, w1=15.973595312316581\n",
      "SubGD iter. 426/499: loss=5.310578167627056, w0=72.62673267326743, w1=15.971985651595494\n",
      "SubGD iter. 427/499: loss=5.310574751788934, w0=72.63366336633673, w1=15.976454351402898\n",
      "SubGD iter. 428/499: loss=5.310627143795689, w0=72.62673267326743, w1=15.97432825161716\n",
      "SubGD iter. 429/499: loss=5.310579853032204, w0=72.62673267326743, w1=15.972718590896074\n",
      "SubGD iter. 430/499: loss=5.310576151592722, w0=72.62673267326743, w1=15.971108930174987\n",
      "SubGD iter. 431/499: loss=5.310580348652994, w0=72.63366336633673, w1=15.975577629982391\n",
      "SubGD iter. 432/499: loss=5.310624480942511, w0=72.62673267326743, w1=15.973451530196654\n",
      "SubGD iter. 433/499: loss=5.31057783699787, w0=72.62673267326743, w1=15.971841869475567\n",
      "SubGD iter. 434/499: loss=5.3105756696734066, w0=72.63366336633673, w1=15.97631056928297\n",
      "SubGD iter. 435/499: loss=5.310626707088354, w0=72.62673267326743, w1=15.974184469497233\n",
      "SubGD iter. 436/499: loss=5.310579522403018, w0=72.62673267326743, w1=15.972574808776146\n",
      "SubGD iter. 437/499: loss=5.310575820963536, w0=72.62673267326743, w1=15.97096514805506\n",
      "SubGD iter. 438/499: loss=5.3105812665374685, w0=72.63366336633673, w1=15.975433847862464\n",
      "SubGD iter. 439/499: loss=5.310624044235177, w0=72.62673267326743, w1=15.973307748076726\n",
      "SubGD iter. 440/499: loss=5.310577506368682, w0=72.62673267326743, w1=15.97169808735564\n",
      "SubGD iter. 441/499: loss=5.31057658755788, w0=72.63366336633673, w1=15.976166787163043\n",
      "SubGD iter. 442/499: loss=5.310626270381019, w0=72.62673267326743, w1=15.974040687377306\n",
      "SubGD iter. 443/499: loss=5.31057919177383, w0=72.62673267326743, w1=15.972431026656219\n",
      "SubGD iter. 444/499: loss=5.310575490334348, w0=72.62673267326743, w1=15.970821365935132\n",
      "SubGD iter. 445/499: loss=5.310582184421943, w0=72.63366336633673, w1=15.975290065742536\n",
      "SubGD iter. 446/499: loss=5.310623607527842, w0=72.62673267326743, w1=15.973163965956799\n",
      "SubGD iter. 447/499: loss=5.310577175739496, w0=72.62673267326743, w1=15.971554305235712\n",
      "SubGD iter. 448/499: loss=5.310577505442355, w0=72.63366336633673, w1=15.976023005043116\n",
      "SubGD iter. 449/499: loss=5.310625833673684, w0=72.62673267326743, w1=15.973896905257378\n",
      "SubGD iter. 450/499: loss=5.310578861144643, w0=72.62673267326743, w1=15.972287244536291\n",
      "SubGD iter. 451/499: loss=5.310575159705162, w0=72.62673267326743, w1=15.970677583815204\n",
      "SubGD iter. 452/499: loss=5.310583102306416, w0=72.63366336633673, w1=15.975146283622609\n",
      "SubGD iter. 453/499: loss=5.310623170820506, w0=72.62673267326743, w1=15.973020183836871\n",
      "SubGD iter. 454/499: loss=5.310576845110308, w0=72.62673267326743, w1=15.971410523115784\n",
      "SubGD iter. 455/499: loss=5.310578423326827, w0=72.63366336633673, w1=15.975879222923188\n",
      "SubGD iter. 456/499: loss=5.3106253969663495, w0=72.62673267326743, w1=15.97375312313745\n",
      "SubGD iter. 457/499: loss=5.310578530515456, w0=72.62673267326743, w1=15.972143462416364\n",
      "SubGD iter. 458/499: loss=5.310574829075975, w0=72.62673267326743, w1=15.970533801695277\n",
      "SubGD iter. 459/499: loss=5.3105840201908885, w0=72.63366336633673, w1=15.975002501502681\n",
      "SubGD iter. 460/499: loss=5.3106227341131715, w0=72.62673267326743, w1=15.972876401716944\n",
      "SubGD iter. 461/499: loss=5.310576514481122, w0=72.62673267326743, w1=15.971266740995857\n",
      "SubGD iter. 462/499: loss=5.3105793412113025, w0=72.63366336633673, w1=15.97573544080326\n",
      "SubGD iter. 463/499: loss=5.310624960259014, w0=72.62673267326743, w1=15.973609341017523\n",
      "SubGD iter. 464/499: loss=5.310578199886269, w0=72.62673267326743, w1=15.971999680296436\n",
      "SubGD iter. 465/499: loss=5.310574662231715, w0=72.63366336633673, w1=15.97646838010384\n",
      "SubGD iter. 466/499: loss=5.310627186404858, w0=72.62673267326743, w1=15.974342280318103\n",
      "SubGD iter. 467/499: loss=5.310579885291417, w0=72.62673267326743, w1=15.972732619597016\n",
      "SubGD iter. 468/499: loss=5.310576183851936, w0=72.62673267326743, w1=15.97112295887593\n",
      "SubGD iter. 469/499: loss=5.310580259095775, w0=72.63366336633673, w1=15.975591658683333\n",
      "SubGD iter. 470/499: loss=5.310624523551678, w0=72.62673267326743, w1=15.973465558897596\n",
      "SubGD iter. 471/499: loss=5.310577869257083, w0=72.62673267326743, w1=15.971855898176509\n",
      "SubGD iter. 472/499: loss=5.310575580116187, w0=72.63366336633673, w1=15.976324597983913\n",
      "SubGD iter. 473/499: loss=5.310626749697523, w0=72.62673267326743, w1=15.974198498198175\n",
      "SubGD iter. 474/499: loss=5.3105795546622305, w0=72.62673267326743, w1=15.972588837477089\n",
      "SubGD iter. 475/499: loss=5.3105758532227485, w0=72.62673267326743, w1=15.970979176756002\n",
      "SubGD iter. 476/499: loss=5.310581176980249, w0=72.63366336633673, w1=15.975447876563406\n",
      "SubGD iter. 477/499: loss=5.310624086844345, w0=72.62673267326743, w1=15.973321776777668\n",
      "SubGD iter. 478/499: loss=5.3105775386278955, w0=72.62673267326743, w1=15.971712116056581\n",
      "SubGD iter. 479/499: loss=5.3105764980006605, w0=72.63366336633673, w1=15.976180815863986\n",
      "SubGD iter. 480/499: loss=5.310626312990188, w0=72.62673267326743, w1=15.974054716078248\n",
      "SubGD iter. 481/499: loss=5.3105792240330425, w0=72.62673267326743, w1=15.972445055357161\n",
      "SubGD iter. 482/499: loss=5.3105755225935605, w0=72.62673267326743, w1=15.970835394636074\n",
      "SubGD iter. 483/499: loss=5.310582094864723, w0=72.63366336633673, w1=15.975304094443478\n",
      "SubGD iter. 484/499: loss=5.31062365013701, w0=72.62673267326743, w1=15.97317799465774\n",
      "SubGD iter. 485/499: loss=5.3105772079987075, w0=72.62673267326743, w1=15.971568333936654\n",
      "SubGD iter. 486/499: loss=5.3105774158851355, w0=72.63366336633673, w1=15.976037033744058\n",
      "SubGD iter. 487/499: loss=5.310625876282853, w0=72.62673267326743, w1=15.97391093395832\n",
      "SubGD iter. 488/499: loss=5.310578893403856, w0=72.62673267326743, w1=15.972301273237234\n",
      "SubGD iter. 489/499: loss=5.310575191964374, w0=72.62673267326743, w1=15.970691612516147\n",
      "SubGD iter. 490/499: loss=5.3105830127491975, w0=72.63366336633673, w1=15.97516031232355\n",
      "SubGD iter. 491/499: loss=5.310623213429675, w0=72.62673267326743, w1=15.973034212537813\n",
      "SubGD iter. 492/499: loss=5.310576877369521, w0=72.62673267326743, w1=15.971424551816726\n",
      "SubGD iter. 493/499: loss=5.310578333769609, w0=72.63366336633673, w1=15.97589325162413\n",
      "SubGD iter. 494/499: loss=5.310625439575518, w0=72.62673267326743, w1=15.973767151838393\n",
      "SubGD iter. 495/499: loss=5.310578562774669, w0=72.62673267326743, w1=15.972157491117306\n",
      "SubGD iter. 496/499: loss=5.310574861335188, w0=72.62673267326743, w1=15.97054783039622\n",
      "SubGD iter. 497/499: loss=5.310583930633672, w0=72.63366336633673, w1=15.975016530203623\n",
      "SubGD iter. 498/499: loss=5.310622776722341, w0=72.62673267326743, w1=15.972890430417886\n",
      "SubGD iter. 499/499: loss=5.310576546740335, w0=72.62673267326743, w1=15.971280769696799\n",
      "SubGD: execution time=0.043 seconds\n"
     ]
    }
   ],
   "source": [
    "# Define the parameters of the algorithm.\n",
    "max_iters = 500\n",
    "gamma = 0.7\n",
    "batch_size = 1\n",
    "\n",
    "# Initialization\n",
    "w_initial = np.array([0, 0])\n",
    "\n",
    "# Start SubSGD.\n",
    "start_time = datetime.datetime.now()\n",
    "subgd_losses, subgd_ws = subgradient_descent(y, tx, w_initial, max_iters, gamma)\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "# Print result\n",
    "exection_time = (end_time - start_time).total_seconds()\n",
    "print(\"SubGD: execution time={t:.3f} seconds\".format(t=exection_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c890ebb26a04b14bfce8b6ce77f6e14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=1, description='n_iter', max=501, min=1), Output()), _dom_classes=('widg…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_figure(n_iter)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ipywidgets import IntSlider, interact\n",
    "\n",
    "\n",
    "def plot_figure(n_iter):\n",
    "    fig = gradient_descent_visualization(\n",
    "        subgd_losses,\n",
    "        subgd_ws,\n",
    "        grid_losses,\n",
    "        grid_w0,\n",
    "        grid_w1,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        height,\n",
    "        weight,\n",
    "        n_iter,\n",
    "    )\n",
    "    fig.set_size_inches(10.0, 6.0)\n",
    "\n",
    "\n",
    "interact(plot_figure, n_iter=IntSlider(min=1, max=len(subgd_ws)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stochastic Subgradient Descent\n",
    "\n",
    "**NB** for the computation of the subgradient you can reuse the `compute_subgradient` method that you implemented above, just making sure that you pass in a minibatch as opposed to the full data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stochastic_subgradient_descent(y, tx, initial_w, batch_size, max_iters, gamma):\n",
    "    \"\"\"Compute a stochastic subgradient at w from a data sample batch of size B, where B < N, and their corresponding labels.\n",
    "\n",
    "    Args:\n",
    "        y: numpy array of shape=(B, )\n",
    "        tx: numpy array of shape=(B,2)\n",
    "        initial_w: numpy array of shape=(2, ). The initial guess (or the initialization) for the model parameters\n",
    "        batch_size: a scalar denoting the number of data points in a mini-batch used for computing the stochastic subgradient\n",
    "        max_iters: a scalar denoting the total number of iterations of SubSGD\n",
    "        gamma: a scalar denoting the stepsize\n",
    "\n",
    "    Returns:\n",
    "        losses: a list of length max_iters containing the loss value (scalar) for each iteration of SubSGD\n",
    "        ws: a list of length max_iters containing the model parameters as numpy arrays of shape (2, ), for each iteration of SubSGD\n",
    "    \"\"\"\n",
    "\n",
    "    # Define parameters to store w and loss\n",
    "    ws = [initial_w]\n",
    "    losses = []\n",
    "    w = initial_w\n",
    "\n",
    "    for n_iter in range(max_iters):\n",
    "        ### SOLUTION\n",
    "        for y_batch, tx_batch in batch_iter(\n",
    "            y, tx, batch_size=batch_size, num_batches=1\n",
    "        ):\n",
    "            # compute a stochastic subgradient and loss\n",
    "            grad, err = compute_subgradient_mae(y_batch, tx_batch, w)\n",
    "            # update w through the stochastic subgradient update\n",
    "            w = w - gamma * grad\n",
    "            # calculate loss\n",
    "            loss = calculate_mae(err)\n",
    "            # store w and loss\n",
    "            ws.append(w)\n",
    "            losses.append(loss)\n",
    "\n",
    "        ### TEMPLATE\n",
    "        # # ***************************************************\n",
    "        # # INSERT YOUR CODE HERE\n",
    "        # # TODO: implement stochastic subgradient descent.\n",
    "        # # ***************************************************\n",
    "        # raise NotImplementedError\n",
    "        ### END SOLUTION\n",
    "\n",
    "        print(\n",
    "            \"SubSGD iter. {bi}/{ti}: loss={l}, w0={w0}, w1={w1}\".format(\n",
    "                bi=n_iter, ti=max_iters - 1, l=loss, w0=w[0], w1=w[1]\n",
    "            )\n",
    "        )\n",
    "    return losses, ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SubSGD iter. 0/499: loss=61.176769716949444, w0=0.7, w1=-0.42578614729073927\n",
      "SubSGD iter. 1/499: loss=88.01184769210465, w0=1.4, w1=0.3201995062966577\n",
      "SubSGD iter. 2/499: loss=76.66475829417655, w0=2.0999999999999996, w1=0.5226363963376568\n",
      "SubSGD iter. 3/499: loss=92.46960137823967, w0=2.8, w1=1.7173752334420822\n",
      "SubSGD iter. 4/499: loss=61.162562433493946, w0=3.5, w1=1.5669185569909547\n",
      "SubSGD iter. 5/499: loss=87.26502181203385, w0=4.2, w1=2.4002561395505317\n",
      "SubSGD iter. 6/499: loss=81.26090797775943, w0=4.9, w1=2.866838265517107\n",
      "SubSGD iter. 7/499: loss=88.88551369491859, w0=5.6000000000000005, w1=3.879276603383528\n",
      "SubSGD iter. 8/499: loss=69.44984886817775, w0=6.300000000000001, w1=4.234304080913354\n",
      "SubSGD iter. 9/499: loss=68.82061884611149, w0=7.000000000000001, w1=4.499089190104122\n",
      "SubSGD iter. 10/499: loss=53.06304336027652, w0=7.700000000000001, w1=3.5222917111622034\n",
      "SubSGD iter. 11/499: loss=44.31361789586226, w0=8.4, w1=2.3963592976610326\n",
      "SubSGD iter. 12/499: loss=47.92594821486605, w0=9.1, w1=1.9645667193776122\n",
      "SubSGD iter. 13/499: loss=43.16072384403251, w0=9.799999999999999, w1=0.7805645189837207\n",
      "SubSGD iter. 14/499: loss=49.20956045830216, w0=10.499999999999998, w1=0.15951294541312722\n",
      "SubSGD iter. 15/499: loss=64.00402597357758, w0=11.199999999999998, w1=0.2918364717669659\n",
      "SubSGD iter. 16/499: loss=73.50293745046478, w0=11.899999999999997, w1=0.8215374458290821\n",
      "SubSGD iter. 17/499: loss=54.917309107326965, w0=12.599999999999996, w1=0.740723140340371\n",
      "SubSGD iter. 18/499: loss=42.70649339899344, w0=13.299999999999995, w1=0.1935608381065007\n",
      "SubSGD iter. 19/499: loss=86.54690809679278, w0=13.999999999999995, w1=1.5563122334982475\n",
      "SubSGD iter. 20/499: loss=69.42613970672586, w0=14.699999999999994, w1=2.503224495240678\n",
      "SubSGD iter. 21/499: loss=43.85699342933127, w0=15.399999999999993, w1=1.7033937611779686\n",
      "SubSGD iter. 22/499: loss=73.53359599722462, w0=16.099999999999994, w1=2.3180547423083553\n",
      "SubSGD iter. 23/499: loss=59.151020653568004, w0=16.799999999999994, w1=2.2856257345409956\n",
      "SubSGD iter. 24/499: loss=57.36920421439075, w0=17.499999999999993, w1=2.8340202382988062\n",
      "SubSGD iter. 25/499: loss=52.13911109035481, w0=18.199999999999992, w1=3.0327811041173307\n",
      "SubSGD iter. 26/499: loss=62.64803780073467, w0=18.89999999999999, w1=3.9909161950191376\n",
      "SubSGD iter. 27/499: loss=52.02704138801819, w0=19.59999999999999, w1=3.4637275562719108\n",
      "SubSGD iter. 28/499: loss=57.555663948185654, w0=20.29999999999999, w1=3.66616444631291\n",
      "SubSGD iter. 29/499: loss=37.18942889835945, w0=20.99999999999999, w1=2.9626595279220473\n",
      "SubSGD iter. 30/499: loss=39.154358416421886, w0=21.69999999999999, w1=2.577375387593919\n",
      "SubSGD iter. 31/499: loss=39.0323131220013, w0=22.399999999999988, w1=2.2321982998689207\n",
      "SubSGD iter. 32/499: loss=30.31340438717053, w0=23.099999999999987, w1=1.0481960994750292\n",
      "SubSGD iter. 33/499: loss=28.962112314490625, w0=23.799999999999986, w1=0.20677758974908456\n",
      "SubSGD iter. 34/499: loss=33.43164694895873, w0=24.499999999999986, w1=0.08688577626075335\n",
      "SubSGD iter. 35/499: loss=74.77798798856563, w0=25.199999999999985, w1=1.2051959968722348\n",
      "SubSGD iter. 36/499: loss=43.62904133203003, w0=25.899999999999984, w1=0.6780073581250077\n",
      "SubSGD iter. 37/499: loss=51.39955828208889, w0=26.599999999999984, w1=0.8438136176925047\n",
      "SubSGD iter. 38/499: loss=58.368091870528076, w0=27.299999999999983, w1=1.4629470136371712\n",
      "SubSGD iter. 39/499: loss=61.5738939405696, w0=27.999999999999982, w1=2.1150039110796266\n",
      "SubSGD iter. 40/499: loss=57.580971417506014, w0=28.69999999999998, w1=2.3261683056458304\n",
      "SubSGD iter. 41/499: loss=46.55139653186586, w0=29.39999999999998, w1=2.2937392978784708\n",
      "SubSGD iter. 42/499: loss=39.18692109776738, w0=30.09999999999998, w1=2.0889057780032476\n",
      "SubSGD iter. 43/499: loss=60.38563744869519, w0=30.79999999999998, w1=2.62986282997439\n",
      "SubSGD iter. 44/499: loss=45.90363113590851, w0=31.49999999999998, w1=2.858984522261148\n",
      "SubSGD iter. 45/499: loss=32.35657687733361, w0=32.19999999999998, w1=2.4279738375125346\n",
      "SubSGD iter. 46/499: loss=39.73959708259166, w0=32.899999999999984, w1=2.327696738272325\n",
      "SubSGD iter. 47/499: loss=29.692628325878307, w0=33.59999999999999, w1=1.9019105909815859\n",
      "SubSGD iter. 48/499: loss=38.124841329949675, w0=34.29999999999999, w1=1.7184448119311666\n",
      "SubSGD iter. 49/499: loss=27.9220411395265, w0=34.99999999999999, w1=1.2926586646404274\n",
      "SubSGD iter. 50/499: loss=48.94559960699614, w0=35.699999999999996, w1=1.8223596387025436\n",
      "SubSGD iter. 51/499: loss=41.26375746060471, w0=36.4, w1=1.8376673054146795\n",
      "SubSGD iter. 52/499: loss=27.5884177839974, w0=37.1, w1=1.687210628963552\n",
      "SubSGD iter. 53/499: loss=17.472418945152945, w0=37.800000000000004, w1=0.7855783645507943\n",
      "SubSGD iter. 54/499: loss=35.227280779053174, w0=38.50000000000001, w1=0.7508875052093232\n",
      "SubSGD iter. 55/499: loss=42.41525035238408, w0=39.20000000000001, w1=0.916340297141949\n",
      "SubSGD iter. 56/499: loss=40.150243839499424, w0=39.90000000000001, w1=0.9536276087598187\n",
      "SubSGD iter. 57/499: loss=60.05613183224501, w0=40.600000000000016, w1=1.9663059501125857\n",
      "SubSGD iter. 58/499: loss=27.239121203801304, w0=41.30000000000002, w1=1.613190198545271\n",
      "SubSGD iter. 59/499: loss=20.906598013207443, w0=42.00000000000002, w1=1.3691633738361202\n",
      "SubSGD iter. 60/499: loss=33.65902239101538, w0=42.700000000000024, w1=1.1281834591694722\n",
      "SubSGD iter. 61/499: loss=49.3654117894962, w0=43.40000000000003, w1=1.8601480526771073\n",
      "SubSGD iter. 62/499: loss=16.001354234368442, w0=44.10000000000003, w1=1.580219060240644\n",
      "SubSGD iter. 63/499: loss=37.39638509836186, w0=44.80000000000003, w1=1.9587588435973755\n",
      "SubSGD iter. 64/499: loss=26.003365153363625, w0=45.500000000000036, w1=1.6223958859212988\n",
      "SubSGD iter. 65/499: loss=7.493038058100289, w0=46.20000000000004, w1=0.7823683144614879\n",
      "SubSGD iter. 66/499: loss=24.138992683223286, w0=46.90000000000004, w1=0.6721941975807475\n",
      "SubSGD iter. 67/499: loss=31.531245794325905, w0=47.600000000000044, w1=0.815524788175264\n",
      "SubSGD iter. 68/499: loss=8.89821324208819, w0=48.30000000000005, w1=0.28000510534905854\n",
      "SubSGD iter. 69/499: loss=46.144466372179714, w0=49.00000000000005, w1=1.1362271578535967\n",
      "SubSGD iter. 70/499: loss=0.8243069916754422, w0=48.30000000000005, w1=2.2621595713547675\n",
      "SubSGD iter. 71/499: loss=33.22537554756729, w0=49.00000000000005, w1=2.415053328469349\n",
      "SubSGD iter. 72/499: loss=13.263762912214837, w0=49.70000000000005, w1=2.1325055093831238\n",
      "SubSGD iter. 73/499: loss=39.0046552899796, w0=50.400000000000055, w1=2.7559936316693014\n",
      "SubSGD iter. 74/499: loss=6.174700834534249, w0=51.10000000000006, w1=2.052488713278439\n",
      "SubSGD iter. 75/499: loss=18.367160037433706, w0=51.80000000000006, w1=1.5253000745312117\n",
      "SubSGD iter. 76/499: loss=22.44584630332953, w0=52.500000000000064, w1=1.6576236008850505\n",
      "SubSGD iter. 77/499: loss=0.9030309563637857, w0=51.80000000000006, w1=3.0115804949182783\n",
      "SubSGD iter. 78/499: loss=37.58849483710228, w0=52.500000000000064, w1=3.8067502200356254\n",
      "SubSGD iter. 79/499: loss=34.20720717250604, w0=53.20000000000007, w1=4.05106294374476\n",
      "SubSGD iter. 80/499: loss=6.4210237007584325, w0=53.90000000000007, w1=3.2668302157054354\n",
      "SubSGD iter. 81/499: loss=32.06265959233325, w0=54.60000000000007, w1=4.146560345537914\n",
      "SubSGD iter. 82/499: loss=25.30072527938973, w0=55.300000000000075, w1=4.343080428316417\n",
      "SubSGD iter. 83/499: loss=22.36703028136632, w0=56.00000000000008, w1=5.000263131511401\n",
      "SubSGD iter. 84/499: loss=2.830207450724963, w0=56.70000000000008, w1=4.296758213120539\n",
      "SubSGD iter. 85/499: loss=32.07941838514029, w0=57.400000000000084, w1=4.837715265091681\n",
      "SubSGD iter. 86/499: loss=8.004031837036507, w0=58.10000000000009, w1=4.2339491951251675\n",
      "SubSGD iter. 87/499: loss=41.40746670751365, w0=58.80000000000009, w1=5.774011533512353\n",
      "SubSGD iter. 88/499: loss=0.43123845205315803, w0=58.10000000000009, w1=6.252059735938476\n",
      "SubSGD iter. 89/499: loss=3.5648685173115098, w0=57.400000000000084, w1=6.932658913594559\n",
      "SubSGD iter. 90/499: loss=26.103385845070648, w0=58.10000000000009, w1=7.57880614698178\n",
      "SubSGD iter. 91/499: loss=31.985435807932134, w0=58.80000000000009, w1=8.94417559964539\n",
      "SubSGD iter. 92/499: loss=5.119724315552752, w0=59.50000000000009, w1=8.042543335232633\n",
      "SubSGD iter. 93/499: loss=20.081045701949193, w0=60.200000000000095, w1=8.656457748610215\n",
      "SubSGD iter. 94/499: loss=3.327109502765367, w0=59.50000000000009, w1=9.337056926266298\n",
      "SubSGD iter. 95/499: loss=21.783964239571247, w0=60.200000000000095, w1=9.983204159653518\n",
      "SubSGD iter. 96/499: loss=8.52642024940264, w0=60.9000000000001, w1=9.782454622604266\n",
      "SubSGD iter. 97/499: loss=16.837166341803467, w0=61.6000000000001, w1=10.516730366096066\n",
      "SubSGD iter. 98/499: loss=22.37265010054564, w0=62.300000000000104, w1=11.057687418067209\n",
      "SubSGD iter. 99/499: loss=3.4522735024664044, w0=63.00000000000011, w1=10.775139598980983\n",
      "SubSGD iter. 100/499: loss=27.134337141946688, w0=63.70000000000011, w1=11.95594429968088\n",
      "SubSGD iter. 101/499: loss=16.616145908884718, w0=64.4000000000001, w1=13.074254520292362\n",
      "SubSGD iter. 102/499: loss=2.335719482859922, w0=65.10000000000011, w1=12.234226948832552\n",
      "SubSGD iter. 103/499: loss=12.85378908850317, w0=65.80000000000011, w1=11.838250705610719\n",
      "SubSGD iter. 104/499: loss=5.293701479287158, w0=66.50000000000011, w1=11.067427639542531\n",
      "SubSGD iter. 105/499: loss=9.963601122689525, w0=67.20000000000012, w1=11.813413293129928\n",
      "SubSGD iter. 106/499: loss=6.575915168685327, w0=67.90000000000012, w1=12.444578633124781\n",
      "SubSGD iter. 107/499: loss=14.953121955602285, w0=68.60000000000012, w1=13.057571130308341\n",
      "SubSGD iter. 108/499: loss=6.562565657388959, w0=69.30000000000013, w1=12.50346405631176\n",
      "SubSGD iter. 109/499: loss=0.5865175399819762, w0=68.60000000000012, w1=13.160345024064723\n",
      "SubSGD iter. 110/499: loss=5.095703363925736, w0=69.30000000000013, w1=14.244220420756026\n",
      "SubSGD iter. 111/499: loss=2.894979720065173, w0=68.60000000000012, w1=14.655295605298251\n",
      "SubSGD iter. 112/499: loss=7.1322354661310925, w0=69.30000000000013, w1=15.105475351504545\n",
      "SubSGD iter. 113/499: loss=4.351550292024385, w0=70.00000000000013, w1=14.729889513217975\n",
      "SubSGD iter. 114/499: loss=8.614429390687008, w0=70.70000000000013, w1=14.767176824835845\n",
      "SubSGD iter. 115/499: loss=3.596534260435554, w0=71.40000000000013, w1=14.414061073268531\n",
      "SubSGD iter. 116/499: loss=3.304830016255096, w0=72.10000000000014, w1=13.735714077293988\n",
      "SubSGD iter. 117/499: loss=2.726409866729341, w0=72.80000000000014, w1=13.552248298243569\n",
      "SubSGD iter. 118/499: loss=5.985943497614649, w0=72.10000000000014, w1=13.89933228386356\n",
      "SubSGD iter. 119/499: loss=2.5127015529703414, w0=71.40000000000013, w1=14.339020688948745\n",
      "SubSGD iter. 120/499: loss=4.939308237576647, w0=70.70000000000013, w1=14.75009587349097\n",
      "SubSGD iter. 121/499: loss=4.377762070689812, w0=70.00000000000013, w1=15.586134730763787\n",
      "SubSGD iter. 122/499: loss=3.056512622981849, w0=69.30000000000013, w1=15.736591407214915\n",
      "SubSGD iter. 123/499: loss=3.824023278622718, w0=70.00000000000013, w1=16.080304028398306\n",
      "SubSGD iter. 124/499: loss=5.117945837781818, w0=70.70000000000013, w1=16.936526080902844\n",
      "SubSGD iter. 125/499: loss=5.118870075351566, w0=71.40000000000013, w1=17.579455779666638\n",
      "SubSGD iter. 126/499: loss=8.032535172800074, w0=72.10000000000014, w1=16.995143242120747\n",
      "SubSGD iter. 127/499: loss=2.6060299292995452, w0=71.40000000000013, w1=17.41838266689326\n",
      "SubSGD iter. 128/499: loss=1.942964016759987, w0=72.10000000000014, w1=17.399803207994385\n",
      "SubSGD iter. 129/499: loss=6.534721841214889, w0=72.80000000000014, w1=18.062947525037433\n",
      "SubSGD iter. 130/499: loss=2.104897332894396, w0=72.10000000000014, w1=17.534891709526786\n",
      "SubSGD iter. 131/499: loss=3.6324068896008797, w0=72.80000000000014, w1=16.735060975464076\n",
      "SubSGD iter. 132/499: loss=1.9259534033930734, w0=72.10000000000014, w1=17.262018011232925\n",
      "SubSGD iter. 133/499: loss=0.524848591516033, w0=72.80000000000014, w1=16.719001549867826\n",
      "SubSGD iter. 134/499: loss=1.1868296599578372, w0=73.50000000000014, w1=16.618724450627617\n",
      "SubSGD iter. 135/499: loss=8.735595286267142, w0=74.20000000000014, w1=16.863037174336753\n",
      "SubSGD iter. 136/499: loss=9.562484466697583, w0=73.50000000000014, w1=15.694706373819285\n",
      "SubSGD iter. 137/499: loss=5.268271858682439, w0=74.20000000000014, w1=16.2479122898954\n",
      "SubSGD iter. 138/499: loss=6.581136168355897, w0=74.90000000000015, w1=16.00693237522875\n",
      "SubSGD iter. 139/499: loss=2.8051512547274484, w0=74.20000000000014, w1=17.132864788729922\n",
      "SubSGD iter. 140/499: loss=6.885787279289005, w0=74.90000000000015, w1=16.891884874063273\n",
      "SubSGD iter. 141/499: loss=0.2899495393280205, w0=74.20000000000014, w1=17.26747071234984\n",
      "SubSGD iter. 142/499: loss=4.79902065835892, w0=73.50000000000014, w1=17.81944214428752\n",
      "SubSGD iter. 143/499: loss=1.007949040565606, w0=72.80000000000014, w1=17.76728179875781\n",
      "SubSGD iter. 144/499: loss=2.094665426483189, w0=72.10000000000014, w1=17.95144257707613\n",
      "SubSGD iter. 145/499: loss=10.161349429573448, w0=72.80000000000014, w1=17.42680965236439\n",
      "SubSGD iter. 146/499: loss=5.825499715604096, w0=72.10000000000014, w1=16.220793362593945\n",
      "SubSGD iter. 147/499: loss=5.223562395972195, w0=72.80000000000014, w1=16.744973914045662\n",
      "SubSGD iter. 148/499: loss=1.8654655707977525, w0=72.10000000000014, w1=16.494692797583607\n",
      "SubSGD iter. 149/499: loss=0.8937713853899965, w0=72.80000000000014, w1=17.328030380143183\n",
      "SubSGD iter. 150/499: loss=8.191757811992375, w0=73.50000000000014, w1=17.539194774709387\n",
      "SubSGD iter. 151/499: loss=2.4097963533223634, w0=72.80000000000014, w1=17.01113895919874\n",
      "SubSGD iter. 152/499: loss=6.153798490801506, w0=73.50000000000014, w1=17.55209601116988\n",
      "SubSGD iter. 153/499: loss=8.03999960078606, w0=72.80000000000014, w1=17.353335145351355\n",
      "SubSGD iter. 154/499: loss=2.2035693377311816, w0=72.10000000000014, w1=17.537495923669674\n",
      "SubSGD iter. 155/499: loss=1.7765870541516051, w0=72.80000000000014, w1=16.91644435009908\n",
      "SubSGD iter. 156/499: loss=1.2345282115750038, w0=73.50000000000014, w1=16.132211622059756\n",
      "SubSGD iter. 157/499: loss=3.811788203711032, w0=72.80000000000014, w1=15.227036518872692\n",
      "SubSGD iter. 158/499: loss=1.328048036857986, w0=72.10000000000014, w1=15.658047203621306\n",
      "SubSGD iter. 159/499: loss=5.055827507142752, w0=71.40000000000013, w1=15.00086450042632\n",
      "SubSGD iter. 160/499: loss=2.142032930979852, w0=70.70000000000013, w1=15.570824624962306\n",
      "SubSGD iter. 161/499: loss=9.848043985902464, w0=71.40000000000013, w1=15.329844710295658\n",
      "SubSGD iter. 162/499: loss=5.139007947194372, w0=72.10000000000014, w1=14.118421822369479\n",
      "SubSGD iter. 163/499: loss=1.58803477479168, w0=72.80000000000014, w1=14.083730963028009\n",
      "SubSGD iter. 164/499: loss=0.30333211679500494, w0=73.50000000000014, w1=13.708145124741439\n",
      "SubSGD iter. 165/499: loss=7.335907426423304, w0=74.20000000000014, w1=13.821965451555752\n",
      "SubSGD iter. 166/499: loss=4.651941556398839, w0=74.90000000000015, w1=13.425989208333919\n",
      "SubSGD iter. 167/499: loss=4.537006086257939, w0=74.20000000000014, w1=13.856999893082532\n",
      "SubSGD iter. 168/499: loss=10.804595283024554, w0=73.50000000000014, w1=14.288792471365953\n",
      "SubSGD iter. 169/499: loss=9.793573833335671, w0=72.80000000000014, w1=13.683389319921968\n",
      "SubSGD iter. 170/499: loss=0.08582501584086799, w0=73.50000000000014, w1=14.76726471661327\n",
      "SubSGD iter. 171/499: loss=1.9434218357578885, w0=72.80000000000014, w1=13.561248426842822\n",
      "SubSGD iter. 172/499: loss=5.4798686476611, w0=72.10000000000014, w1=14.401275998302632\n",
      "SubSGD iter. 173/499: loss=8.200241232568658, w0=72.80000000000014, w1=14.051338066822314\n",
      "SubSGD iter. 174/499: loss=8.441120155317932, w0=73.50000000000014, w1=14.592295118793457\n",
      "SubSGD iter. 175/499: loss=0.9598301666104732, w0=74.20000000000014, w1=14.970834902150187\n",
      "SubSGD iter. 176/499: loss=10.609593779731142, w0=73.50000000000014, w1=15.256009396699481\n",
      "SubSGD iter. 177/499: loss=7.674094352591197, w0=74.20000000000014, w1=15.452353990216709\n",
      "SubSGD iter. 178/499: loss=1.5520748646633251, w0=73.50000000000014, w1=15.56252810709745\n",
      "SubSGD iter. 179/499: loss=3.0218087168896375, w0=72.80000000000014, w1=14.970210145829272\n",
      "SubSGD iter. 180/499: loss=0.9302480161028939, w0=73.50000000000014, w1=14.935519286487802\n",
      "SubSGD iter. 181/499: loss=7.097996116626, w0=72.80000000000014, w1=15.482681588721672\n",
      "SubSGD iter. 182/499: loss=9.062659903687276, w0=73.50000000000014, w1=15.100555604171337\n",
      "SubSGD iter. 183/499: loss=7.149819030860172, w0=74.20000000000014, w1=14.75061767269102\n",
      "SubSGD iter. 184/499: loss=9.407050967740346, w0=74.90000000000015, w1=15.098584719081433\n",
      "SubSGD iter. 185/499: loss=1.2521468850541453, w0=74.20000000000014, w1=15.474170557368003\n",
      "SubSGD iter. 186/499: loss=11.879338547902222, w0=74.90000000000015, w1=16.713486858346005\n",
      "SubSGD iter. 187/499: loss=4.0335447569765535, w0=75.60000000000015, w1=15.502063970419826\n",
      "SubSGD iter. 188/499: loss=5.096939681430669, w0=74.90000000000015, w1=14.596888867232762\n",
      "SubSGD iter. 189/499: loss=2.5246002104425074, w0=74.20000000000014, w1=15.396719601295471\n",
      "SubSGD iter. 190/499: loss=1.1351122911875677, w0=73.50000000000014, w1=14.671699411214668\n",
      "SubSGD iter. 191/499: loss=2.477493483304542, w0=74.20000000000014, w1=15.79000963182615\n",
      "SubSGD iter. 192/499: loss=4.526021504757928, w0=73.50000000000014, w1=16.493514550217014\n",
      "SubSGD iter. 193/499: loss=1.0500512688523145, w0=74.20000000000014, w1=16.94369429642331\n",
      "SubSGD iter. 194/499: loss=0.8200126604935747, w0=74.90000000000015, w1=17.822942262319483\n",
      "SubSGD iter. 195/499: loss=8.607988830625715, w0=74.20000000000014, w1=16.616925972549037\n",
      "SubSGD iter. 196/499: loss=1.3815581309754918, w0=74.90000000000015, w1=16.433460193498618\n",
      "SubSGD iter. 197/499: loss=13.73269049240914, w0=74.20000000000014, w1=16.377150578453314\n",
      "SubSGD iter. 198/499: loss=3.8214212389378517, w0=73.50000000000014, w1=16.9471107029893\n",
      "SubSGD iter. 199/499: loss=9.162083356678266, w0=74.20000000000014, w1=16.564984718438964\n",
      "SubSGD iter. 200/499: loss=119.05908545140606, w0=74.90000000000015, w1=13.191841040849287\n",
      "SubSGD iter. 201/499: loss=1.25788505886581, w0=74.20000000000014, w1=13.226531900190757\n",
      "SubSGD iter. 202/499: loss=2.9126631672177012, w0=73.50000000000014, w1=13.22010401931241\n",
      "SubSGD iter. 203/499: loss=0.6311912466148328, w0=74.20000000000014, w1=13.954379762804209\n",
      "SubSGD iter. 204/499: loss=0.6092107019036348, w0=73.50000000000014, w1=14.054656862044418\n",
      "SubSGD iter. 205/499: loss=3.330876066551852, w0=72.80000000000014, w1=14.711537829797381\n",
      "SubSGD iter. 206/499: loss=14.629543077813068, w0=73.50000000000014, w1=15.950854130775381\n",
      "SubSGD iter. 207/499: loss=2.5367198578270376, w0=74.20000000000014, w1=16.567241079612632\n",
      "SubSGD iter. 208/499: loss=6.521177133603828, w0=73.50000000000014, w1=17.190027486432925\n",
      "SubSGD iter. 209/499: loss=0.5541372697508962, w0=72.80000000000014, w1=17.394861006308147\n",
      "SubSGD iter. 210/499: loss=0.8793825547381928, w0=72.10000000000014, w1=18.234888577767958\n",
      "SubSGD iter. 211/499: loss=3.2212143691917703, w0=71.40000000000013, w1=17.393698409543315\n",
      "SubSGD iter. 212/499: loss=2.4503521453489157, w0=72.10000000000014, w1=17.359007550201845\n",
      "SubSGD iter. 213/499: loss=6.928248248248096, w0=72.80000000000014, w1=18.53981225090174\n",
      "SubSGD iter. 214/499: loss=6.728472061078001, w0=72.10000000000014, w1=18.819741243338203\n",
      "SubSGD iter. 215/499: loss=1.5110510518628573, w0=71.40000000000013, w1=19.30942477673479\n",
      "SubSGD iter. 216/499: loss=6.438982725031025, w0=70.70000000000013, w1=19.110663910916266\n",
      "SubSGD iter. 217/499: loss=1.2609434671095983, w0=71.40000000000013, w1=18.71522711897326\n",
      "SubSGD iter. 218/499: loss=3.142609505808295, w0=72.10000000000014, w1=18.058346151220295\n",
      "SubSGD iter. 219/499: loss=3.213837875355445, w0=72.80000000000014, w1=17.27411342318097\n",
      "SubSGD iter. 220/499: loss=1.804983596072347, w0=73.50000000000014, w1=18.153361389077144\n",
      "SubSGD iter. 221/499: loss=0.5972959596320209, w0=74.20000000000014, w1=18.769748337914393\n",
      "SubSGD iter. 222/499: loss=5.98851771810763, w0=74.90000000000015, w1=18.96609293143162\n",
      "SubSGD iter. 223/499: loss=4.042708906369356, w0=75.60000000000015, w1=18.195269865363436\n",
      "SubSGD iter. 224/499: loss=9.338345473660922, w0=74.90000000000015, w1=18.412526869543186\n",
      "SubSGD iter. 225/499: loss=5.170972758348128, w0=75.60000000000015, w1=18.5263471963575\n",
      "SubSGD iter. 226/499: loss=1.4591051889710087, w0=76.30000000000015, w1=19.707151897057393\n",
      "SubSGD iter. 227/499: loss=1.8879917865718454, w0=75.60000000000015, w1=19.618624383568267\n",
      "SubSGD iter. 228/499: loss=9.824522535288956, w0=76.30000000000015, w1=18.879291584124324\n",
      "SubSGD iter. 229/499: loss=6.128579012729176, w0=75.60000000000015, w1=18.62901046766227\n",
      "SubSGD iter. 230/499: loss=8.452225405395943, w0=74.90000000000015, w1=19.107058670088392\n",
      "SubSGD iter. 231/499: loss=3.6666622037588894, w0=74.20000000000014, w1=19.100630789210047\n",
      "SubSGD iter. 232/499: loss=1.8892473735778026, w0=73.50000000000014, w1=18.221382823313874\n",
      "SubSGD iter. 233/499: loss=7.71553148585447, w0=74.20000000000014, w1=19.460699124291875\n",
      "SubSGD iter. 234/499: loss=4.002441433400165, w0=73.50000000000014, w1=18.930998150229758\n",
      "SubSGD iter. 235/499: loss=10.349233928678515, w0=72.80000000000014, w1=17.736259313125334\n",
      "SubSGD iter. 236/499: loss=1.8584550326673082, w0=72.10000000000014, w1=17.208203497614686\n",
      "SubSGD iter. 237/499: loss=4.8078682292167585, w0=71.40000000000013, w1=17.358660174065815\n",
      "SubSGD iter. 238/499: loss=3.1866533429028294, w0=70.70000000000013, w1=17.003632696535988\n",
      "SubSGD iter. 239/499: loss=0.8195000823848844, w0=71.40000000000013, w1=16.577846549245248\n",
      "SubSGD iter. 240/499: loss=3.8099285469712783, w0=72.10000000000014, w1=16.224730797677932\n",
      "SubSGD iter. 241/499: loss=0.37500797509943595, w0=72.80000000000014, w1=15.134935774604712\n",
      "SubSGD iter. 242/499: loss=4.491937304931255, w0=73.50000000000014, w1=16.014183740500883\n",
      "SubSGD iter. 243/499: loss=13.56095506538395, w0=72.80000000000014, w1=16.134075553989213\n",
      "SubSGD iter. 244/499: loss=5.285161794848179, w0=72.10000000000014, w1=16.54515073853144\n",
      "SubSGD iter. 245/499: loss=0.873853770578819, w0=72.80000000000014, w1=15.45535571545822\n",
      "SubSGD iter. 246/499: loss=0.5562971743927463, w0=73.50000000000014, w1=14.65552498139551\n",
      "SubSGD iter. 247/499: loss=0.8745156021812761, w0=72.80000000000014, w1=13.921249237903712\n",
      "SubSGD iter. 248/499: loss=5.605839615425083, w0=73.50000000000014, w1=13.394060599156486\n",
      "SubSGD iter. 249/499: loss=1.2323332134515113, w0=72.80000000000014, w1=12.789528493302326\n",
      "SubSGD iter. 250/499: loss=13.134420048025369, w0=72.10000000000014, w1=12.803283689277816\n",
      "SubSGD iter. 251/499: loss=6.088113136832938, w0=72.80000000000014, w1=12.11658905377168\n",
      "SubSGD iter. 252/499: loss=0.5275202676501038, w0=73.50000000000014, w1=12.01631195453147\n",
      "SubSGD iter. 253/499: loss=7.702636880883148, w0=72.80000000000014, w1=11.755599733149607\n",
      "SubSGD iter. 254/499: loss=10.840384374012721, w0=72.10000000000014, w1=12.027079915132186\n",
      "SubSGD iter. 255/499: loss=4.006242257140258, w0=72.80000000000014, w1=12.170410505726702\n",
      "SubSGD iter. 256/499: loss=4.757795653629202, w0=73.50000000000014, w1=11.431077706282757\n",
      "SubSGD iter. 257/499: loss=2.3440542916268825, w0=74.20000000000014, w1=10.404145508690224\n",
      "SubSGD iter. 258/499: loss=2.7426057477485273, w0=73.50000000000014, w1=9.457233246947794\n",
      "SubSGD iter. 259/499: loss=4.86131761789261, w0=74.20000000000014, w1=9.483605897572614\n",
      "SubSGD iter. 260/499: loss=10.993513442974397, w0=73.50000000000014, w1=10.302264267378243\n",
      "SubSGD iter. 261/499: loss=4.672554644703922, w0=74.20000000000014, w1=10.498784350156745\n",
      "SubSGD iter. 262/499: loss=8.277867958733815, w0=73.50000000000014, w1=11.400416614569503\n",
      "SubSGD iter. 263/499: loss=8.044489641500242, w0=72.80000000000014, w1=12.134530937570492\n",
      "SubSGD iter. 264/499: loss=8.16786795900289, w0=73.50000000000014, w1=12.85269426160168\n",
      "SubSGD iter. 265/499: loss=0.30823924989999796, w0=72.80000000000014, w1=13.531041257576224\n",
      "SubSGD iter. 266/499: loss=1.313816181222819, w0=73.50000000000014, w1=14.059097073086873\n",
      "SubSGD iter. 267/499: loss=3.9454482723934063, w0=74.20000000000014, w1=15.424466525750484\n",
      "SubSGD iter. 268/499: loss=4.417897442966975, w0=73.50000000000014, w1=16.313298349916977\n",
      "SubSGD iter. 269/499: loss=2.6868882918207504, w0=74.20000000000014, w1=17.04526294342461\n",
      "SubSGD iter. 270/499: loss=8.837926250520312, w0=73.50000000000014, w1=17.47705552170803\n",
      "SubSGD iter. 271/499: loss=3.2436160517610375, w0=72.80000000000014, w1=16.857922125763366\n",
      "SubSGD iter. 272/499: loss=1.7341789537737924, w0=72.10000000000014, w1=17.69934063548931\n",
      "SubSGD iter. 273/499: loss=0.32886065934496855, w0=71.40000000000013, w1=17.470218943202553\n",
      "SubSGD iter. 274/499: loss=10.313413365467056, w0=70.70000000000013, w1=16.512083852300748\n",
      "SubSGD iter. 275/499: loss=1.4368110890648964, w0=71.40000000000013, w1=16.14181576177195\n",
      "SubSGD iter. 276/499: loss=8.678207391090467, w0=72.10000000000014, w1=15.614627123024725\n",
      "SubSGD iter. 277/499: loss=2.9719789472271856, w0=72.80000000000014, w1=16.409796848142072\n",
      "SubSGD iter. 278/499: loss=4.981214612459098, w0=72.10000000000014, w1=16.59166069171724\n",
      "SubSGD iter. 279/499: loss=4.113994955603239, w0=71.40000000000013, w1=16.874208510803467\n",
      "SubSGD iter. 280/499: loss=12.200638751215877, w0=72.10000000000014, w1=18.113524811781467\n",
      "SubSGD iter. 281/499: loss=4.512956125487065, w0=71.40000000000013, w1=17.617898551312635\n",
      "SubSGD iter. 282/499: loss=0.8710093555787921, w0=72.10000000000014, w1=16.809784810270703\n",
      "SubSGD iter. 283/499: loss=3.175307442004481, w0=71.40000000000013, w1=16.47455675496745\n",
      "SubSGD iter. 284/499: loss=13.214970878541322, w0=72.10000000000014, w1=15.855415599553066\n",
      "SubSGD iter. 285/499: loss=3.6561461861277493, w0=72.80000000000014, w1=14.671413399159174\n",
      "SubSGD iter. 286/499: loss=1.1240755126300002, w0=72.10000000000014, w1=14.421132282697117\n",
      "SubSGD iter. 287/499: loss=2.67662407011008, w0=72.80000000000014, w1=14.93031812664337\n",
      "SubSGD iter. 288/499: loss=2.7117810464367267, w0=73.50000000000014, w1=15.073648717237885\n",
      "SubSGD iter. 289/499: loss=8.78027439648875, w0=72.80000000000014, w1=15.551696919664009\n",
      "SubSGD iter. 290/499: loss=2.3126437070915102, w0=72.10000000000014, w1=14.959378958395831\n",
      "SubSGD iter. 291/499: loss=8.920234518199699, w0=72.80000000000014, w1=13.932446760803298\n",
      "SubSGD iter. 292/499: loss=4.458432058002842, w0=73.50000000000014, w1=14.584503658245753\n",
      "SubSGD iter. 293/499: loss=7.372371144153121, w0=72.80000000000014, w1=15.131665960479623\n",
      "SubSGD iter. 294/499: loss=5.954195281164409, w0=72.10000000000014, w1=15.282122636930751\n",
      "SubSGD iter. 295/499: loss=1.092491571029825, w0=71.40000000000013, w1=16.371917660003973\n",
      "SubSGD iter. 296/499: loss=6.942423693144946, w0=72.10000000000014, w1=15.160494772077794\n",
      "SubSGD iter. 297/499: loss=1.1053824267733035, w0=71.40000000000013, w1=15.703511233442892\n",
      "SubSGD iter. 298/499: loss=1.0672304168372335, w0=70.70000000000013, w1=14.823781103610415\n",
      "SubSGD iter. 299/499: loss=0.7533908782038026, w0=70.00000000000013, w1=14.122847106175662\n",
      "SubSGD iter. 300/499: loss=3.971501442061488, w0=70.70000000000013, w1=13.769731354608348\n",
      "SubSGD iter. 301/499: loss=1.243231025261153, w0=71.40000000000013, w1=14.374263460462508\n",
      "SubSGD iter. 302/499: loss=2.4862973754055417, w0=72.10000000000014, w1=13.60344039439432\n",
      "SubSGD iter. 303/499: loss=9.53762719404375, w0=72.80000000000014, w1=13.799784987911549\n",
      "SubSGD iter. 304/499: loss=4.2552923758058085, w0=72.10000000000014, w1=14.925717401412719\n",
      "SubSGD iter. 305/499: loss=2.113141837080775, w0=71.40000000000013, w1=13.730978564308293\n",
      "SubSGD iter. 306/499: loss=2.8073902716205055, w0=70.70000000000013, w1=14.820773587381513\n",
      "SubSGD iter. 307/499: loss=2.961419835218422, w0=71.40000000000013, w1=15.545793777462316\n",
      "SubSGD iter. 308/499: loss=2.418764128210853, w0=72.10000000000014, w1=15.445516678222107\n",
      "SubSGD iter. 309/499: loss=2.7542517341117687, w0=72.80000000000014, w1=14.748931926981344\n",
      "SubSGD iter. 310/499: loss=5.02268948162228, w0=73.50000000000014, w1=15.48089652048898\n",
      "SubSGD iter. 311/499: loss=6.7876923115258165, w0=72.80000000000014, w1=16.103682927309272\n",
      "SubSGD iter. 312/499: loss=2.3211875795892283, w0=73.50000000000014, w1=15.499916857342757\n",
      "SubSGD iter. 313/499: loss=2.8951590230296134, w0=72.80000000000014, w1=15.925703004633498\n",
      "SubSGD iter. 314/499: loss=11.591845332635579, w0=72.10000000000014, w1=15.869393389588193\n",
      "SubSGD iter. 315/499: loss=10.015435776887841, w0=72.80000000000014, w1=16.309317832472374\n",
      "SubSGD iter. 316/499: loss=0.6031421376965937, w0=73.50000000000014, w1=16.818503676418626\n",
      "SubSGD iter. 317/499: loss=3.1395363881376284, w0=72.80000000000014, w1=16.553718567227858\n",
      "SubSGD iter. 318/499: loss=5.475368826279706, w0=73.50000000000014, w1=15.980708484213235\n",
      "SubSGD iter. 319/499: loss=13.566688507074389, w0=72.80000000000014, w1=16.100600297701565\n",
      "SubSGD iter. 320/499: loss=6.732923142230391, w0=73.50000000000014, w1=14.746643403668337\n",
      "SubSGD iter. 321/499: loss=0.20428344058113623, w0=74.20000000000014, w1=14.646366304428128\n",
      "SubSGD iter. 322/499: loss=1.370280129638914, w0=73.50000000000014, w1=14.536066270967336\n",
      "SubSGD iter. 323/499: loss=10.00743145840454, w0=72.80000000000014, w1=13.93066311952335\n",
      "SubSGD iter. 324/499: loss=8.745843746965093, w0=73.50000000000014, w1=14.127007713040578\n",
      "SubSGD iter. 325/499: loss=1.3010707867605475, w0=72.80000000000014, w1=13.043132316349276\n",
      "SubSGD iter. 326/499: loss=2.4975726689269493, w0=73.50000000000014, w1=13.421672099706008\n",
      "SubSGD iter. 327/499: loss=0.15350499389863614, w0=74.20000000000014, w1=13.386981240364538\n",
      "SubSGD iter. 328/499: loss=3.8299412848288057, w0=74.90000000000015, w1=14.50529146097602\n",
      "SubSGD iter. 329/499: loss=0.9156206452997395, w0=75.60000000000015, w1=14.47286245320866\n",
      "SubSGD iter. 330/499: loss=7.308731366050502, w0=74.90000000000015, w1=15.085660624210071\n",
      "SubSGD iter. 331/499: loss=0.22880080731320618, w0=74.20000000000014, w1=14.252323041650495\n",
      "SubSGD iter. 332/499: loss=4.873565910585974, w0=73.50000000000014, w1=14.609048560006332\n",
      "SubSGD iter. 333/499: loss=3.823187516300891, w0=74.20000000000014, w1=15.075630685972907\n",
      "SubSGD iter. 334/499: loss=2.668601160968592, w0=73.50000000000014, w1=14.82534956951085\n",
      "SubSGD iter. 335/499: loss=5.281327947335171, w0=74.20000000000014, w1=14.978243326625432\n",
      "SubSGD iter. 336/499: loss=3.85036804198522, w0=74.90000000000015, w1=15.69640665065662\n",
      "SubSGD iter. 337/499: loss=6.0387414068152125, w0=74.20000000000014, w1=16.20142438248094\n",
      "SubSGD iter. 338/499: loss=2.3164068816415977, w0=74.90000000000015, w1=16.397944465259442\n",
      "SubSGD iter. 339/499: loss=6.67494982974123, w0=74.20000000000014, w1=16.745028450879435\n",
      "SubSGD iter. 340/499: loss=4.231972386726426, w0=74.90000000000015, w1=16.172018367864812\n",
      "SubSGD iter. 341/499: loss=6.047526170985819, w0=75.60000000000015, w1=16.4501453366665\n",
      "SubSGD iter. 342/499: loss=6.843506741180271, w0=74.90000000000015, w1=17.00211676860418\n",
      "SubSGD iter. 343/499: loss=8.457077501827484, w0=74.20000000000014, w1=15.807377931499753\n",
      "SubSGD iter. 344/499: loss=5.717933374848002, w0=73.50000000000014, w1=14.612639094395327\n",
      "SubSGD iter. 345/499: loss=0.9842389319186395, w0=72.80000000000014, w1=14.722813211276067\n",
      "SubSGD iter. 346/499: loss=0.07720441896107388, w0=72.10000000000014, w1=14.670652865746357\n",
      "SubSGD iter. 347/499: loss=6.68570120046617, w0=71.40000000000013, w1=14.695950631135199\n",
      "SubSGD iter. 348/499: loss=5.2765896065163105, w0=72.10000000000014, w1=15.70838896900162\n",
      "SubSGD iter. 349/499: loss=0.33166510952156614, w0=71.40000000000013, w1=16.139399653750232\n",
      "SubSGD iter. 350/499: loss=1.615036383393587, w0=70.70000000000013, w1=15.259669523917754\n",
      "SubSGD iter. 351/499: loss=1.283121223495371, w0=70.00000000000013, w1=14.064930686813328\n",
      "SubSGD iter. 352/499: loss=2.411007229966316, w0=69.30000000000013, w1=14.18543523829237\n",
      "SubSGD iter. 353/499: loss=2.6575763234733145, w0=68.60000000000012, w1=14.335891914743499\n",
      "SubSGD iter. 354/499: loss=7.151678843151302, w0=69.30000000000013, w1=14.585490377430677\n",
      "SubSGD iter. 355/499: loss=1.240216287602756, w0=68.60000000000012, w1=15.090508109255\n",
      "SubSGD iter. 356/499: loss=1.1638257345620175, w0=69.30000000000013, w1=14.27184973944937\n",
      "SubSGD iter. 357/499: loss=2.7938593092277557, w0=70.00000000000013, w1=15.177024842636435\n",
      "SubSGD iter. 358/499: loss=9.177392552530591, w0=70.70000000000013, w1=15.730230758712548\n",
      "SubSGD iter. 359/499: loss=9.902920666628845, w0=71.40000000000013, w1=15.4892508440459\n",
      "SubSGD iter. 360/499: loss=7.17398004415395, w0=72.10000000000014, w1=15.52653815566377\n",
      "SubSGD iter. 361/499: loss=1.6824449226420484, w0=72.80000000000014, w1=15.692344415231267\n",
      "SubSGD iter. 362/499: loss=1.0397579465640945, w0=73.50000000000014, w1=15.592067315991057\n",
      "SubSGD iter. 363/499: loss=7.029580798565064, w0=74.20000000000014, w1=15.70588764280537\n",
      "SubSGD iter. 364/499: loss=4.271796366136293, w0=74.90000000000015, w1=14.852041372380102\n",
      "SubSGD iter. 365/499: loss=7.777804672644805, w0=74.20000000000014, w1=14.673689251000287\n",
      "SubSGD iter. 366/499: loss=1.4385144229394058, w0=73.50000000000014, w1=14.444567558713528\n",
      "SubSGD iter. 367/499: loss=2.234795616261259, w0=72.80000000000014, w1=15.228800286752852\n",
      "SubSGD iter. 368/499: loss=5.434871193688906, w0=72.10000000000014, w1=14.060469486235384\n",
      "SubSGD iter. 369/499: loss=2.2621760293039372, w0=72.80000000000014, w1=13.38212249026084\n",
      "SubSGD iter. 370/499: loss=4.057464747196249, w0=73.50000000000014, w1=12.3446082964473\n",
      "SubSGD iter. 371/499: loss=5.145752383954083, w0=72.80000000000014, w1=12.701333814803137\n",
      "SubSGD iter. 372/499: loss=101.8410145685306, w0=73.50000000000014, w1=9.32819013721346\n",
      "SubSGD iter. 373/499: loss=4.953506984450826, w0=74.20000000000014, w1=9.942104550591042\n",
      "SubSGD iter. 374/499: loss=2.8091982684898795, w0=73.50000000000014, w1=8.983969459689234\n",
      "SubSGD iter. 375/499: loss=4.2880262338531026, w0=74.20000000000014, w1=10.152300260206703\n",
      "SubSGD iter. 376/499: loss=10.211467390385486, w0=73.50000000000014, w1=10.970958630012332\n",
      "SubSGD iter. 377/499: loss=11.985903529802812, w0=74.20000000000014, w1=11.318925676402745\n",
      "SubSGD iter. 378/499: loss=0.8459703973389168, w0=73.50000000000014, w1=11.208625642941954\n",
      "SubSGD iter. 379/499: loss=0.12859025822254466, w0=72.80000000000014, w1=11.098325609481162\n",
      "SubSGD iter. 380/499: loss=2.271729707623919, w0=72.10000000000014, w1=11.794910360721925\n",
      "SubSGD iter. 381/499: loss=4.57758926734941, w0=72.80000000000014, w1=13.000926650492373\n",
      "SubSGD iter. 382/499: loss=0.5090087168068465, w0=73.50000000000014, w1=13.230048342779131\n",
      "SubSGD iter. 383/499: loss=0.26820310528235325, w0=72.80000000000014, w1=13.248627801678008\n",
      "SubSGD iter. 384/499: loss=11.462213557372799, w0=72.10000000000014, w1=13.929226979334091\n",
      "SubSGD iter. 385/499: loss=0.9204328193174405, w0=72.80000000000014, w1=13.553641141047521\n",
      "SubSGD iter. 386/499: loss=6.030809793302971, w0=72.10000000000014, w1=14.178457240858467\n",
      "SubSGD iter. 387/499: loss=1.3606362274363946, w0=72.80000000000014, w1=13.574691170891953\n",
      "SubSGD iter. 388/499: loss=5.631503696252935, w0=72.10000000000014, w1=12.627778909149523\n",
      "SubSGD iter. 389/499: loss=102.18656990430031, w0=72.80000000000014, w1=9.254635231559845\n",
      "SubSGD iter. 390/499: loss=0.8793028957167621, w0=73.50000000000014, w1=9.364935265020637\n",
      "SubSGD iter. 391/499: loss=12.784250968017318, w0=74.20000000000014, w1=9.712902311411051\n",
      "SubSGD iter. 392/499: loss=1.5743326626447782, w0=73.50000000000014, w1=9.055719608216066\n",
      "SubSGD iter. 393/499: loss=2.0384846908575582, w0=74.20000000000014, w1=9.258156498257065\n",
      "SubSGD iter. 394/499: loss=7.359477654689627, w0=74.90000000000015, w1=9.371976825071378\n",
      "SubSGD iter. 395/499: loss=1.898661712471494, w0=75.60000000000015, w1=9.387284491783515\n",
      "SubSGD iter. 396/499: loss=5.895896858198256, w0=76.30000000000015, w1=10.019558920265204\n",
      "SubSGD iter. 397/499: loss=13.823253128011842, w0=75.60000000000015, w1=10.528060038005037\n",
      "SubSGD iter. 398/499: loss=0.8572055640056817, w0=74.90000000000015, w1=9.923527932150877\n",
      "SubSGD iter. 399/499: loss=12.493012934860118, w0=74.20000000000014, w1=10.43202904989071\n",
      "SubSGD iter. 400/499: loss=6.873432032699313, w0=73.50000000000014, w1=10.975045511255809\n",
      "SubSGD iter. 401/499: loss=7.29227805236431, w0=74.20000000000014, w1=11.589706492386195\n",
      "SubSGD iter. 402/499: loss=9.276111582212998, w0=73.50000000000014, w1=12.141677924323874\n",
      "SubSGD iter. 403/499: loss=7.622315930018118, w0=72.80000000000014, w1=13.267610337825044\n",
      "SubSGD iter. 404/499: loss=9.34169811007861, w0=73.50000000000014, w1=14.280288679177811\n",
      "SubSGD iter. 405/499: loss=1.0769605059555118, w0=72.80000000000014, w1=12.917537283786064\n",
      "SubSGD iter. 406/499: loss=5.481864714621338, w0=72.10000000000014, w1=11.959402192884257\n",
      "SubSGD iter. 407/499: loss=2.5987525878483666, w0=72.80000000000014, w1=12.161839082925257\n",
      "SubSGD iter. 408/499: loss=6.228379746568841, w0=72.10000000000014, w1=12.585078507697771\n",
      "SubSGD iter. 409/499: loss=0.13818218020804807, w0=71.40000000000013, w1=12.320293398507003\n",
      "SubSGD iter. 410/499: loss=5.731689116452678, w0=70.70000000000013, w1=12.945109498317949\n",
      "SubSGD iter. 411/499: loss=1.3871185129805212, w0=71.40000000000013, w1=13.077433024671787\n",
      "SubSGD iter. 412/499: loss=4.9957282242759575, w0=70.70000000000013, w1=13.227889701122916\n",
      "SubSGD iter. 413/499: loss=1.043052435853994, w0=70.00000000000013, w1=13.428639238172167\n",
      "SubSGD iter. 414/499: loss=4.191064726746191, w0=70.70000000000013, w1=13.956695053682816\n",
      "SubSGD iter. 415/499: loss=8.67377071531611, w0=70.00000000000013, w1=14.637294231338899\n",
      "SubSGD iter. 416/499: loss=5.062708588641144, w0=70.70000000000013, w1=14.45382845228848\n",
      "SubSGD iter. 417/499: loss=1.2708771647840393, w0=70.00000000000013, w1=14.810553970644317\n",
      "SubSGD iter. 418/499: loss=4.005834167981391, w0=70.70000000000013, w1=14.206787900677803\n",
      "SubSGD iter. 419/499: loss=0.9195805961252788, w0=70.00000000000013, w1=14.646476305762988\n",
      "SubSGD iter. 420/499: loss=11.406180236491892, w0=70.70000000000013, w1=14.264350321212653\n",
      "SubSGD iter. 421/499: loss=4.435458053860891, w0=70.00000000000013, w1=14.822474024961693\n",
      "SubSGD iter. 422/499: loss=7.401734373338385, w0=70.70000000000013, w1=15.44596214724787\n",
      "SubSGD iter. 423/499: loss=13.005191944432035, w0=71.40000000000013, w1=14.826820991833488\n",
      "SubSGD iter. 424/499: loss=1.9624683824067404, w0=70.70000000000013, w1=15.353778027602337\n",
      "SubSGD iter. 425/499: loss=15.502597797076916, w0=71.40000000000013, w1=15.698246274472805\n",
      "SubSGD iter. 426/499: loss=5.913339566122417, w0=70.70000000000013, w1=15.915503278652555\n",
      "SubSGD iter. 427/499: loss=5.405350025362289, w0=70.00000000000013, w1=16.393551481078678\n",
      "SubSGD iter. 428/499: loss=6.644665684617053, w0=69.30000000000013, w1=15.446639219336248\n",
      "SubSGD iter. 429/499: loss=2.2732394464764454, w0=70.00000000000013, w1=15.076371128807452\n",
      "SubSGD iter. 430/499: loss=5.461226957153272, w0=70.70000000000013, w1=16.441740581471063\n",
      "SubSGD iter. 431/499: loss=2.759703474078961, w0=71.40000000000013, w1=16.820280364827795\n",
      "SubSGD iter. 432/499: loss=5.636618056376832, w0=72.10000000000014, w1=16.141933368853252\n",
      "SubSGD iter. 433/499: loss=9.996284318205014, w0=71.40000000000013, w1=15.536530217409267\n",
      "SubSGD iter. 434/499: loss=2.128475686262661, w0=72.10000000000014, w1=16.415702529640072\n",
      "SubSGD iter. 435/499: loss=4.5560002036497735, w0=71.40000000000013, w1=17.038488936460364\n",
      "SubSGD iter. 436/499: loss=2.170720301689528, w0=70.70000000000013, w1=16.446170975192185\n",
      "SubSGD iter. 437/499: loss=11.595819585651597, w0=71.40000000000013, w1=16.69048369890132\n",
      "SubSGD iter. 438/499: loss=5.76860685342357, w0=72.10000000000014, w1=15.50648149850743\n",
      "SubSGD iter. 439/499: loss=0.6326238948257696, w0=71.40000000000013, w1=15.464154941903939\n",
      "SubSGD iter. 440/499: loss=0.9158299655744315, w0=70.70000000000013, w1=14.832989601909086\n",
      "SubSGD iter. 441/499: loss=5.998228879769428, w0=71.40000000000013, w1=14.577557720807516\n",
      "SubSGD iter. 442/499: loss=0.11641648487183431, w0=70.70000000000013, w1=13.946392380812663\n",
      "SubSGD iter. 443/499: loss=5.013247701081113, w0=71.40000000000013, w1=14.089722971407179\n",
      "SubSGD iter. 444/499: loss=11.708870862521671, w0=70.70000000000013, w1=14.103478167382669\n",
      "SubSGD iter. 445/499: loss=4.0751920213315245, w0=70.00000000000013, w1=14.253934843833797\n",
      "SubSGD iter. 446/499: loss=1.5519059228611738, w0=69.30000000000013, w1=14.988049166834786\n",
      "SubSGD iter. 447/499: loss=3.9784579005101364, w0=70.00000000000013, w1=14.96946970793591\n",
      "SubSGD iter. 448/499: loss=4.240681256674776, w0=70.70000000000013, w1=15.848642020166716\n",
      "SubSGD iter. 449/499: loss=3.02700571796818, w0=70.00000000000013, w1=16.030505863741887\n",
      "SubSGD iter. 450/499: loss=8.139576807220813, w0=70.70000000000013, w1=15.446193326195996\n",
      "SubSGD iter. 451/499: loss=3.1763511069781813, w0=70.00000000000013, w1=15.728741145282221\n",
      "SubSGD iter. 452/499: loss=6.546077070207801, w0=70.70000000000013, w1=16.607989111178394\n",
      "SubSGD iter. 453/499: loss=1.4401140602686056, w0=70.00000000000013, w1=17.03122853595091\n",
      "SubSGD iter. 454/499: loss=1.3607384750866416, w0=70.70000000000013, w1=16.29711421294992\n",
      "SubSGD iter. 455/499: loss=1.5297909172173405, w0=70.00000000000013, w1=15.961886157646667\n",
      "SubSGD iter. 456/499: loss=4.178894547014025, w0=69.30000000000013, w1=15.538344904592652\n",
      "SubSGD iter. 457/499: loss=5.63922838094102, w0=70.00000000000013, w1=15.626872418081778\n",
      "SubSGD iter. 458/499: loss=0.6984810863481243, w0=69.30000000000013, w1=16.23967058908319\n",
      "SubSGD iter. 459/499: loss=9.566623789924272, w0=68.60000000000012, w1=16.25342578505868\n",
      "SubSGD iter. 460/499: loss=13.471668155762913, w0=69.30000000000013, w1=15.853968012672102\n",
      "SubSGD iter. 461/499: loss=8.045469157036905, w0=70.00000000000013, w1=16.019420804604728\n",
      "SubSGD iter. 462/499: loss=6.180963499941086, w0=70.70000000000013, w1=16.8986687705009\n",
      "SubSGD iter. 463/499: loss=5.966111434094984, w0=71.40000000000013, w1=16.127845704432715\n",
      "SubSGD iter. 464/499: loss=4.965602382394472, w0=72.10000000000014, w1=15.449498708458172\n",
      "SubSGD iter. 465/499: loss=0.5252480141277616, w0=71.40000000000013, w1=15.819766798986969\n",
      "SubSGD iter. 466/499: loss=7.589728211287394, w0=72.10000000000014, w1=14.46580990495374\n",
      "SubSGD iter. 467/499: loss=1.323061156143794, w0=72.80000000000014, w1=15.22716685557281\n",
      "SubSGD iter. 468/499: loss=3.061089711494759, w0=72.10000000000014, w1=15.754123891341658\n",
      "SubSGD iter. 469/499: loss=2.5462047801751737, w0=71.40000000000013, w1=14.391372495949911\n",
      "SubSGD iter. 470/499: loss=107.36104577709031, w0=72.10000000000014, w1=11.621072799060922\n",
      "SubSGD iter. 471/499: loss=0.0839434635672518, w0=72.80000000000014, w1=12.116699059529754\n",
      "SubSGD iter. 472/499: loss=0.6770789163880693, w0=72.10000000000014, w1=12.226873176410495\n",
      "SubSGD iter. 473/499: loss=7.248886259498974, w0=72.80000000000014, w1=12.379766933525076\n",
      "SubSGD iter. 474/499: loss=6.381250631462592, w0=73.50000000000014, w1=12.996153882362325\n",
      "SubSGD iter. 475/499: loss=0.44993584799127007, w0=74.20000000000014, w1=14.080029279053626\n",
      "SubSGD iter. 476/499: loss=0.0749261826980927, w0=73.50000000000014, w1=13.91422301948613\n",
      "SubSGD iter. 477/499: loss=0.8946639884967738, w0=74.20000000000014, w1=14.443923993548246\n",
      "SubSGD iter. 478/499: loss=12.403546788223807, w0=74.90000000000015, w1=15.569053953678553\n",
      "SubSGD iter. 479/499: loss=15.179801594559173, w0=74.20000000000014, w1=15.582809149654043\n",
      "SubSGD iter. 480/499: loss=1.5412868265324136, w0=73.50000000000014, w1=15.530648804124333\n",
      "SubSGD iter. 481/499: loss=10.867607744039375, w0=72.80000000000014, w1=14.925245652680347\n",
      "SubSGD iter. 482/499: loss=3.6478789514902843, w0=72.10000000000014, w1=15.659359975681337\n",
      "SubSGD iter. 483/499: loss=4.723189546549854, w0=71.40000000000013, w1=16.217483679430376\n",
      "SubSGD iter. 484/499: loss=0.11784829412246722, w0=72.10000000000014, w1=16.978840630049447\n",
      "SubSGD iter. 485/499: loss=2.2734958135992116, w0=71.40000000000013, w1=17.603656729860393\n",
      "SubSGD iter. 486/499: loss=3.2018178090740577, w0=70.70000000000013, w1=17.724161281339434\n",
      "SubSGD iter. 487/499: loss=4.981449354737137, w0=70.00000000000013, w1=16.361409885947687\n",
      "SubSGD iter. 488/499: loss=5.019991021971677, w0=70.70000000000013, w1=16.234395440719663\n",
      "SubSGD iter. 489/499: loss=1.9164463823947813, w0=70.00000000000013, w1=15.879367963189837\n",
      "SubSGD iter. 490/499: loss=8.68340425835411, w0=70.70000000000013, w1=15.025521692764569\n",
      "SubSGD iter. 491/499: loss=5.1397214313390265, w0=71.40000000000013, w1=14.993092684997208\n",
      "SubSGD iter. 492/499: loss=0.5132849765681584, w0=70.70000000000013, w1=14.388560579143048\n",
      "SubSGD iter. 493/499: loss=0.2556775503875812, w0=71.40000000000013, w1=13.957549894394434\n",
      "SubSGD iter. 494/499: loss=6.062577682305985, w0=70.70000000000013, w1=14.466051012134267\n",
      "SubSGD iter. 495/499: loss=4.7372745009631245, w0=70.00000000000013, w1=13.519138750391837\n",
      "SubSGD iter. 496/499: loss=1.9484966352910362, w0=69.30000000000013, w1=14.089098874927823\n",
      "SubSGD iter. 497/499: loss=13.934285880504603, w0=70.00000000000013, w1=14.529023317812005\n",
      "SubSGD iter. 498/499: loss=1.7877055672147577, w0=70.70000000000013, w1=14.661346844165843\n",
      "SubSGD iter. 499/499: loss=2.160335536649008, w0=70.00000000000013, w1=15.364851762556706\n",
      "SubSGD: execution time=0.080 seconds\n"
     ]
    }
   ],
   "source": [
    "# Define the parameters of the algorithm.\n",
    "max_iters = 500\n",
    "gamma = 0.7\n",
    "batch_size = 1\n",
    "\n",
    "# Initialization\n",
    "w_initial = np.array([0, 0])\n",
    "\n",
    "# Start SubSGD.\n",
    "start_time = datetime.datetime.now()\n",
    "subsgd_losses, subsgd_ws = stochastic_subgradient_descent(\n",
    "    y, tx, w_initial, batch_size, max_iters, gamma\n",
    ")\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "# Print result\n",
    "exection_time = (end_time - start_time).total_seconds()\n",
    "print(\"SubSGD: execution time={t:.3f} seconds\".format(t=exection_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c495071f2fc24071b1e2a79e269a368f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=1, description='n_iter', max=501, min=1), Output()), _dom_classes=('widg…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.plot_figure(n_iter)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ipywidgets import IntSlider, interact\n",
    "\n",
    "\n",
    "def plot_figure(n_iter):\n",
    "    fig = gradient_descent_visualization(\n",
    "        subsgd_losses,\n",
    "        subsgd_ws,\n",
    "        grid_losses,\n",
    "        grid_w0,\n",
    "        grid_w1,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        height,\n",
    "        weight,\n",
    "        n_iter,\n",
    "    )\n",
    "    fig.set_size_inches(10.0, 6.0)\n",
    "\n",
    "\n",
    "interact(plot_figure, n_iter=IntSlider(min=1, max=len(subsgd_ws)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}