{ "cells": [ { "cell_type": "markdown", "id": "6f10f133", "metadata": {}, "source": [ "# A1 · L4 — Is it any good?\n", "\n", "Three ways to answer, and one plot that beats all three. Same model as lesson 3:\n", "sales from TV budget alone.\n", "\n", "Every figure and number this notebook produces is what the lesson prints." ] }, { "cell_type": "code", "execution_count": 1, "id": "d41179ec", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:55.077905Z", "iopub.status.busy": "2026-08-19T09:11:55.077727Z", "iopub.status.idle": "2026-08-19T09:11:55.517714Z", "shell.execute_reply": "2026-08-19T09:11:55.517345Z" } }, "outputs": [], "source": [ "# hide — plumbing so this file runs both in the repo and in Colab.\n", "try:\n", " from _figkit import save_fig, record\n", "except ImportError: # Colab — no repo, no problem\n", " def save_fig(name, plot, **kw):\n", " import matplotlib.pyplot as plt\n", " fig, ax = plt.subplots(figsize=kw.get('figsize', (7, 4.2))); plot(ax); plt.show()\n", " def record(key, value):\n", " print(f'{key} = {value}'); return value\n", "\n", "from pathlib import Path\n", "REL = 'public/datasets/a1-regression/advertising-media-mix.csv'\n", "here = Path.cwd()\n", "LOCAL = next((p / REL for p in [here, *here.parents] if (p / REL).exists()), None)\n", "CSV = LOCAL or 'https://raw.githubusercontent.com/Dr-Shashank-S-Sharma/expedify-ai-courses/main/datasets/a1-regression/advertising-media-mix.csv'" ] }, { "cell_type": "code", "execution_count": 2, "id": "ca47fe46", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:55.519189Z", "iopub.status.busy": "2026-08-19T09:11:55.519059Z", "iopub.status.idle": "2026-08-19T09:11:56.684054Z", "shell.execute_reply": "2026-08-19T09:11:56.683762Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sales = 7.03 + 0.0475 x TV\n" ] } ], "source": [ "#| caption: Fit the same line as lesson 3\n", "import numpy as np\n", "import pandas as pd\n", "\n", "df = pd.read_csv(CSV)\n", "x, y = df['tv_spend'], df['sales']\n", "\n", "slope, intercept = np.polyfit(x, y, 1)\n", "predicted = intercept + slope * x\n", "error = y - predicted # positive = we under-predicted this market\n", "\n", "print(f'sales = {intercept:.2f} + {slope:.4f} x TV')" ] }, { "cell_type": "markdown", "id": "3968f07c", "metadata": {}, "source": [ "## 1. Three ways to say \"how wrong\"\n", "\n", "They answer different questions, and only two of them are in units anybody can act on." ] }, { "cell_type": "code", "execution_count": 3, "id": "3d59230b", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.686506Z", "iopub.status.busy": "2026-08-19T09:11:56.686378Z", "iopub.status.idle": "2026-08-19T09:11:56.689302Z", "shell.execute_reply": "2026-08-19T09:11:56.689042Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "R2 = 0.612 share of the variation in sales the model explains\n", "RMSE = 3.24k units typical miss, big misses weighted heavily\n", "MAE = 2.55k units typical miss, every market counted equally\n", "mean sales = 14.02k units <- RMSE is 23% of it\n" ] } ], "source": [ "#| caption: R², RMSE and MAE\n", "r2 = 1 - (error ** 2).sum() / ((y - y.mean()) ** 2).sum()\n", "rmse = np.sqrt((error ** 2).mean())\n", "mae = error.abs().mean()\n", "\n", "print(f'R2 = {r2:.3f} share of the variation in sales the model explains')\n", "print(f'RMSE = {rmse:.2f}k units typical miss, big misses weighted heavily')\n", "print(f'MAE = {mae:.2f}k units typical miss, every market counted equally')\n", "print(f'mean sales = {y.mean():.2f}k units <- RMSE is {100 * rmse / y.mean():.0f}% of it')" ] }, { "cell_type": "code", "execution_count": 4, "id": "b9cdbcb1", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.690446Z", "iopub.status.busy": "2026-08-19T09:11:56.690358Z", "iopub.status.idle": "2026-08-19T09:11:56.768374Z", "shell.execute_reply": "2026-08-19T09:11:56.767919Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "r2 = 0.612\n", "rmse = 3.24\n", "mae = 2.55\n", "mean_sales = 14.02\n", "rmse_pct_of_mean = 23%\n", "rmse_over_mae = 1.27\n", "worst_miss = 8.39\n", "share_within_mae = 61%\n", "figure error-sizes -> is-it-any-good.error-sizes.{light,dark}.svg\n" ] } ], "source": [ "# hide — metrics + figure\n", "record('r2', round(float(r2), 3))\n", "record('rmse', round(float(rmse), 2))\n", "record('mae', round(float(mae), 2))\n", "record('mean_sales', round(float(y.mean()), 2))\n", "record('rmse_pct_of_mean', f'{100 * rmse / y.mean():.0f}%')\n", "record('rmse_over_mae', round(float(rmse / mae), 2))\n", "record('worst_miss', round(float(error.abs().max()), 2))\n", "record('share_within_mae', f'{100 * (error.abs() <= mae).mean():.0f}%')\n", "\n", "def plot(ax):\n", " ax.hist(error.abs(), bins=20, edgecolor='none')\n", " ax.axvline(mae, linestyle='--', linewidth=1.5, color='#8a8a94')\n", " ax.axvline(rmse, linestyle='-', linewidth=1.5, color='#8a8a94')\n", " ax.annotate('MAE', xy=(mae, 0), xytext=(-26, 30), textcoords='offset points',\n", " fontsize=9, color='#8a8a94')\n", " ax.annotate('RMSE', xy=(rmse, 0), xytext=(6, 44), textcoords='offset points',\n", " fontsize=9, color='#8a8a94')\n", " ax.set_xlabel('How far off, in thousands of units')\n", " ax.set_ylabel('Markets')\n", "\n", "save_fig('error-sizes', plot, figsize=(7, 3.9))" ] }, { "cell_type": "markdown", "id": "2455df01", "metadata": {}, "source": [ "## 2. Predicted against actual\n", "\n", "The most honest picture of a regression: every market plotted against what the model\n", "said. Perfect prediction is the diagonal." ] }, { "cell_type": "code", "execution_count": 5, "id": "d7511b72", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.770542Z", "iopub.status.busy": "2026-08-19T09:11:56.770144Z", "iopub.status.idle": "2026-08-19T09:11:56.773022Z", "shell.execute_reply": "2026-08-19T09:11:56.772689Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25% of markets predicted within 1,000 units\n", "worst market: off by 8.39k units\n" ] } ], "source": [ "#| caption: How far from the diagonal?\n", "on_the_nose = (error.abs() <= 1.0).mean() # within 1,000 units\n", "print(f'{100 * on_the_nose:.0f}% of markets predicted within 1,000 units')\n", "print(f'worst market: off by {error.abs().max():.2f}k units')" ] }, { "cell_type": "code", "execution_count": 6, "id": "7bbe2ec3", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.775113Z", "iopub.status.busy": "2026-08-19T09:11:56.775011Z", "iopub.status.idle": "2026-08-19T09:11:56.842237Z", "shell.execute_reply": "2026-08-19T09:11:56.841951Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "share_within_1k = 25%\n", "figure predicted-vs-actual -> is-it-any-good.predicted-vs-actual.{light,dark}.svg\n" ] } ], "source": [ "# hide — metrics + figure\n", "record('share_within_1k', f'{100 * on_the_nose:.0f}%')\n", "\n", "def plot(ax):\n", " lim = [min(y.min(), predicted.min()) - 1, max(y.max(), predicted.max()) + 1]\n", " ax.plot(lim, lim, linestyle='--', linewidth=1.3, color='#8a8a94', label='perfect prediction')\n", " ax.scatter(predicted, y, s=20, alpha=0.6, label='markets')\n", " ax.set_xlabel('Predicted sales (thousands of units)')\n", " ax.set_ylabel('Actual sales (thousands of units)')\n", " ax.legend(frameon=False, fontsize=9)\n", "\n", "save_fig('predicted-vs-actual', plot, figsize=(7, 4.4))" ] }, { "cell_type": "markdown", "id": "9aa8c7f9", "metadata": {}, "source": [ "## 3. The residual plot\n", "\n", "The one plot worth more than every score above. If the model has caught everything\n", "there is to catch, what is left over is noise — a shapeless band around zero. Any\n", "*shape* is signal the model missed." ] }, { "cell_type": "code", "execution_count": 7, "id": "798f3e72", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.843581Z", "iopub.status.busy": "2026-08-19T09:11:56.843489Z", "iopub.status.idle": "2026-08-19T09:11:56.849957Z", "shell.execute_reply": "2026-08-19T09:11:56.849736Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "low TV : average error -0.07k -> over-predicts\n", "mid TV : average error +0.26k -> under-predicts\n", "high TV : average error -0.19k -> over-predicts\n" ] } ], "source": [ "#| caption: Is the error random, or does it have a shape?\n", "thirds = pd.qcut(x, 3, labels=['low TV', 'mid TV', 'high TV'])\n", "bias = error.groupby(thirds, observed=True).mean()\n", "\n", "for band, mean_error in bias.items():\n", " direction = 'under-predicts' if mean_error > 0 else 'over-predicts'\n", " print(f'{band:8s}: average error {mean_error:+.2f}k -> {direction}')" ] }, { "cell_type": "code", "execution_count": 8, "id": "75a62676", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.850943Z", "iopub.status.busy": "2026-08-19T09:11:56.850858Z", "iopub.status.idle": "2026-08-19T09:11:56.923125Z", "shell.execute_reply": "2026-08-19T09:11:56.922865Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "bias_low_tv = -0.07\n", "bias_mid_tv = 0.26\n", "bias_high_tv = -0.19\n", "spread_low_tv = 1.86\n", "spread_high_tv = 4.42\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "figure residuals -> is-it-any-good.residuals.{light,dark}.svg\n" ] } ], "source": [ "# hide — metrics + figure\n", "record('bias_low_tv', round(float(bias['low TV']), 2))\n", "record('bias_mid_tv', round(float(bias['mid TV']), 2))\n", "record('bias_high_tv', round(float(bias['high TV']), 2))\n", "record('spread_low_tv', round(float(error[thirds == 'low TV'].std()), 2))\n", "record('spread_high_tv', round(float(error[thirds == 'high TV'].std()), 2))\n", "\n", "def plot(ax):\n", " ax.axhline(0, linewidth=1.2, color='#8a8a94')\n", " ax.scatter(predicted, error, s=20, alpha=0.6)\n", " order = np.argsort(predicted.values)\n", " smooth = pd.Series(error.values[order]).rolling(25, center=True, min_periods=5).mean()\n", " ax.plot(predicted.values[order], smooth, linewidth=2, label='local average error')\n", " ax.set_xlabel('Predicted sales (thousands of units)')\n", " ax.set_ylabel('Error: actual − predicted')\n", " ax.legend(frameon=False, fontsize=9)\n", "\n", "save_fig('residuals', plot, figsize=(7, 4.4))" ] }, { "cell_type": "markdown", "id": "f25885cd", "metadata": {}, "source": [ "## 4. What the misses cost\n", "\n", "A score is an abstraction. A market you got badly wrong is a plan somebody built." ] }, { "cell_type": "code", "execution_count": 9, "id": "c209167e", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.924227Z", "iopub.status.busy": "2026-08-19T09:11:56.924148Z", "iopub.status.idle": "2026-08-19T09:11:56.931816Z", "shell.execute_reply": "2026-08-19T09:11:56.931620Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tvactualpredictederror
178276.711.820.19-8.39
35290.712.820.85-8.05
25262.912.019.53-7.53
55198.923.716.497.21
128220.324.717.507.20
131265.212.719.64-6.94
147243.225.418.596.81
175276.927.020.206.80
\n", "
" ], "text/plain": [ " tv actual predicted error\n", "178 276.7 11.8 20.19 -8.39\n", "35 290.7 12.8 20.85 -8.05\n", "25 262.9 12.0 19.53 -7.53\n", "55 198.9 23.7 16.49 7.21\n", "128 220.3 24.7 17.50 7.20\n", "131 265.2 12.7 19.64 -6.94\n", "147 243.2 25.4 18.59 6.81\n", "175 276.9 27.0 20.20 6.80" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#| caption: The eight worst markets\n", "worst = (pd.DataFrame({'tv': x, 'actual': y, 'predicted': predicted, 'error': error})\n", " .assign(miss=lambda d: d.error.abs())\n", " .nlargest(8, 'miss')\n", " .round(2))\n", "worst[['tv', 'actual', 'predicted', 'error']]" ] }, { "cell_type": "code", "execution_count": 10, "id": "34f2072e", "metadata": { "execution": { "iopub.execute_input": "2026-08-19T09:11:56.932947Z", "iopub.status.busy": "2026-08-19T09:11:56.932855Z", "iopub.status.idle": "2026-08-19T09:11:56.987419Z", "shell.execute_reply": "2026-08-19T09:11:56.987195Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "worst_tv = 276.7\n", "worst_actual = 11.8\n", "worst_predicted = 20.19\n", "n_over_2k_off = 102\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "figure worst-markets -> is-it-any-good.worst-markets.{light,dark}.svg\n" ] } ], "source": [ "# hide — metrics + figure\n", "record('worst_tv', round(float(worst.iloc[0]['tv']), 1))\n", "record('worst_actual', round(float(worst.iloc[0]['actual']), 2))\n", "record('worst_predicted', round(float(worst.iloc[0]['predicted']), 2))\n", "record('n_over_2k_off', int((error.abs() > 2).sum()))\n", "\n", "def plot(ax):\n", " labels = [f\"${row.tv:,.0f}k\" for row in worst.itertuples()]\n", " ax.barh(labels, worst['error'])\n", " ax.axvline(0, linewidth=1, color='#8a8a94')\n", " ax.set_xlabel('Error: actual − predicted (thousands of units)')\n", " ax.set_ylabel('Market, by TV budget')\n", " ax.invert_yaxis()\n", "\n", "save_fig('worst-markets', plot, figsize=(7, 4.0))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 5 }