Overfitting — perfect on what it has seen

from pathlib import Path
import pandas as pd

# Local checkout first, the published copy otherwise — so this cell works
# unchanged in Colab, where there is no repo.
REL = 'public/datasets/decision-trees/hr-employee-attrition.csv'
here = Path.cwd()
LOCAL = next((p / REL for p in [here, *here.parents] if (p / REL).exists()), None)
CSV = LOCAL or 'https://raw.githubusercontent.com/Dr-Shashank-S-Sharma/expedify-ai-courses/main/datasets/decision-trees/hr-employee-attrition.csv'

df = pd.read_csv(CSV)
record("n_people", len(df))
record("n_columns", df.shape[1] - 2)      # not the ID, not the answer
record("n_left", int((df.Attrition == "Yes").sum()))
record("share_left", round(float((df.Attrition == "Yes").mean()), 3))
record("share_left_pct", f"{(df.Attrition == 'Yes').mean():.1%}")
record("n_stayed", int((df.Attrition == "No").sum()))
print(f"{len(df)} people, {df.shape[1] - 2} things known about each, "
      f"{(df.Attrition == 'Yes').sum()} of them left")
df.head()

y = (df.Attrition == "Yes").astype(int)
X = pd.get_dummies(df.drop(columns=["Attrition", "EmployeeID"]), drop_first=True)

baseline = round(float(1 - y.mean()), 3)
record("baseline", baseline)
# Percentages for the prose. A share of 1.0 interpolates into a sentence as
# "1", which reads as one person rather than all of them.
record("baseline_pct", f"{baseline:.1%}")
print(f"say 'stays' to all {len(y)} people   ->   right {baseline:.1%} of the time")
print(f"leavers found by that strategy       ->   0 of {int(y.sum())}")

from sklearn.model_selection import train_test_split

Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.35, random_state=7, stratify=y)
record("n_train", len(Xtr))
record("n_test", len(Xte))
record("n_test_leavers", int(yte.sum()))
print(f"grown on {len(Xtr)} people, tested on {len(Xte)} it has never seen "
      f"({int(yte.sum())} of whom left)")

from sklearn.tree import DecisionTreeClassifier

DEPTHS = list(range(1, 21))
rows = []
for d in DEPTHS:
    t = DecisionTreeClassifier(max_depth=d, criterion="gini", random_state=7).fit(Xtr, ytr)
    pred = t.predict(Xte)
    rows.append({
        "depth": d,
        "train": round(float(t.score(Xtr, ytr)), 3),
        "test": round(float(t.score(Xte, yte)), 3),
        "leaves": int(t.get_n_leaves()),
        "flagged": int((pred == 1).sum()),
        "caught": int(((pred == 1) & (yte == 1)).sum()),
    })

print(f"{'depth':>5} {'leaves':>7} {'on what it saw':>16} {'on what it did not':>20}")
for r in rows:
    print(f"{r['depth']:>5} {r['leaves']:>7} {r['train']:>16.3f} {r['test']:>20.3f}")

record("sweep", rows)
deepest = rows[-1]
record("deep_train", deepest["train"])
record("deep_test", deepest["test"])
record("deep_leaves", deepest["leaves"])
best = max(rows, key=lambda r: r["test"])
record("best_depth", best["depth"])
record("best_test", best["test"])
record("gap_at_deepest", round(deepest["train"] - deepest["test"], 3))
record("deep_train_pct", f"{deepest['train']:.1%}")
record("deep_test_pct", f"{deepest['test']:.1%}")
record("gap_pct", f"{deepest['train'] - deepest['test']:.1%}")
record("best_test_pct", f"{best['test']:.1%}")

def plot(ax):
    ax.plot([r["depth"] for r in rows], [r["train"] for r in rows],
            marker="o", ms=4, color="#9aa0aa", label="on the people it learned from")
    ax.plot([r["depth"] for r in rows], [r["test"] for r in rows],
            marker="o", ms=4, color="#e2574c", label="on the people it has never met")
    ax.axhline(baseline, ls="--", lw=1.2, color="#3b6fd4")
    ax.text(20, baseline + 0.006, 'saying "stays" to everybody', ha="right", fontsize=9, color="#3b6fd4")
    ax.set_xlabel("How many questions deep the tree may go")
    ax.set_ylabel("Share called right")
    ax.set_xticks([1, 5, 10, 15, 20])
    ax.set_ylim(0.75, 1.02)
    ax.legend(frameon=False, fontsize=9, loc="center right")

save_fig("learned-and-memorised", plot, figsize=(7, 4.0))

def plot2(ax):
    gaps = [r["train"] - r["test"] for r in rows]
    ax.bar([r["depth"] for r in rows], gaps, color="#e2574c", width=0.62)
    ax.set_xlabel("How many questions deep the tree may go")
    ax.set_ylabel("Score that exists only\non the training set")
    ax.set_xticks([1, 5, 10, 15, 20])
    ax.grid(axis="x", visible=False)

save_fig("the-gap", plot2, figsize=(7, 3.2))

below = [r for r in rows if r["test"] < baseline]
record("first_depth_below_baseline", below[0]["depth"] if below else None)
record("n_depths_beating_baseline", sum(1 for r in rows if r["test"] > baseline))
print(f"depths that beat 'stays for everybody': {sum(1 for r in rows if r['test'] > baseline)} of {len(rows)}")
print(f"best of them: depth {best['depth']} at {best['test']:.3f}, against a free {baseline:.3f}")

import numpy as np

LABEL = {
    "MonthlyIncome": "Monthly income", "Age": "Age", "DistanceFromHome": "Distance from home",
    "YearsAtCompany": "Years at company", "TotalWorkingYears": "Total working years",
    "YearsSinceLastPromotion": "Years since promotion", "YearsInCurrentRole": "Years in role",
    "YearsWithCurrManager": "Years with manager", "StockOptionLevel": "Stock option level",
    "JobLevel": "Job level", "EnvironmentSatisfaction": "Environment score",
    "JobSatisfaction": "Job satisfaction", "WorkLifeBalance": "Work-life balance",
    "TrainingTimesLastYear": "Trainings last year", "NumCompaniesWorked": "Employers before this",
    "PercentSalaryHike": "Last salary hike %", "RelationshipSatisfaction": "Relationship score",
    "Education": "Education level", "PerformanceRating": "Performance rating",
    "OverTime_Yes": "Works overtime?", "Gender_Male": "Male?",
    "Department_R&D": "In R and D?", "Department_Sales": "In Sales?",
}

def to_nodes(clf, names, node=0):
    """sklearn's arrays -> the {q, yes, no, note} shape the lesson draws.

    sklearn always sends `feature <= threshold` LEFT. For a yes/no column that
    means left is the NO side, so the branches are swapped and the question is
    asked the way a person would ask it. Getting this backwards draws a tree
    that is a mirror image of the one that was fitted, and nothing about the
    picture would look wrong.
    """
    t = clf.tree_
    stayed, left_co = t.value[node][0]
    n = int(t.n_node_samples[node])
    share = left_co / (stayed + left_co)
    note = f"{n} people - {round(share * n)} left"
    if t.children_left[node] == -1:
        return {"leaf": "leaves" if share >= 0.5 else "stays", "note": note}
    col = names[t.feature[node]]
    thr = t.threshold[node]
    lo = to_nodes(clf, names, t.children_left[node])
    hi = to_nodes(clf, names, t.children_right[node])
    if col.endswith("_Yes") or col.startswith(("Gender_", "Department_")):
        return {"q": LABEL.get(col, col), "note": note, "yes": hi, "no": lo}
    # A threshold of 0.5 on a whole-number column reads as "under 0" if it is
    # simply rounded, which is not what the tree does. Round UP: "under 1"
    # means zero, which is the split that was actually made.
    whole = bool(X[col].dropna().mod(1).eq(0).all())
    step = np.ceil(thr) if whole else thr
    return {"q": f"{LABEL.get(col, col)} under {step:,.0f}?", "note": note, "yes": lo, "no": hi}

shallow = DecisionTreeClassifier(max_depth=2, criterion="gini", random_state=7).fit(Xtr, ytr)
record("shallow_tree", to_nodes(shallow, list(X.columns)))
record("shallow_test", round(float(shallow.score(Xte, yte)), 3))
record("shallow_test_pct", f"{shallow.score(Xte, yte):.1%}")
record("shallow_flagged", int((shallow.predict(Xte) == 1).sum()))

# The leaf a manager should be most alarmed by, and the tree calls it "stays".
def worst_leaf(node):
    if "leaf" in node:
        n, left = int(node["note"].split()[0]), int(node["note"].split("- ")[1].split()[0])
        return (left / n, n, left, node["leaf"])
    return max(worst_leaf(node["yes"]), worst_leaf(node["no"]))

share, n_leaf, n_left_leaf, call = worst_leaf(record("shallow_tree", to_nodes(shallow, list(X.columns))))
record("worst_leaf_people", n_leaf)
record("worst_leaf_left", n_left_leaf)
record("worst_leaf_share", round(share, 2))
record("worst_leaf_pct", f"{share:.0%}")
record("worst_leaf_call", call)
print(f"depth 2 scores {shallow.score(Xte, yte):.3f} against a free {baseline:.3f}")
print(f"and flags {int((shallow.predict(Xte) == 1).sum())} people out of {len(yte)} for a conversation")

print(f"{'depth':>5} {'test score':>11} {'people flagged':>15} {'leavers caught':>15}")
for r in rows[:8]:
    print(f"{r['depth']:>5} {r['test']:>11.3f} {r['flagged']:>15} {r['caught']:>10} of {int(yte.sum())}")

useful = max(rows, key=lambda r: r["caught"])
record("useful_depth", useful["depth"])
record("useful_test", useful["test"])
record("useful_test_pct", f"{useful['test']:.1%}")
record("useful_flagged", useful["flagged"])
record("useful_caught", useful["caught"])

def plot3(ax):
    ax.bar([r["depth"] for r in rows], [r["caught"] for r in rows], color="#e2574c", width=0.62)
    ax.axhline(int(yte.sum()), ls="--", lw=1.2, color="#3b6fd4")
    ax.text(20, int(yte.sum()) + 1.2, f"all {int(yte.sum())} leavers in the test set",
            ha="right", fontsize=9, color="#3b6fd4")
    ax.set_xlabel("How many questions deep the tree may go")
    ax.set_ylabel("Leavers it actually found")
    ax.set_xticks([1, 5, 10, 15, 20])
    ax.set_ylim(0, int(yte.sum()) * 1.18)
    ax.grid(axis="x", visible=False)

save_fig("what-it-catches", plot3, figsize=(7, 3.2))