Ranking and lift — turning a tree into a list of names

from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

REL = 'public/datasets/decision-trees/hr-employee-attrition.csv'
here = Path.cwd()
LOCAL = next((p / REL for p in [here, *here.parents] if (p / REL).exists()), None)
CSV = LOCAL or 'https://raw.githubusercontent.com/Dr-Shashank-S-Sharma/expedify-ai-courses/main/datasets/decision-trees/hr-employee-attrition.csv'

df = pd.read_csv(CSV)
y = (df.Attrition == "Yes").astype(int)
X = pd.get_dummies(df.drop(columns=["Attrition", "EmployeeID"]), drop_first=True)

tr, te = train_test_split(range(len(df)), test_size=0.35, random_state=7, stratify=y)
ytr, yte = y.iloc[tr], y.iloc[te].values

tree = DecisionTreeClassifier(max_depth=4, random_state=7).fit(X.iloc[tr], ytr)
risk = tree.predict_proba(X.iloc[te])[:, 1]
leaf = tree.apply(X.iloc[te])

record("n_test", len(te))
record("n_leavers", int(yte.sum()))
record("base_rate", f"{yte.mean():.1%}")
print(f"{len(te)} people the tree has never seen, {int(yte.sum())} of whom left "
      f"({yte.mean():.1%})")

rows = []
for lf in sorted(set(leaf), key=lambda l: -risk[leaf == l][0]):
    m = leaf == lf
    rows.append({"leaf": int(lf), "risk": round(float(risk[m][0]), 2),
                 "people": int(m.sum()), "left": int(yte[m].sum())})

print(f"{'the leaf says':>14} {'people in it':>13} {'who actually left':>19}")
for r in rows:
    print(f"{r['risk']:>13.0%} {r['people']:>13} {r['left']:>19}")

record("leaf_rows", rows)
big = max((r for r in rows if r["people"] >= 20), key=lambda r: r["risk"])
record("big_leaf_risk", f"{big['risk']:.0%}")
record("big_leaf_people", big["people"])
record("big_leaf_left", big["left"])
record("n_leaves_shown", len(rows))

order = np.argsort(-risk, kind="stable")
found = np.cumsum(yte[order])

budgets = [10, 25, 50, 75, 100, 150, 200]
print(f"{'conversations':>14} {'leavers reached':>17} {'if chosen at random':>21} {'lift':>7}")
table = []
for k in budgets:
    at_random = k * float(yte.mean())
    table.append({"budget": k, "found": int(found[k - 1]),
                  "random": round(at_random, 1), "lift": round(found[k - 1] / at_random, 1)})
    print(f"{k:>14} {int(found[k-1]):>17} {at_random:>21.1f} {found[k-1]/at_random:>6.1f}x")

record("budget_rows", table)
FIFTY = next(r for r in table if r["budget"] == 50)
record("fifty_found", FIFTY["found"])
record("fifty_random", FIFTY["random"])
record("fifty_lift", f"{FIFTY['lift']}x")
record("share_of_leavers_at_fifty", f"{FIFTY['found'] / yte.sum():.0%}")

def plot(ax):
    n = len(order)
    ax.plot(range(1, n + 1), found, color="#e2574c", label="working down the tree's ranking")
    ax.plot([0, n], [0, yte.sum()], ls="--", lw=1.2, color="#9aa0aa", label="picking people at random")
    ax.plot([0, yte.sum(), n], [0, yte.sum(), yte.sum()], ls=":", lw=1.2, color="#3b6fd4",
            label="a perfect list")
    ax.axvline(50, lw=1, color="#c9c9d1")
    ax.annotate(f"50 conversations\n{int(found[49])} leavers reached", (50, found[49]),
                textcoords="offset points", xytext=(14, -6), fontsize=9, color="#77777f")
    ax.set_xlabel("Conversations you are willing to have")
    ax.set_ylabel("Leavers you reach")
    ax.legend(frameon=False, fontsize=9, loc="lower right")

save_fig("working-down-the-list", plot, figsize=(7, 4.0))

tiny = [r for r in rows if r["risk"] >= 0.99]
print(f"leaves claiming certainty: {len(tiny)}, holding {sum(r['people'] for r in tiny)} people "
      f"between them, of whom {sum(r['left'] for r in tiny)} left")
print(f"the big leaf:              {big['people']} people, {big['left']} left "
      f"({big['left'] / big['people']:.0%})")

record("n_certain_leaves", len(tiny))
record("certain_people", sum(r["people"] for r in tiny))
record("certain_left", sum(r["left"] for r in tiny))

LABEL = {"OverTime_Yes": "works overtime", "JobSatisfaction": "job satisfaction",
         "MonthlyIncome": "monthly income", "Age": "age", "YearsAtCompany": "years at company",
         "TotalWorkingYears": "total working years", "StockOptionLevel": "stock option level",
         "DistanceFromHome": "distance from home", "EnvironmentSatisfaction": "environment score",
         "NumCompaniesWorked": "employers before this", "Education": "education level",
         "YearsSinceLastPromotion": "years since promotion", "JobLevel": "job level",
         "WorkLifeBalance": "work-life balance", "YearsWithCurrManager": "years with manager",
         "YearsInCurrentRole": "years in role", "TrainingTimesLastYear": "trainings last year",
         "PercentSalaryHike": "last salary hike %", "RelationshipSatisfaction": "relationship score",
         "PerformanceRating": "performance rating", "Gender_Male": "is male",
         "Department_R&D": "is in R and D", "Department_Sales": "is in Sales"}

# The leaf id travels in `rows`, so this is a lookup rather than a float
# comparison against a rounded number — which is how the first version of this
# cell died.
target = big["leaf"]

def path_to(clf, names, want, node=0, acc=None):
    acc = acc or []
    t = clf.tree_
    if t.children_left[node] == -1:
        return acc if node == want else None
    col, thr = names[t.feature[node]], t.threshold[node]
    binary = set(np.unique(X[col])) <= {0, 1, True, False}
    whole = bool(pd.Series(X[col]).dropna().mod(1).eq(0).all())
    step = int(np.ceil(thr)) if whole else round(float(thr), 1)
    lo = f"does not {LABEL.get(col, col)}" if binary else f"{LABEL.get(col, col)} under {step:,}"
    hi = f"{LABEL.get(col, col)}" if binary else f"{LABEL.get(col, col)} {step:,} or more"
    return (path_to(clf, names, want, t.children_left[node], acc + [lo])
            or path_to(clf, names, want, t.children_right[node], acc + [hi]))

steps = path_to(tree, list(X.columns), target)
record("big_leaf_path", steps)
for s in steps:
    print(" ", s)
print(f"-> {big['people']} such people in the held-out set, {big['left']} of them left")

import collections
rng = np.random.default_rng(0)
firsts = collections.Counter()
overlap = []
top50 = set(order[:50])

for _ in range(30):
    keep = rng.choice(len(tr), int(len(tr) * 0.95), replace=False)
    idx = [tr[j] for j in keep]
    t2 = DecisionTreeClassifier(max_depth=4, random_state=7).fit(X.iloc[idx], y.iloc[idx])
    firsts[X.columns[t2.tree_.feature[0]]] += 1
    r2 = t2.predict_proba(X.iloc[te])[:, 1]
    overlap.append(len(top50 & set(np.argsort(-r2, kind="stable")[:50])) / 50)

print("first question, over 30 refits:", dict(firsts))
print(f"top-fifty list overlap: mean {np.mean(overlap):.0%}, worst {np.min(overlap):.0%}")

record("first_question_stability", f"{max(firsts.values())} of 30")
record("first_question", LABEL.get(max(firsts, key=firsts.get), max(firsts, key=firsts.get)))
record("list_overlap_mean", f"{np.mean(overlap):.0%}")
record("list_overlap_worst", f"{np.min(overlap):.0%}")
record("list_churn", f"{1 - np.mean(overlap):.0%}")

def plot2(ax):
    ax.hist(overlap, bins=8, color="#e2574c")
    ax.axvline(float(np.mean(overlap)), ls="--", lw=1.2, color="#3b6fd4")
    ax.text(float(np.mean(overlap)), 7.4, f" mean {np.mean(overlap):.0%}", fontsize=9, color="#3b6fd4")
    ax.set_xlabel("Share of the original fifty names still on the list")
    ax.set_ylabel("Refits")
    ax.grid(axis="x", visible=False)

save_fig("the-list-moves", plot2, figsize=(7, 3.2))