Decision Trees

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris, load_breast_cancer, fetch_california_housing
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor, plot_tree, export_text
from sklearn.metrics import classification_report, mean_absolute_error

# === Classification ===
data = load_iris()
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, stratify=data.target, random_state=42)

clf = DecisionTreeClassifier(
    criterion='gini',          # 'gini' or 'entropy' or 'log_loss'
    max_depth=4,                # PRUNE — single biggest knob
    min_samples_split=20,
    min_samples_leaf=10,
    random_state=42,
)
clf.fit(X_train, y_train)
print(classification_report(y_test, clf.predict(X_test), target_names=data.target_names))

# Probabilities
clf.predict_proba(X_test[:5])

# === Regression ===
hx, hy = fetch_california_housing(return_X_y=True)
rx_tr, rx_te, ry_tr, ry_te = train_test_split(hx, hy, test_size=0.2, random_state=42)

reg = DecisionTreeRegressor(
    max_depth=8,
    min_samples_leaf=20,
    random_state=42,
).fit(rx_tr, ry_tr)
print('MAE:', mean_absolute_error(ry_te, reg.predict(rx_te)))

# === Visualise ===
fig, ax = plt.subplots(figsize=(20, 12))
plot_tree(
    clf,
    feature_names=data.feature_names,
    class_names=data.target_names,
    filled=True,
    rounded=True,
    ax=ax,
)
plt.savefig('tree.png', dpi=100)

# Text export
print(export_text(clf, feature_names=data.feature_names))
# --- example output ---
# |--- petal length (cm) <= 2.45
# |   |--- class: setosa
# |--- petal length (cm) >  2.45
# |   |--- petal length (cm) <= 4.85
# ...

# === Feature importance ===
import pandas as pd
importances = pd.Series(clf.feature_importances_, index=data.feature_names).sort_values()
print(importances)
importances.plot.barh()

# Permutation importance — more robust
from sklearn.inspection import permutation_importance
pi = permutation_importance(clf, X_test, y_test, n_repeats=10, random_state=42)

# === Hyperparameters ===
# criterion           : split quality measure ('gini'|'entropy'|'log_loss')
# max_depth           : maximum depth of the tree (None = unlimited)
# min_samples_split   : min samples needed to split (default 2; raise to 10-20 for noisy data)
# min_samples_leaf    : min samples in a leaf (raise to smooth predictions)
# max_features        : how many features to consider per split (None = all)
# class_weight        : 'balanced' for imbalanced classification
# ccp_alpha           : cost-complexity pruning (post-pruning)
# max_leaf_nodes      : alternative to max_depth — limit total leaves

# === Pruning ===

# Pre-pruning (during fit) — limit max_depth, min_samples_leaf, etc.

# Post-pruning — cost complexity
path = clf.cost_complexity_pruning_path(X_train, y_train)
ccp_alphas = path.ccp_alphas

# Train trees at each alpha
trees = [DecisionTreeClassifier(random_state=42, ccp_alpha=alpha).fit(X_train, y_train) for alpha in ccp_alphas]

# Pick alpha by validation score
best_alpha = ccp_alphas[np.argmax([t.score(X_test, y_test) for t in trees])]
final = DecisionTreeClassifier(random_state=42, ccp_alpha=best_alpha).fit(X_train, y_train)

# === Grid search ===
param_grid = {
    'max_depth':         [3, 5, 7, 10, None],
    'min_samples_leaf':  [1, 5, 10, 20],
    'criterion':         ['gini', 'entropy'],
}
grid = GridSearchCV(DecisionTreeClassifier(random_state=42), param_grid, cv=5, n_jobs=-1)
grid.fit(X_train, y_train)
print(grid.best_params_, grid.best_score_)

# === Imbalanced classes ===
clf_balanced = DecisionTreeClassifier(class_weight='balanced', max_depth=6, random_state=42)

# Or class_weight=dict — manual weights
clf_balanced2 = DecisionTreeClassifier(class_weight={0: 1, 1: 10})

# === Decision path — explain a single prediction ===
sample = X_test[0:1]
path = clf.decision_path(sample)
feature = clf.tree_.feature
threshold = clf.tree_.threshold

# Walk the tree manually for explanation
node_indicator = path.toarray()[0]
for node_id in range(len(node_indicator)):
    if node_indicator[node_id]:
        if feature[node_id] >= 0:
            print(f'Node {node_id}: feature[{data.feature_names[feature[node_id]]}] '
                  f'{"<=" if sample[0, feature[node_id]] <= threshold[node_id] else ">"} {threshold[node_id]:.2f}')

# === Why trees can overfit ===
# A tree can keep splitting until every leaf has 1 sample → perfect train accuracy, bad test.
# Pruning + ensemble methods (RF, GBDT) address this.

# === Pros + cons ===
# Pros:
#   • Interpretable (visualisable, rule-extractable)
#   • No scaling required
#   • Handles missing values (via surrogate splits — sklearn doesn't support directly)
#   • Mixed types (numeric + categorical via encoding)
#   • Captures non-linear boundaries + feature interactions
#
# Cons:
#   • High variance (small data change → very different tree)
#   • Greedy splits → not globally optimal
#   • Prone to overfit without pruning
#   • Doesn't extrapolate well for regression

# === When to use vs alternatives ===
# Decision tree single  : interpretable baseline, audit-friendly, regulated industries
# Random Forest         : strong out-of-the-box accuracy, less interpretable
# Gradient Boosting      : highest accuracy, more tuning
# Linear models          : when relationships are roughly linear + interpretable coefficients matter

# === sklearn API patterns ===
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder

numeric_cols = ['age', 'salary']
categorical_cols = ['city', 'role']

preprocessor = ColumnTransformer([
    ('num', 'passthrough', numeric_cols),                            # trees don't need scaling
    ('cat', OneHotEncoder(handle_unknown='ignore'), categorical_cols),
])

pipe = Pipeline([
    ('prep', preprocessor),
    ('clf',  DecisionTreeClassifier(max_depth=6, random_state=42)),
])
pipe.fit(X_train, y_train)

# === Save / load ===
import joblib
joblib.dump(clf, 'tree.pkl')
loaded = joblib.load('tree.pkl')

# === Tips ===
#   • Always start with max_depth=5-10; deeper = noisier
#   • Use class_weight='balanced' for imbalanced data
#   • For regression, set min_samples_leaf to prevent overfitting to outliers
#   • Plot the tree for small max_depth (<=4) for review with stakeholders
#   • For production accuracy, switch to Random Forest / XGBoost / LightGBM

from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(max_depth=5, min_samples_leaf=20).fit(Xtr, ytr)