Imbalanced Data

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import (
    confusion_matrix, classification_report,
    roc_auc_score, average_precision_score,
    precision_recall_curve, roc_curve,
)
import numpy as np
import matplotlib.pyplot as plt

# 1) Imbalanced data — 5% positives
X, y = make_classification(n_samples=10_000, n_features=20, weights=[0.95, 0.05], random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)
# stratify=y preserves the imbalance across splits.

# 2) Why accuracy lies
pipe = Pipeline([('scale', StandardScaler()), ('clf', LogisticRegression(max_iter=1000))]).fit(X_train, y_train)
print('accuracy:', pipe.score(X_test, y_test))            # ~0.95 — looks great
print('confusion:\n', confusion_matrix(y_test, pipe.predict(X_test)))
# Predicting the majority class everywhere yields 95% accuracy. Useless.

# 3) Better metrics for imbalance
probs = pipe.predict_proba(X_test)[:, 1]
print('AUC-ROC:', roc_auc_score(y_test, probs))
print('AUC-PR :', average_precision_score(y_test, probs))
print(classification_report(y_test, pipe.predict(X_test)))
# AUC-PR is more informative than AUC-ROC under heavy imbalance.
# precision / recall / F1 give per-class views.

# 4) Class weights — let the model see the imbalance
LogisticRegression(class_weight='balanced', max_iter=1000)
RandomForestClassifier(class_weight='balanced', random_state=42)
# 'balanced' = inversely proportional to class frequency.
# Or pass explicit dict: class_weight={0: 1, 1: 19}  (1:19 if 5% positives).

# 5) sample_weight — per-row weight (e.g. via business cost matrix)
from sklearn.utils.class_weight import compute_sample_weight
weights = compute_sample_weight(class_weight='balanced', y=y_train)
GradientBoostingClassifier().fit(X_train, y_train, sample_weight=weights)

# 6) Resampling — pip install imbalanced-learn
from imblearn.over_sampling import RandomOverSampler, SMOTE
from imblearn.under_sampling import RandomUnderSampler
from imblearn.combine import SMOTEENN, SMOTETomek
from imblearn.pipeline import Pipeline as ImbPipeline

# SMOTE — synthetic over-sampling along the line between a minority point + its k-nearest minority neighbours
imb_pipe = ImbPipeline([
    ('scale', StandardScaler()),
    ('smote', SMOTE(random_state=42)),
    ('clf',   LogisticRegression(max_iter=1000)),
])
imb_pipe.fit(X_train, y_train)

# RandomOverSampler — duplicate minority rows (risk: overfitting)
# RandomUnderSampler — drop majority rows (risk: throwing away signal)
# SMOTEENN / SMOTETomek — over + clean noisy boundary points

# 7) Pipeline + cross-validation done right
# ALWAYS resample INSIDE the CV fold to avoid leakage.
from sklearn.model_selection import cross_val_score
scores = cross_val_score(imb_pipe, X_train, y_train, cv=StratifiedKFold(5),
                                                  scoring='average_precision')
print('CV AUC-PR:', scores.mean(), '+/-', scores.std())

# 8) Threshold calibration
# By default, predict_proba > 0.5 = class 1. For imbalanced data, that threshold rarely matches your goal.
# Plot the precision-recall curve, pick the threshold for desired precision OR recall.
prec, recall, thresholds = precision_recall_curve(y_test, probs)

# Find threshold for at least 70% precision
idx = np.where(prec[:-1] >= 0.70)[0]
if len(idx) > 0:
    t = thresholds[idx[0]]
    print(f'threshold={t:.3f}  precision={prec[idx[0]]:.3f}  recall={recall[idx[0]]:.3f}')

# Calibrate probabilities if your downstream code uses them directly (e.g. expected loss):
from sklearn.calibration import CalibratedClassifierCV
cal = CalibratedClassifierCV(estimator=GradientBoostingClassifier(), cv=3, method='isotonic')
cal.fit(X_train, y_train)

# 9) Cost-sensitive learning — bake the business cost in
# FN = miss fraud → $X loss; FP = block legit txn → $Y friction
# Minimise expected loss = TP*0 + FP*Y + FN*X + TN*0
FN_cost, FP_cost = 100, 5
expected_costs = []
for t in thresholds:
    pred = (probs >= t).astype(int)
    fp = ((pred == 1) & (y_test == 0)).sum()
    fn = ((pred == 0) & (y_test == 1)).sum()
    expected_costs.append(fp * FP_cost + fn * FN_cost)
best_t = thresholds[np.argmin(expected_costs)]
print('best threshold by cost:', best_t)

# 10) Anomaly detection — when positives are SO rare you can't train binary classifiers
from sklearn.ensemble import IsolationForest
from sklearn.svm import OneClassSVM

iso = IsolationForest(contamination=0.05, random_state=42).fit(X_train)
anomaly_scores = iso.decision_function(X_test)
# Score < threshold = anomaly.

# 11) Stratification + folding
# Always StratifiedKFold for classification.
# For tiny minority class (< 5 per fold), use RepeatedStratifiedKFold or LeaveOneOut for the minority.

# 12) Treat the eval set like production
# • Same class ratio as production (don't artificially rebalance test set)
# • Same time / region distribution (use a TIME-based split for streaming data)
# • Same feature dropout / quality
# An optimistic eval set is the most common source of post-deploy disappointment.

# 13) Visualise to communicate
fig, ax = plt.subplots(1, 2, figsize=(12, 4))
ax[0].plot(recall, prec); ax[0].set_xlabel('recall'); ax[0].set_ylabel('precision'); ax[0].set_title('PR curve')
fpr, tpr, _ = roc_curve(y_test, probs)
ax[1].plot(fpr, tpr); ax[1].set_xlabel('FPR'); ax[1].set_ylabel('TPR'); ax[1].set_title('ROC curve')

# 14) Common bugs
# • Reporting accuracy on 95/5 data — meaningless; report AUC-PR + recall@precision
# • Resampling BEFORE CV split → data leakage; synthetic minority points share info across folds
# • SMOTE on categorical / mixed features → use SMOTEN (nominal) or SMOTENC (mixed)
# • Predicting at threshold 0.5 without calibration — almost never optimal
# • Rebalancing the TEST set — destroys realism
# • Calling 'class_weight=balanced' on tree models without verifying it triggers per-tree weighting (sklearn ensemble vs xgboost differ)
# • Treating fraud as static — concept drift; retrain on recent windows
# • Pretending 99% AUC-ROC is good when AUC-PR is 0.2 — different metrics tell different stories

# Try class_weight='balanced' first.
# Then resampling (SMOTE from imblearn).
# Watch precision/recall on the minority class, not accuracy.