Logistic Regression

import numpy as np
from sklearn.datasets import load_breast_cancer, load_iris
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    classification_report, roc_auc_score, RocCurveDisplay,
    PrecisionRecallDisplay, log_loss, brier_score_loss,
)

# === Binary classification ===
X, y = load_breast_cancer(return_X_y=True)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, stratify=y, test_size=0.2, random_state=42)

# 1) Pipeline — ALWAYS scale features for logistic
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('lr',     LogisticRegression(
        penalty='l2',         # 'l1' | 'l2' | 'elasticnet' | None
        C=1.0,                # inverse of regularisation strength (smaller = stronger reg)
        solver='lbfgs',       # 'lbfgs' default; 'liblinear' for small; 'saga' for elasticnet
        max_iter=1000,
        random_state=42,
        class_weight='balanced',  # for imbalanced data
    )),
])
pipe.fit(X_tr, y_tr)

# 2) Evaluate
print(classification_report(y_te, pipe.predict(X_te)))
print('AUC:', roc_auc_score(y_te, pipe.predict_proba(X_te)[:, 1]))
RocCurveDisplay.from_estimator(pipe, X_te, y_te)
PrecisionRecallDisplay.from_estimator(pipe, X_te, y_te)

# 3) Pick a threshold — default 0.5 is rarely optimal
from sklearn.metrics import precision_recall_curve
probs = pipe.predict_proba(X_te)[:, 1]
p, r, t = precision_recall_curve(y_te, probs)
f1 = 2 * p * r / (p + r + 1e-9)
best = np.argmax(f1)
print(f'best threshold: {t[best]:.3f}  precision={p[best]:.3f}  recall={r[best]:.3f}')

# 4) Calibration — well-calibrated probabilities matter for decisions
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
calibrated = CalibratedClassifierCV(pipe, method='isotonic', cv=5)
calibrated.fit(X_tr, y_tr)
print('Brier:', brier_score_loss(y_te, calibrated.predict_proba(X_te)[:, 1]))

# 5) Regularisation choice
#   l2 (Ridge)         — shrinks all coefficients; default
#   l1 (Lasso)         — zeroes weakest features; sparse model, feature selection
#   elasticnet         — l1 + l2; tunable mix
#   None               — no regularisation; rarely best

# 6) Hyperparameter search
param_grid = {
    'lr__C':       [0.001, 0.01, 0.1, 1, 10, 100],
    'lr__penalty': ['l2'],
    'lr__class_weight': [None, 'balanced'],
}
grid = GridSearchCV(pipe, param_grid, cv=5, scoring='roc_auc', n_jobs=-1)
grid.fit(X_tr, y_tr)
print(grid.best_params_, grid.best_score_)

# 7) Interpret — coefficients = log-odds change per standardised unit
lr = grid.best_estimator_.named_steps['lr']
import pandas as pd
coef = pd.Series(lr.coef_[0], index=load_breast_cancer().feature_names).sort_values()
print(coef.head(5))   # most negative (decrease likelihood of class 1)
print(coef.tail(5))   # most positive

# Odds ratios — easier to communicate
import numpy as np
odds = np.exp(coef)
# 1 standard-deviation increase in feature X multiplies the odds by `odds[X]`

# === Multiclass classification ===
Xi, yi = load_iris(return_X_y=True)
Xi_tr, Xi_te, yi_tr, yi_te = train_test_split(Xi, yi, stratify=yi, test_size=0.2, random_state=42)

multi = Pipeline([
    ('scaler', StandardScaler()),
    ('lr',     LogisticRegression(multi_class='multinomial', solver='lbfgs', max_iter=1000)),
]).fit(Xi_tr, yi_tr)
print(classification_report(yi_te, multi.predict(Xi_te)))

# 'ovr' (one-vs-rest) vs 'multinomial' (softmax). Multinomial is preferred for true multi-class.

# === Practical tips ===
# - ALWAYS scale features (StandardScaler) — gradient-based solvers depend on it
# - For very high-dim sparse text features → use 'saga' solver + L1
# - Look at log_loss, not just accuracy — accuracy can hide a bad calibration
# - Logistic regression assumes a LINEAR boundary in feature space — engineer features
#   (interactions, polynomials) or switch to GBDT for non-linear data
# - For imbalanced data: class_weight='balanced' + AUC/PR-AUC metrics + adjusted threshold
#
# When to use logistic regression
#   • Baseline for ANY binary classification task
#   • You need interpretable coefficients (regulated industries, comms with stakeholders)
#   • Low-latency inference (single dot product)
#   • Calibrated probabilities for decision making
#
# When to switch
#   • Non-linear decision boundary that engineered features can't capture → GBDT / RF / NN
#   • Very large feature space + non-linearity → wide + deep, NN
#   • Sequence / image / text raw input → CNN / RNN / Transformer

from sklearn.linear_model import LogisticRegression
clf = LogisticRegression(max_iter=200, C=1.0).fit(Xtr, ytr)
print(clf.predict_proba(Xte[:3]))