Data & Splits

# ===== Three axes you must control =====
# 1. The labels — what are you actually predicting?
# 2. The splits — train, validation, test must be DISJOINT
# 3. The features — what is the model allowed to see at inference time?

# ===== Splitting (the most common bug) =====
from sklearn.model_selection import train_test_split

# Stratified for classification:
X_train, X_temp, y_train, y_temp = train_test_split(
    X, y, test_size=0.30, stratify=y, random_state=42
)
X_val, X_test, y_val, y_test = train_test_split(
    X_temp, y_temp, test_size=0.50, stratify=y_temp, random_state=42
)
# 70% train, 15% val, 15% test.

# Time series: split by time, not random:
df = df.sort_values('date')
cut = int(len(df) * 0.8)
train, test = df.iloc[:cut], df.iloc[cut:]
# Mixing dates leaks the future into the past.

# Group-based: avoid splitting the same USER / SESSION / PATIENT across sets.
from sklearn.model_selection import GroupShuffleSplit
gss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
train_idx, test_idx = next(gss.split(X, y, groups=user_ids))

# ===== Leakage (the most expensive bug) =====
# Features that would not be available at prediction time:
#   - 'days_until_churn'        knows the future
#   - 'is_in_validation_set'    obvious
#   - target-encoded features fit on the full dataset
#   - normalisation fit on train+test together
# Always FIT pre-processing on train only, then TRANSFORM val + test.

# Use sklearn Pipelines to enforce this:
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
pipe = Pipeline([
    ('scale', StandardScaler()),
    ('clf', LogisticRegression(max_iter=2000)),
])
pipe.fit(X_train, y_train)
print(pipe.score(X_test, y_test))

# ===== Class imbalance =====
# Accuracy on 99-1 splits is meaningless.
# - Use F1, precision/recall, AUC, log-loss
# - Stratify the split
# - Class weights or upsample / downsample carefully
# - Calibrate probabilities before thresholding

# ===== Missing data =====
# Strategies (in order of preference):
# 1. Investigate WHY it is missing
# 2. Encode missingness as a feature flag
# 3. Median / mean impute within Pipeline (NEVER outside)
# 4. Model-based imputation (KNN, MICE)

# ===== Data quality checks (every pipeline) =====
assert X.shape[0] == y.shape[0]
assert not df.duplicated(subset=['user_id', 'event_id']).any()
assert df['amount'].between(0, 1_000_000).all()
assert df.date.dt.tz == pytz.UTC

# ===== Patterns to internalise =====
# - Train / val / test from the start; never tune on test
# - Stratify classification splits; time-aware splits for time series
# - Pipelines so pre-processing fits on train only
# - Assertions at boundaries; let bad data fail loudly

# ===== Pitfalls =====
# - Random split on time-series data (looks great, fails in prod)
# - Imputing with column means computed on the full dataset
# - target-encoding categorical features without inside-fold encoding
# - Tuning hyperparameters on the test set; reporting that as your honest number

from sklearn.model_selection import train_test_split
Xtr, Xte, ytr, yte = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y,
)