SVM

import numpy as np
from sklearn.datasets import load_wine, fetch_california_housing
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC, SVR, LinearSVC
from sklearn.metrics import classification_report, mean_absolute_error

# === Classification ===
data = load_wine()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.2, stratify=data.target, random_state=42
)

# 1) ALWAYS scale before SVM — distances dominate
pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('svm',    SVC(kernel='rbf', C=1.0, gamma='scale', probability=True, random_state=42)),
])
pipe.fit(X_train, y_train)
print(classification_report(y_test, pipe.predict(X_test), target_names=data.target_names))

# 2) Common kernels
# - linear  : fast, high-dim sparse data (text), interpretable
# - rbf     : default — smooth non-linear boundaries
# - poly    : polynomial features without explicit expansion
# - sigmoid : neural-net-like; rarely best

# 3) Hyperparameters
# C       : regularisation strength (1/lambda). Higher = fit train more.
# gamma   : RBF kernel width. Lower = smoother. 'scale' = 1/(n_features * X.var())
# degree  : for poly kernel
# class_weight : 'balanced' for imbalanced targets

param_grid = {
    'svm__C':     [0.1, 1, 10, 100],
    'svm__gamma': ['scale', 'auto', 0.01, 0.1, 1],
    'svm__kernel':['rbf', 'poly'],
}
grid = GridSearchCV(pipe, param_grid, cv=5, n_jobs=-1, scoring='f1_macro')
grid.fit(X_train, y_train)
print(grid.best_params_, grid.best_score_)

# 4) LinearSVC — faster, scales to MANY samples + features
linear = Pipeline([
    ('scaler', StandardScaler()),
    ('svm',    LinearSVC(C=1.0, max_iter=10_000, dual=False, random_state=42)),
])
linear.fit(X_train, y_train)
# Use LinearSVC instead of SVC(kernel='linear') for n > 10_000 samples — much faster.

# 5) Decision function + probability
probs = pipe.predict_proba(X_test)[:, 1]
scores = pipe.decision_function(X_test)        # raw margin (signed distance)

# 6) Regression — SVR
hx, hy = fetch_california_housing(return_X_y=True)
rx_tr, rx_te, ry_tr, ry_te = train_test_split(hx, hy, test_size=0.2, random_state=42)
reg = Pipeline([
    ('scaler', StandardScaler()),
    ('svr',    SVR(kernel='rbf', C=10, gamma='scale', epsilon=0.1)),
]).fit(rx_tr, ry_tr)
print('MAE:', mean_absolute_error(ry_te, reg.predict(rx_te)))

# 7) Imbalanced classification — class_weight
SVC(kernel='rbf', class_weight='balanced')
# Or pass {0: 1, 1: 10} to manually weight

# 8) Out-of-the-box pitfalls
# - Scaling: forgetting it = your SVM trains on whichever feature has the biggest variance
# - C too high: overfits training data; check with CV
# - gamma too high: every point becomes its own region (overfit)
# - SVC scales poorly past ~100k samples — use LinearSVC or SGDClassifier(loss='hinge')

# 9) When SVMs are the right pick
# - Small-to-medium tabular data (1k-100k rows)
# - High-dimensional sparse data (text, after TF-IDF) → LinearSVC
# - You need a strong non-linear baseline without tuning a neural network
# Otherwise reach for gradient boosting (LightGBM, XGBoost, HistGBDT)

# 10) Inspect the model
clf = grid.best_estimator_.named_steps['svm']
print('support vectors:', clf.n_support_)
print('support indices:', clf.support_[:10])
# Only support vectors define the boundary; the rest don't matter.

from sklearn.svm import SVC
clf = SVC(kernel='rbf', C=1.0, gamma='scale').fit(Xtr, ytr)