Model Evaluation & Tuning

Classification models predict categories, so we need metrics that measure how often predictions are correct. But accuracy alone can be misleading - especially with imbalanced datasets. Let's understand the full toolkit.

The Accuracy Trap

Consider a fraud detection model where 99% of transactions are legitimate. A model that predicts "not fraud" for everything achieves 99% accuracy - but catches zero fraudsters! This is why we need precision, recall, and F1 score.

Accuracy

# Accuracy Score
from sklearn.metrics import accuracy_score

y_true = [1, 0, 1, 1, 0, 1, 0, 0, 1, 1]
y_pred = [1, 0, 1, 0, 0, 1, 1, 0, 1, 1]

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy:.2%}")  # 80%

Precision

# Precision Score
from sklearn.metrics import precision_score

precision = precision_score(y_true, y_pred)
print(f"Precision: {precision:.2%}")
# "Of all emails I marked as spam, X% were actually spam"

Recall (Sensitivity, True Positive Rate)

# Recall Score
from sklearn.metrics import recall_score

recall = recall_score(y_true, y_pred)
print(f"Recall: {recall:.2%}")
# "Of all actual spam emails, I caught X%"

F1 Score - The Balance

# F1 Score
from sklearn.metrics import f1_score

f1 = f1_score(y_true, y_pred)
print(f"F1 Score: {f1:.2%}")

Complete Classification Report

# Complete classification evaluation
from sklearn.metrics import classification_report, accuracy_score
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier

# Generate imbalanced data
X, y = make_classification(
    n_samples=1000, n_features=10, n_classes=2,
    weights=[0.9, 0.1],  # 90% class 0, 10% class 1
    random_state=42
)

# Split and train
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
model = RandomForestClassifier(random_state=42)
model.fit(X_train, y_train)

# Predict
y_pred = model.predict(X_test)

# Classification report - shows all metrics per class
print("CLASSIFICATION REPORT")
print("=" * 50)
print(classification_report(y_test, y_pred, target_names=['Negative', 'Positive']))

# For cancer detection, optimize for recall
from sklearn.metrics import recall_score, make_scorer

# You can use this as scoring in GridSearchCV
recall_scorer = make_scorer(recall_score)

# Manual calculation
TP, FP, FN, TN = 80, 20, 10, 90

precision = TP / (TP + FP)  # 80 / 100 = 0.80
recall = TP / (TP + FN)     # 80 / 90 = 0.889
f1 = 2 * (precision * recall) / (precision + recall)  # 0.842

print(f"Precision: {precision:.3f}")
print(f"Recall: {recall:.3f}")
print(f"F1 Score: {f1:.3f}")

# Verify with sklearn
from sklearn.metrics import precision_score, recall_score, f1_score

# Reconstruct y_true and y_pred from confusion matrix values
y_true = [1]*TP + [1]*FN + [0]*FP + [0]*TN
y_pred = [1]*TP + [0]*FN + [1]*FP + [0]*TN

print(f"\nSklearn verification:")
print(f"Precision: {precision_score(y_true, y_pred):.3f}")
print(f"Recall: {recall_score(y_true, y_pred):.3f}")
print(f"F1: {f1_score(y_true, y_pred):.3f}")

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report

# Load multi-class data
iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.3, random_state=42
)

# Train and predict
model = RandomForestClassifier(random_state=42)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)

# Classification report
print(classification_report(y_test, y_pred, target_names=iris.target_names))

# Macro avg: Simple average across classes (treats all classes equally)
# Weighted avg: Average weighted by support (class frequency)
# Use macro when all classes are equally important
# Use weighted when you want to account for class imbalance

The confusion matrix gives you the complete picture of classification performance, while ROC curves help you understand performance across all thresholds.

The Confusion Matrix

# Creating and visualizing a confusion matrix
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

# Sample predictions
y_true = [1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1]
y_pred = [1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1]

# Create confusion matrix
cm = confusion_matrix(y_true, y_pred)
print("Confusion Matrix:")
print(cm)
# [[4, 2],    # TN=4, FP=2
#  [2, 7]]    # FN=2, TP=7

# Visualize with heatmap
fig, ax = plt.subplots(figsize=(8, 6))
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Negative', 'Positive'])
disp.plot(cmap='Blues', ax=ax)
plt.title('Confusion Matrix')
plt.show()

Normalized Confusion Matrix

Raw counts can be misleading with imbalanced classes. Normalize to see proportions!

# Normalized confusion matrix (row-wise)
cm_normalized = confusion_matrix(y_true, y_pred, normalize='true')
print("Normalized Confusion Matrix (by true label):")
print(cm_normalized)

# Visualize
disp = ConfusionMatrixDisplay(confusion_matrix=cm_normalized, 
                               display_labels=['Negative', 'Positive'])
disp.plot(cmap='Blues', values_format='.2%')
plt.title('Normalized Confusion Matrix')
plt.show()

ROC Curve

# ROC Curve
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

# Generate sample data
X, y = make_classification(n_samples=500, n_features=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Train model and get probability predictions
model = LogisticRegression()
model.fit(X_train, y_train)
y_proba = model.predict_proba(X_test)[:, 1]  # Probability of positive class

# Calculate ROC curve
fpr, tpr, thresholds = roc_curve(y_test, y_proba)

# Plot ROC curve
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, 'b-', linewidth=2, label='Model')
plt.plot([0, 1], [0, 1], 'r--', label='Random Classifier')
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR)')
plt.title('ROC Curve')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

AUC Score

# AUC Score
from sklearn.metrics import roc_auc_score

auc = roc_auc_score(y_test, y_proba)
print(f"AUC Score: {auc:.3f}")

# Interpretation:
# AUC = 0.5  → Random guessing
# AUC = 0.7  → Acceptable
# AUC = 0.8  → Good
# AUC = 0.9+ → Excellent

Precision-Recall Curve (Alternative)

# Precision-Recall Curve (better for imbalanced data)
from sklearn.metrics import precision_recall_curve, average_precision_score
import matplotlib.pyplot as plt

# Calculate precision-recall curve
precision, recall, thresholds = precision_recall_curve(y_test, y_proba)
ap = average_precision_score(y_test, y_proba)

# Plot
plt.figure(figsize=(8, 6))
plt.plot(recall, precision, 'b-', linewidth=2)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title(f'Precision-Recall Curve (AP = {ap:.3f})')
plt.grid(True, alpha=0.3)
plt.show()

print(f"Average Precision: {ap:.3f}")

from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt

# Train two models
rf = RandomForestClassifier(random_state=42)
lr = LogisticRegression(random_state=42)

rf.fit(X_train, y_train)
lr.fit(X_train, y_train)

# Get probabilities
rf_proba = rf.predict_proba(X_test)[:, 1]
lr_proba = lr.predict_proba(X_test)[:, 1]

# Calculate ROC curves
rf_fpr, rf_tpr, _ = roc_curve(y_test, rf_proba)
lr_fpr, lr_tpr, _ = roc_curve(y_test, lr_proba)

# Calculate AUC
rf_auc = roc_auc_score(y_test, rf_proba)
lr_auc = roc_auc_score(y_test, lr_proba)

# Plot both
plt.figure(figsize=(8, 6))
plt.plot(rf_fpr, rf_tpr, 'b-', label=f'Random Forest (AUC={rf_auc:.3f})')
plt.plot(lr_fpr, lr_tpr, 'g-', label=f'Logistic Reg (AUC={lr_auc:.3f})')
plt.plot([0, 1], [0, 1], 'r--', label='Random')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve Comparison')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

# Confusion matrix format in sklearn:
# [[TN, FP],
#  [FN, TP]]

TN, FP = 85, 15
FN, TP = 25, 75

print(f"True Negatives (TN): {TN}")
print(f"False Positives (FP): {FP}")
print(f"False Negatives (FN): {FN}")
print(f"True Positives (TP): {TP}")

accuracy = (TP + TN) / (TP + TN + FP + FN)
print(f"\nAccuracy: {accuracy:.2%}")  # 80%

from sklearn.metrics import roc_curve
import numpy as np

# Get ROC curve values
fpr, tpr, thresholds = roc_curve(y_test, y_proba)

# Youden's J statistic: J = TPR - FPR (maximize this)
j_scores = tpr - fpr
optimal_idx = np.argmax(j_scores)
optimal_threshold = thresholds[optimal_idx]

print(f"Optimal Threshold: {optimal_threshold:.3f}")
print(f"At this threshold:")
print(f"  TPR (Recall): {tpr[optimal_idx]:.3f}")
print(f"  FPR: {fpr[optimal_idx]:.3f}")

# Apply optimal threshold
y_pred_optimal = (y_proba >= optimal_threshold).astype(int)

# Compare with default 0.5 threshold
from sklearn.metrics import f1_score
print(f"\nF1 with 0.5 threshold: {f1_score(y_test, y_proba >= 0.5):.3f}")
print(f"F1 with optimal threshold: {f1_score(y_test, y_pred_optimal):.3f}")

Training on one split and testing on another can give lucky or unlucky results. Cross-validation provides a more robust estimate of model performance by using multiple splits.

Why Cross-Validation?

K-Fold Cross-Validation

# ============================================
# K-Fold Cross-Validation - Complete Example
# ============================================
from sklearn.model_selection import cross_val_score, KFold
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import numpy as np

# Generate sample data
X, y = make_classification(
    n_samples=1000,      # 1000 samples
    n_features=20,       # 20 features
    n_classes=2,         # Binary classification
    random_state=42
)

print(f"Dataset: {X.shape[0]} samples, {X.shape[1]} features")
print()

# Create model (not trained yet!)
model = RandomForestClassifier(random_state=42)

# METHOD 1: Simple 5-fold CV (one line!)
print("=== Method 1: Simple 5-Fold CV ===")
scores = cross_val_score(
    model,              # The model to evaluate
    X, y,               # Data
    cv=5,               # Number of folds
    scoring='accuracy'  # Metric to use
)

print(f"Scores for each fold: {scores}")
print(f"Mean accuracy: {scores.mean():.3f}")
print(f"Std deviation: {scores.std():.3f}")
print(f"95% Confidence: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})")
print()

# METHOD 2: More control with KFold object
print("=== Method 2: Explicit KFold with Shuffle ===")
kfold = KFold(
    n_splits=5,          # 5 folds
    shuffle=True,        # IMPORTANT: Shuffle before splitting!
    random_state=42      # For reproducibility
)

scores = cross_val_score(model, X, y, cv=kfold)
print(f"Shuffled KFold mean: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})")
print()

# WHY SHUFFLE?
print("=== Why Shuffle Matters ===")
kfold_no_shuffle = KFold(n_splits=5, shuffle=False)  # Don't shuffle
scores_no_shuffle = cross_val_score(model, X, y, cv=kfold_no_shuffle)

print(f"Without shuffle: {scores_no_shuffle.mean():.3f}")
print(f"With shuffle:    {scores.mean():.3f}")
print("→ If data is ordered (e.g., all class 0 first, then class 1),")
print("→ not shuffling gives biased results!")

Stratified K-Fold

# Stratified K-Fold (preserves class proportions)
from sklearn.model_selection import StratifiedKFold

# Imbalanced data (90% class 0, 10% class 1)
X, y = make_classification(n_samples=1000, weights=[0.9, 0.1], random_state=42)

print(f"Class distribution: {np.bincount(y)}")  # [900, 100]

# Stratified ensures each fold has ~90/10 split
strat_kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

scores = cross_val_score(model, X, y, cv=strat_kfold, scoring='f1')
print(f"Stratified KFold F1: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})")

Leave-One-Out (LOO)

# Leave-One-Out CV (expensive!)
from sklearn.model_selection import LeaveOneOut, cross_val_score
from sklearn.neighbors import KNeighborsClassifier

# Small dataset
X_small, y_small = make_classification(n_samples=50, n_features=5, random_state=42)

# LOO CV
loo = LeaveOneOut()
model = KNeighborsClassifier(n_neighbors=3)

scores = cross_val_score(model, X_small, y_small, cv=loo)
print(f"LOO CV Accuracy: {scores.mean():.3f}")
print(f"Number of folds: {len(scores)}")  # 50 folds!

Time Series Split

For time series data, you can't randomly shuffle! The training set must always come before the test set in time.

# Time Series Split (for temporal data)
from sklearn.model_selection import TimeSeriesSplit
import matplotlib.pyplot as plt

# Time series CV - always train on past, test on future
tscv = TimeSeriesSplit(n_splits=5)

# Visualize the splits
X_time = np.arange(100)  # Simulated time series indices

fig, axes = plt.subplots(5, 1, figsize=(10, 8))
for i, (train_idx, test_idx) in enumerate(tscv.split(X_time)):
    axes[i].scatter(train_idx, [1]*len(train_idx), c='blue', label='Train', s=10)
    axes[i].scatter(test_idx, [1]*len(test_idx), c='red', label='Test', s=10)
    axes[i].set_title(f'Split {i+1}')
    axes[i].set_yticks([])
plt.tight_layout()
plt.show()

Multiple Metrics in CV

# Cross-validation with multiple metrics
from sklearn.model_selection import cross_validate
from sklearn.ensemble import GradientBoostingClassifier

model = GradientBoostingClassifier(random_state=42)

# Evaluate multiple metrics at once
scoring = ['accuracy', 'precision', 'recall', 'f1', 'roc_auc']
cv_results = cross_validate(model, X, y, cv=5, scoring=scoring, return_train_score=True)

# Print results
print("CROSS-VALIDATION RESULTS")
print("=" * 50)
for metric in scoring:
    train_key = f'train_{metric}'
    test_key = f'test_{metric}'
    print(f"{metric.upper()}")
    print(f"  Train: {cv_results[train_key].mean():.3f} (+/- {cv_results[train_key].std():.3f})")
    print(f"  Test:  {cv_results[test_key].mean():.3f} (+/- {cv_results[test_key].std():.3f})")

# Nested Cross-Validation
from sklearn.model_selection import cross_val_score, GridSearchCV, StratifiedKFold
from sklearn.svm import SVC

# Outer CV for performance estimation
# Inner CV for hyperparameter tuning
outer_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)

# Hyperparameter search (inner loop)
param_grid = {'C': [0.1, 1, 10], 'kernel': ['rbf', 'linear']}
grid_search = GridSearchCV(SVC(), param_grid, cv=inner_cv, scoring='accuracy')

# Outer loop for unbiased estimation
nested_scores = cross_val_score(grid_search, X, y, cv=outer_cv, scoring='accuracy')

print(f"Nested CV Accuracy: {nested_scores.mean():.3f} (+/- {nested_scores.std():.3f})")
print("This estimate is unbiased - it doesn't 'see' the test data during tuning!")

from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

# Create model
model = RandomForestClassifier(n_estimators=100, random_state=42)

# 5-fold CV
scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')

print(f"Individual fold scores: {scores}")
print(f"Mean accuracy: {scores.mean():.3f}")
print(f"Standard deviation: {scores.std():.3f}")
print(f"95% CI: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})")

from sklearn.model_selection import cross_validate
from sklearn.ensemble import GradientBoostingClassifier

model = GradientBoostingClassifier(random_state=42)

# Multiple metrics at once
scoring = ['accuracy', 'precision', 'recall', 'f1']

cv_results = cross_validate(
    model, X, y, cv=5, 
    scoring=scoring, 
    return_train_score=True
)

# Print formatted results
print("CROSS-VALIDATION RESULTS")
print("=" * 50)
for metric in scoring:
    train_mean = cv_results[f'train_{metric}'].mean()
    train_std = cv_results[f'train_{metric}'].std()
    test_mean = cv_results[f'test_{metric}'].mean()
    test_std = cv_results[f'test_{metric}'].std()
    
    print(f"\n{metric.upper()}:")
    print(f"  Train: {train_mean:.3f} (+/- {train_std:.3f})")
    print(f"  Test:  {test_mean:.3f} (+/- {test_std:.3f})")
    
    # Check for overfitting
    if train_mean - test_mean > 0.1:
        print(f"  Warning: Gap suggests overfitting!")

Hyperparameters are settings you choose before training - like the number of trees in a forest or the learning rate. Finding the best combination is crucial for model performance.

What are Hyperparameters?

GridSearchCV - Exhaustive Search

# ============================================
# GridSearchCV - Complete Example
# ============================================
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import time

# Generate sample data
X, y = make_classification(n_samples=1000, n_features=20, random_state=42)

print("=== Setting Up GridSearch ===")

# Define the base model (hyperparameters will be set by GridSearch)
rf = RandomForestClassifier(random_state=42)

# Define hyperparameter grid
# Start with a COARSE grid (few values) to explore broadly
param_grid = {
    'n_estimators': [50, 100, 200],           # Number of trees - 3 values
    'max_depth': [5, 10, 20, None],           # Max tree depth - 4 values  
    'min_samples_split': [2, 5, 10],          # Min samples to split - 3 values
    'min_samples_leaf': [1, 2, 4]             # Min samples in leaf - 3 values
}

print(f"Parameter grid:")
for param, values in param_grid.items():
    print(f"  {param}: {values}")

# Calculate total combinations
total_combinations = 1
for values in param_grid.values():
    total_combinations *= len(values)

print(f"\nTotal combinations: {total_combinations}")
print(f"With 5-fold CV: {total_combinations * 5} model fits!")
print("\nStarting GridSearch... (this may take a while)\n")

# Create GridSearchCV object
grid_search = GridSearchCV(
    estimator=rf,                    # Model to tune
    param_grid=param_grid,           # Grid of parameters
    cv=5,                            # 5-fold cross-validation
    scoring='accuracy',              # Metric to optimize
    n_jobs=-1,                       # Use ALL CPU cores (parallel)
    verbose=1,                       # Show progress (2=more detail, 3=even more)
    return_train_score=True          # Also return training scores
)

# Fit - this tries all combinations!
start_time = time.time()
grid_search.fit(X, y)
elapsed = time.time() - start_time

# Display results
print("\n" + "="*50)
print("GRIDSEARCH RESULTS")
print("="*50)
print(f"Time taken: {elapsed:.1f} seconds")
print(f"\nBest parameters found:")
for param, value in grid_search.best_params_.items():
    print(f"  {param}: {value}")

print(f"\nBest cross-validation score: {grid_search.best_score_:.4f}")

# Access the best model (already trained!)
best_model = grid_search.best_estimator_
print(f"\nBest model: {best_model}")

# Get all results as a DataFrame (useful for analysis)
import pandas as pd
results_df = pd.DataFrame(grid_search.cv_results_)

# Show top 5 parameter combinations
print("\nTop 5 parameter combinations:")
top_5 = results_df.nsmallest(5, 'rank_test_score')[[
    'params', 'mean_test_score', 'std_test_score', 'rank_test_score'
]]
print(top_5.to_string(index=False))

RandomizedSearchCV - Faster Alternative

# RandomizedSearchCV - Random sampling
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform

# Define distributions for random sampling
param_distributions = {
    'n_estimators': randint(50, 500),           # Sample from 50-500
    'max_depth': randint(5, 50),                # Sample from 5-50
    'min_samples_split': randint(2, 20),        # Sample from 2-20
    'min_samples_leaf': randint(1, 10),         # Sample from 1-10
    'max_features': uniform(0.1, 0.9)           # Sample from 0.1-1.0
}

# Random search
random_search = RandomizedSearchCV(
    estimator=rf,
    param_distributions=param_distributions,
    n_iter=50,        # Try 50 random combinations
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    random_state=42,
    verbose=1
)

random_search.fit(X, y)

print(f"Best parameters: {random_search.best_params_}")
print(f"Best CV score: {random_search.best_score_:.3f}")

Analyzing Search Results

# Analyzing search results
import pandas as pd
import matplotlib.pyplot as plt

# Get results as DataFrame
results_df = pd.DataFrame(grid_search.cv_results_)

# View key columns
print(results_df[['params', 'mean_test_score', 'std_test_score', 'rank_test_score']].head(10))

# Plot hyperparameter importance (example with max_depth)
depths = [p['max_depth'] for p in results_df['params']]
scores = results_df['mean_test_score']

plt.figure(figsize=(10, 5))
plt.scatter(depths, scores, alpha=0.5)
plt.xlabel('max_depth')
plt.ylabel('Mean CV Score')
plt.title('Effect of max_depth on Performance')
plt.show()

Best Practices

Common Hyperparameters by Model

# Complete tuning pipeline
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import classification_report

# Split data - keep test set completely separate!
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# Create pipeline
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('svc', SVC(random_state=42))
])

# Define parameter grid (note: prefix with step name__)
param_grid = {
    'svc__C': [0.1, 1, 10, 100],
    'svc__kernel': ['rbf', 'linear'],
    'svc__gamma': ['scale', 'auto', 0.1, 1]
}

# Grid search on training data only
grid_search = GridSearchCV(
    pipeline, param_grid, cv=5, 
    scoring='f1', n_jobs=-1
)
grid_search.fit(X_train, y_train)

print(f"Best params: {grid_search.best_params_}")
print(f"Best CV F1: {grid_search.best_score_:.3f}")

# Evaluate on held-out test set
y_pred = grid_search.predict(X_test)
print("\n" + "="*50)
print("TEST SET PERFORMANCE (Never seen during tuning!)")
print("="*50)
print(classification_report(y_test, y_pred))

from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier
import pandas as pd

# Define model and parameter grid
knn = KNeighborsClassifier()
param_grid = {'n_neighbors': [3, 5, 7, 9, 11]}

# Grid search
grid_search = GridSearchCV(
    knn, param_grid, cv=5, 
    scoring='accuracy'
)
grid_search.fit(X, y)

print(f"Best K: {grid_search.best_params_['n_neighbors']}")
print(f"Best CV Accuracy: {grid_search.best_score_:.3f}")

# View all results
results = pd.DataFrame(grid_search.cv_results_)
print(results[['param_n_neighbors', 'mean_test_score', 'rank_test_score']])

from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
from scipy.stats import randint, uniform

# Model
rf = RandomForestClassifier(random_state=42)

# Parameter distributions (not fixed values!)
param_dist = {
    'n_estimators': randint(50, 500),
    'max_depth': randint(3, 30),
    'min_samples_split': randint(2, 20),
    'min_samples_leaf': randint(1, 10),
    'max_features': uniform(0.1, 0.9)
}

# Random search - try 20 random combinations
random_search = RandomizedSearchCV(
    rf, param_dist, 
    n_iter=20,  # Only 20 combinations instead of all
    cv=5, 
    scoring='accuracy',
    random_state=42,
    n_jobs=-1
)
random_search.fit(X, y)

print(f"Best params: {random_search.best_params_}")
print(f"Best CV Accuracy: {random_search.best_score_:.3f}")

Use these tables to quickly decide which metrics, CV strategy, and tuning method to use for your specific problem.

When to Use Each Metric

CV Strategy Quick Reference

Tuning Method Decision Matrix