Introduction to Machine Learning

Machine learning algorithms fall into three main categories based on how they learn from data. Understanding these paradigms is essential for choosing the right approach for your problem.

Supervised Learning

In supervised learning, we train models using labeled data - examples where we know the correct answer. The model learns the relationship between inputs (features) and outputs (labels), then uses this knowledge to predict labels for new, unseen data.

Think of it like a teacher grading homework: the student (model) learns from corrected examples and gradually improves. The "supervision" comes from these labeled examples.

Classification Example: Customer Churn Prediction

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
import pandas as pd

# This is LABELED data - each customer has a known outcome
data = {
    "tenure_months": [12, 2, 48, 6, 24, 3, 36, 1, 60, 8],
    "monthly_charges": [50, 80, 45, 90, 55, 95, 40, 100, 42, 85],
    "support_tickets": [0, 5, 1, 8, 2, 6, 0, 10, 1, 7],
    "churned": [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]  # Labels: 0=stayed, 1=left
}
df = pd.DataFrame(data)

print("Training Data Sample:")
print(df.head(3))
print(f"\nTotal customers: {len(df)}")
print(f"Churned: {df['churned'].sum()}, Stayed: {(df['churned']==0).sum()}")

# Separate features from labels
X = df.drop("churned", axis=1)  # Features: what we observe
y = df["churned"]               # Labels: what we want to predict

print(f"Features shape: {X.shape}")  # (10, 3) - 10 customers, 3 features
print(f"Labels shape: {y.shape}")    # (10,) - 10 labels

Step 3: Train Random Forest Classifier

# Random Forest learns patterns like:
# - Low tenure + high charges = likely to churn
# - Many support tickets = likely to churn
model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X, y)

print("Model trained! It learned patterns from historical data.")

Step 4: Predict for New Customer

# Predict for a NEW customer (not in training data)
new_customer = [[4, 85, 3]]  # 4 months, $85/month, 3 support tickets

# Predict whether they will churn
prediction = model.predict(new_customer)
print(f"New customer profile: 4 months tenure, $85/month, 3 tickets")
print(f"Will churn? {'YES' if prediction[0] == 1 else 'NO'}")

Step 5: Get Prediction Probability

# Get probability - how confident is the model?
probability = model.predict_proba(new_customer)
churn_prob = probability[0][1]  # Probability of churning
print(f"Churn probability: {churn_prob:.1%}")

Step 6: Analyze Feature Importance

# Feature importance - what matters most?
importances = model.feature_importances_
feature_names = X.columns

print("Feature Importance:")
for name, importance in zip(feature_names, importances):
    print(f"  {name:20s}: {importance:.1%}")

Regression Example: House Price Prediction

from sklearn.linear_model import LinearRegression
import pandas as pd
import numpy as np

# Historical housing data with known sale prices
houses = {
    "sqft": [1200, 1800, 2400, 1500, 2000, 1000, 2200, 1600],
    "bedrooms": [2, 3, 4, 3, 3, 2, 4, 3],
    "age_years": [10, 5, 2, 15, 8, 20, 3, 12],
    "price": [200000, 350000, 500000, 280000, 400000, 150000, 480000, 320000]
}
df = pd.DataFrame(houses)

print("Historical House Data:")
print(df.to_string(index=False))
print(f"Price range: ${df['price'].min():,} to ${df['price'].max():,}")
print(f"Average price: ${df['price'].mean():,.0f}")

# Prepare features and target
X = df[["sqft", "bedrooms", "age_years"]]
y = df["price"]  # Continuous target (not categories!)

# Train regression model
# price = (coefficient_sqft * sqft) + (coefficient_bedrooms * bedrooms) + ...
model = LinearRegression()
model.fit(X, y)

print("Model Equation:")
print(f"price = {model.intercept_:,.0f}", end="")
for feature, coef in zip(X.columns, model.coef_):
    print(f" + ({coef:,.2f} * {feature})", end="")

# Interpret coefficients - what each feature contributes
print("What Each Feature Contributes:")
for feature, coef in zip(X.columns, model.coef_):
    if coef > 0:
        print(f"  {feature:12s}: +${coef:>10,.2f} per unit")
    else:
        print(f"  {feature:12s}: -${abs(coef):>10,.2f} per unit")

# Predict price for a NEW house
new_house = [[1700, 3, 7]]  # 1700 sqft, 3 bedrooms, 7 years old
predicted_price = model.predict(new_house)

print(f"New House Profile:")
print(f"  Square feet: 1,700")
print(f"  Bedrooms: 3")
print(f"  Age: 7 years")
print(f"Predicted Price: ${predicted_price[0]:,.0f}")

# Evaluate prediction accuracy
# R² score: how well the model fits the data (1.0 = perfect)
r2_score = model.score(X, y)
print(f"Model R² Score: {r2_score:.3f}")

# Compare predictions to actual prices
predictions = model.predict(X)
errors = y - predictions

print("Prediction Accuracy on Training Data:")
for i in range(min(3, len(df))):
    print(f"  House {i+1}: Actual=${y.iloc[i]:,}, Predicted=${predictions[i]:,.0f}")

Unsupervised Learning

Unsupervised learning works with unlabeled data - we don't know the correct answers. Instead, the algorithm finds hidden patterns, structures, or groupings in the data on its own. It's like asking someone to organize a messy closet without telling them how.

Common unsupervised tasks include clustering (grouping similar items), dimensionality reduction (simplifying complex data), and anomaly detection (finding outliers).

Clustering Example: Customer Segmentation

from sklearn.cluster import KMeans
import pandas as pd
import numpy as np

# Customer data WITHOUT any labels or categories
# We don't know which customers are "high value" or "low value"
customers = {
    "customer_id": [101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112],
    "annual_income": [15, 16, 17, 18, 19, 20, 80, 85, 90, 87, 92, 95],  # in thousands
    "spending_score": [39, 81, 6, 77, 40, 20, 76, 96, 94, 86, 98, 88]  # 1-100 scale
}
df = pd.DataFrame(customers)

print("Customer Data (Unlabeled):")
print(df.to_string(index=False))

# Prepare data for clustering
X = df[["annual_income", "spending_score"]]

# K-Means groups similar customers together
# n_clusters=3 means we want to find 3 distinct groups
kmeans = KMeans(n_clusters=3, random_state=42, n_init=10)
df["cluster"] = kmeans.fit_predict(X)

print(f"K-Means found 3 distinct customer segments!")

# Analyze each discovered cluster
print("Cluster Characteristics:")
for cluster_id in sorted(df["cluster"].unique()):
    cluster_data = df[df["cluster"] == cluster_id]
    print(f"\nCluster {cluster_id}: {len(cluster_data)} customers")
    print(f"  Avg Income:  ${cluster_data['annual_income'].mean():.0f}k")
    print(f"  Avg Spending: {cluster_data['spending_score'].mean():.0f}/100")

# Cluster centers - the "typical" customer in each segment
print("Cluster Centers (Typical Customer in Each Segment):")
centers = kmeans.cluster_centers_
for i, center in enumerate(centers):
    print(f"  Cluster {i}: Income=${center[0]:.0f}k, Spending={center[1]:.0f}/100")

# Predict cluster for NEW customer
new_customer = [[45, 65]]  # $45k income, 65 spending score
cluster_prediction = kmeans.predict(new_customer)
print(f"New Customer belongs to Cluster {cluster_prediction[0]}")

Once clusters are identified, you can predict which segment new customers belong to and immediately apply appropriate marketing strategies. This is how companies personalize experiences at scale - by grouping millions of customers into manageable segments with similar characteristics.

# Unsupervised Learning: Dimensionality Reduction
# Reduce complex data to 2D for visualization
from sklearn.decomposition import PCA
from sklearn.datasets import load_digits
import matplotlib.pyplot as plt

# Load handwritten digits (64 features per image)
digits = load_digits()
X = digits.data  # Shape: (1797, 64)

# Reduce to 2 dimensions
pca = PCA(n_components=2)
X_reduced = pca.fit_transform(X)

print(f"Original shape: {X.shape}")      # (1797, 64)
print(f"Reduced shape: {X_reduced.shape}")  # (1797, 2)

# Now we can visualize 64-dimensional data in 2D!
# Each point is a digit, colors represent actual digit values

Reinforcement Learning

Reinforcement learning is different from both supervised and unsupervised learning. An agent learns by interacting with an environment, receiving rewards or penalties for its actions. The goal is to maximize cumulative reward over time.

Think of training a dog: you don't show it labeled examples, but you reward good behavior and discourage bad behavior. The dog learns which actions lead to treats.

# Reinforcement Learning: Conceptual Example
# (Simplified - real RL uses libraries like Gym, Stable-Baselines)

class SimpleAgent:
    def __init__(self):
        self.q_values = {}  # Learned action values
        self.learning_rate = 0.1
    
    def get_action(self, state, explore=True):
        """Choose action based on learned values"""
        if explore and random.random() < 0.1:
            return random.choice(["left", "right"])
        return max(self.q_values.get(state, {}), 
                   key=lambda a: self.q_values.get(state, {}).get(a, 0),
                   default="right")
    
    def learn(self, state, action, reward, next_state):
        """Update values based on reward received"""
        current = self.q_values.setdefault(state, {}).get(action, 0)
        future = max(self.q_values.get(next_state, {}).values(), default=0)
        self.q_values[state][action] = current + self.learning_rate * (
            reward + 0.9 * future - current
        )

# The agent learns through experience:
# - Take action -> Get reward -> Update beliefs -> Repeat
# - Over time, learns which actions lead to highest rewards

Comparison of Learning Types

purchases = {
    "frequency": [2, 3, 15, 18, 20, 1, 2, 16, 19, 3],
    "avg_amount": [50, 45, 200, 180, 220, 30, 55, 190, 210, 40]
}

from sklearn.cluster import KMeans
import pandas as pd

purchases = {
    "frequency": [2, 3, 15, 18, 20, 1, 2, 16, 19, 3],
    "avg_amount": [50, 45, 200, 180, 220, 30, 55, 190, 210, 40]
}
df = pd.DataFrame(purchases)

kmeans = KMeans(n_clusters=2, random_state=42)
df["segment"] = kmeans.fit_predict(df)

print("Cluster centers:")
print(kmeans.cluster_centers_)
# [[2.2, 44.0], [17.6, 200.0]]  - Low vs High value customers

from sklearn.datasets import load_wine

from sklearn.datasets import load_wine
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

# Load dataset
wine = load_wine()
X, y = wine.data, wine.target

# Split 70/30
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=42
)

# Train model
model = LogisticRegression(max_iter=5000)
model.fit(X_train, y_train)

# Evaluate
accuracy = model.score(X_test, y_test)
print(f"Accuracy: {accuracy:.2%}")  # Accuracy: 98.15%

from sklearn.datasets import load_digits
from sklearn.decomposition import PCA

# Load digits (64 features per sample)
digits = load_digits()
X = digits.data

# Reduce to 3 components
pca = PCA(n_components=3)
X_reduced = pca.fit_transform(X)

print(f"Original shape: {X.shape}")
print(f"Reduced shape: {X_reduced.shape}")
print(f"\nExplained variance ratio:")
for i, ratio in enumerate(pca.explained_variance_ratio_):
    print(f"  Component {i+1}: {ratio:.2%}")
# Component 1: 14.89%, Component 2: 13.62%, Component 3: 11.79%

scenarios = [
    "Predict if a loan will default based on historical data with known outcomes",
    "Group news articles by topic without predefined categories",
    "Train a robot to walk by rewarding forward movement"
]

def identify_learning_type(scenario):
    scenario_lower = scenario.lower()
    
    # Check for supervised indicators
    if any(word in scenario_lower for word in 
           ["predict", "known outcomes", "labeled", "classify", "historical data with"]):
        if "without" not in scenario_lower:
            return "Supervised Learning"
    
    # Check for reinforcement indicators
    if any(word in scenario_lower for word in 
           ["reward", "robot", "agent", "trial and error", "game"]):
        return "Reinforcement Learning"
    
    # Check for unsupervised indicators
    if any(word in scenario_lower for word in 
           ["group", "cluster", "without predefined", "segment", "pattern"]):
        return "Unsupervised Learning"
    
    return "Unknown"

scenarios = [
    "Predict if a loan will default based on historical data with known outcomes",
    "Group news articles by topic without predefined categories",
    "Train a robot to walk by rewarding forward movement"
]

for scenario in scenarios:
    print(f"{identify_learning_type(scenario)}: {scenario[:50]}...")

Before deploying a model, we need to know how well it will perform on new, unseen data. The train-test split and cross-validation are fundamental techniques for evaluating model performance and preventing a common pitfall: overestimating how good your model really is.

Why We Split Data

Imagine studying for an exam by memorizing all the practice questions and answers. You'd score 100% on those exact questions, but would likely struggle with new questions on the real exam. Machine learning models can do the same thing - "memorize" training data without truly learning.

If we evaluate a model on the same data we trained it on, we get an overly optimistic estimate. The model might have memorized the training data rather than learning general patterns. To get an honest assessment, we must test on data the model has never seen.

Step 1: Load Sample Data

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris
from sklearn.tree import DecisionTreeClassifier
import numpy as np

# Iris dataset: flower measurements to predict species
iris = load_iris()
X, y = iris.data, iris.target

print("Original Dataset:")
print(f"  Total samples: {len(X)}")
print(f"  Features: {iris.feature_names}")
print(f"  Classes: {iris.target_names}")

Step 2: Split into Training and Test Sets

# Why 80/20? It's a common balance:
# - Enough training data (80%) for model to learn
# - Enough test data (20%) for reliable evaluation
X_train, X_test, y_train, y_test = train_test_split(
    X, y, 
    test_size=0.2,      # 20% for testing (30 samples)
    random_state=42,    # Fixed seed for reproducibility
    stratify=y          # Maintain class proportions
)

print(f"Training samples: {len(X_train)} ({len(X_train)/len(X)*100:.0f}%)")
print(f"Testing samples: {len(X_test)} ({len(X_test)/len(X)*100:.0f}%)")

Step 3: Train on Training Data Only

print("Training the model...")
model = DecisionTreeClassifier(random_state=42)
model.fit(X_train, y_train)  # Model learns ONLY from training set

print("Model trained! It has never seen the test set.")

Step 4: Evaluate on Both Sets

train_score = model.score(X_train, y_train)
test_score = model.score(X_test, y_test)

print(f"Training accuracy: {train_score:.2%}")
print(f"Test accuracy: {test_score:.2%}")
print(f"Difference: {abs(train_score - test_score):.2%}")

Step 5: Make Predictions

predictions = model.predict(X_test)

print("Sample Predictions (first 5):")
for i in range(5):
    actual = iris.target_names[y_test[i]]
    predicted = iris.target_names[predictions[i]]
    match = "✓" if y_test[i] == predictions[i] else "✗"
    print(f"  {match} Sample {i+1}: Actual={actual}, Predicted={predicted}")

The train-test split is foundational to machine learning. Think of it like studying for an exam: you practice with some problems (training set), but the real test has different problems (test set). If you only memorize practice problems, you'll fail the real test. Similarly, models need to learn general patterns, not memorize specific examples.

The random_state=42 parameter is important for reproducibility. Without it, you'd get different splits each time you run the code, making results difficult to compare. By fixing the random seed, everyone running this code gets the same split, which is crucial for debugging and collaboration.

The stratify=y parameter ensures balanced class distribution. If we have 50 samples of each flower species in the full dataset, stratification ensures our test set also has proportional representation (about 10 of each). Without this, random chance might give us a test set with no examples of one species!

When evaluating, we check both training and test accuracy. The training accuracy tells us if the model learned anything at all. The test accuracy tells us if it learned generalizable patterns. If training is 100% but test is 70%, we have a memorization problem (overfitting). If both are around 60%, our model is too simple (underfitting).

The Validation Set

When developing a model, you often need to tune hyperparameters (settings that control how the model learns, like tree depth or learning rate). These are different from model parameters, which the model learns from data. Hyperparameters are set by you before training.

Here's the problem: if you use the test set to choose hyperparameters, you're indirectly "peeking" at the test data. You might try 10 different settings, check test accuracy each time, and pick the best one. But now your test set influenced model design decisions - it's no longer truly unseen. You've contaminated your test set.

The solution is a three-way split: training (60-70% - builds the model), validation (15-20% - tunes hyperparameters), and test (15-20% - final evaluation only). The validation set is your "development test set" that you can use freely while developing. The test set remains locked away until the very end.

Three-Way Split: Train, Validation, Test

from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris
import numpy as np

iris = load_iris()
X, y = iris.data, iris.target
print(f"Total samples: {len(X)}")

# FIRST SPLIT: Separate test set (LOCK IT AWAY!)
# This test set will NOT be touched until final evaluation
X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
print(f"Test set: {len(X_test)} samples (20%) - LOCKED for final eval")

# SECOND SPLIT: Separate validation from training
# 0.25 of remaining 80% = 20% overall for validation
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42, stratify=y_temp
)

print(f"Training set: {len(X_train)} samples (60%)")
print(f"Validation set: {len(X_val)} samples (20%)")
print(f"Test set: {len(X_test)} samples (20%)")

# Use validation set to tune hyperparameters
print("Tuning n_estimators (number of trees):")

best_val_score = 0
best_n_estimators = 10

for n in [10, 25, 50, 100, 200]:
    model = RandomForestClassifier(n_estimators=n, random_state=42, max_depth=5)
    model.fit(X_train, y_train)
    
    # Evaluate on VALIDATION set (not test!)
    train_score = model.score(X_train, y_train)
    val_score = model.score(X_val, y_val)
    
    print(f"n_estimators={n:3d}: Train={train_score:.2%}, Val={val_score:.2%}")
    
    if val_score > best_val_score:
        best_val_score = val_score
        best_n_estimators = n

print(f"\nBest hyperparameter: n_estimators={best_n_estimators}")

# FINAL EVALUATION on test set (ONLY ONCE!)
# Combine train + validation for final training
X_train_full = np.vstack([X_train, X_val])
y_train_full = np.hstack([y_train, y_val])

final_model = RandomForestClassifier(
    n_estimators=best_n_estimators, max_depth=5, random_state=42
)
final_model.fit(X_train_full, y_train_full)

# NOW we can touch the test set
test_score = final_model.score(X_test, y_test)

print(f"Final test accuracy: {test_score:.2%}")
print(f"This is the score we report - test set was untouched until now!")

This three-way split is crucial for honest model evaluation. The training set teaches the model, the validation set helps you pick the best configuration, and the test set gives you an unbiased estimate of real-world performance. Without this separation, you risk overfitting to your test set through repeated tuning.

Notice how we tried multiple values for n_estimators and checked validation performance each time. We picked n=100 because it had the highest validation score. This is legitimate because the validation set is meant for this purpose. We never looked at test set performance during this process - that would defeat its purpose.

At the very end, we train a final model using both training and validation data combined (since we're done tuning), then evaluate once on the test set. This final test score is what we report as our model's expected performance on new data. If we had repeatedly tested different configurations on the test set, this score would be artificially inflated.

Think of it like a student taking practice exams (validation) to prepare, but the final exam (test) is only taken once. If the student knew the final exam questions beforehand and optimized their study strategy for those specific questions, their final exam score wouldn't reflect their true ability.

Cross-Validation

A single train-test split can be unreliable - what if by chance, easy examples ended up in the test set, making your model look better than it really is? Or what if difficult examples ended up in the test set, making it look worse? Cross-validation solves this problem by performing multiple train-test splits and averaging the results, giving you a much more stable and reliable estimate of model performance.

The most common approach is K-Fold Cross-Validation. Imagine dividing your deck of cards into 5 equal piles. You use 4 piles for training and 1 pile for testing, then repeat this 5 times, using each pile as the test set once. This way, every single example gets to be in the test set exactly once, and you get 5 different performance scores to average.

Let's see cross-validation in action with a practical example:

# Step 1: Import required libraries and load data
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_iris

# Load the iris dataset
iris = load_iris()
X, y = iris.data, iris.target

print("Dataset Information:")
print(f"  Total samples: {len(X)}")
print(f"  Features: {X.shape[1]}")
print(f"  Classes: {len(set(y))}")

We start by loading the Iris dataset, which contains 150 flower samples with 4 measurements each. This is a classic dataset for testing classification algorithms. Now let's create our model:

# Step 2: Create the model (but don't train yet!)
model = LogisticRegression(max_iter=200, random_state=42)

print("\nModel created: Logistic Regression")
print("Ready for cross-validation...")

Notice we haven't called .fit() yet. Cross-validation will handle the training internally, multiple times. Now comes the magic - performing 5-fold cross-validation:

# Step 3: Perform 5-fold cross-validation
# This will:
# - Split data into 5 folds
# - Train on 4 folds, test on 1 fold
# - Repeat 5 times, rotating which fold is the test set
# - Return 5 accuracy scores (one per fold)

scores = cross_val_score(model, X, y, cv=5)

print("\n5-Fold Cross-Validation Results:")
print("="*50)
for i, score in enumerate(scores, 1):
    print(f"  Fold {i}: {score:.2%} accuracy")

# Scores might look like: [0.967, 1.0, 0.933, 0.967, 1.0]

Each score represents how well the model performed on a different 20% of the data (30 samples). Notice the scores vary - Fold 2 and Fold 5 achieved 100% accuracy, while Fold 3 got 93.3%. This variation is normal and shows why a single split isn't enough. Now let's calculate summary statistics:

# Step 4: Calculate and interpret summary statistics
import numpy as np

mean_accuracy = scores.mean()
std_accuracy = scores.std()
min_accuracy = scores.min()
max_accuracy = scores.max()

print("\nSummary Statistics:")
print(f"  Mean accuracy: {mean_accuracy:.2%}")    # 97.33%
print(f"  Std deviation: {std_accuracy:.2%}")     # 2.49%
print(f"  Min accuracy: {min_accuracy:.2%}")      # 93.33%
print(f"  Max accuracy: {max_accuracy:.2%}")      # 100.00%
print(f"  95% Confidence Interval: {mean_accuracy:.2%} ± {1.96*std_accuracy:.2%}")

# Interpretation
if std_accuracy < 0.05:
    print("\n✓ Low standard deviation - consistent performance across folds")
else:
    print("\n⚠ High standard deviation - performance varies significantly")

The mean tells us the expected performance (97.33%), while the standard deviation (2.49%) tells us how much the performance varies between folds. A low standard deviation means the model performs consistently regardless of which data it sees - a good sign! The 95% confidence interval gives us a range we're confident the true performance falls within.

Cross-validation is particularly powerful for comparing different algorithms to find which one works best for your data. Let's compare four popular algorithms:

# Step 1: Import multiple algorithms to compare
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC

print("Preparing model comparison...")

We'll test four different algorithms: Logistic Regression (linear model), Decision Tree (tree-based), Random Forest (ensemble of trees), and SVM (support vector machine). Each has different strengths and weaknesses. Now let's create instances of each:

# Step 2: Create a dictionary of models to test
models = {
    "Logistic Regression": LogisticRegression(max_iter=200, random_state=42),
    "Decision Tree": DecisionTreeClassifier(random_state=42),
    "Random Forest": RandomForestClassifier(n_estimators=100, random_state=42),
    "SVM": SVC(kernel='rbf', random_state=42)
}

print(f"\nTesting {len(models)} different algorithms")
print(f"Each will be evaluated with 5-fold cross-validation\n")

We organize our models in a dictionary for easy iteration. The random_state parameter ensures reproducible results. Now comes the comparison - we'll run 5-fold cross-validation for each model:

# Step 3: Evaluate each model with cross-validation
print("5-Fold Cross-Validation Results:")
print("="*60)
print(f"{'Algorithm':<25} {'Mean Accuracy':<15} {'95% CI'}")
print("-"*60)

results = {}
for name, model in models.items():
    # Perform 5-fold CV - trains model 5 times
    scores = cross_val_score(model, X, y, cv=5)
    
    # Store results
    results[name] = {
        'mean': scores.mean(),
        'std': scores.std(),
        'scores': scores
    }
    
    # Display with confidence interval
    mean = scores.mean()
    margin = scores.std() * 1.96  # 95% confidence
    print(f"{name:<25} {mean:.2%}           ± {margin:.2%}")

# Example output:
# Logistic Regression       97.33%          ± 4.88%
# Decision Tree             95.33%          ± 6.02%
# Random Forest             96.00%          ± 4.58%
# SVM                       97.33%          ± 3.92%

The results show Logistic Regression and SVM tied for best performance at 97.33% accuracy. However, SVM has a smaller confidence interval (±3.92% vs ±4.88%), suggesting more consistent performance across folds. The Random Forest did well but was slightly less accurate, while the Decision Tree showed both lower accuracy and higher variance. For this dataset, we'd likely choose SVM as our final model due to its combination of high accuracy and low variance.

Stratified Splitting

Imagine you're building a fraud detection system where only 2% of transactions are fraudulent. If you randomly split your data, your test set might end up with zero fraud cases, or your training set might be missing critical fraud examples. Stratified splitting solves this problem by ensuring each subset maintains the same class distribution as the original dataset. This is crucial for imbalanced classification problems.

Let's see the difference between regular and stratified splitting with a concrete example:

# Step 1: Create an imbalanced dataset
from sklearn.model_selection import train_test_split
import numpy as np

np.random.seed(42)

# Simulate 1000 credit card transactions
# 90% legitimate (class 0), 10% fraudulent (class 1)
X = np.random.randn(1000, 5)  # 5 features per transaction
y = np.array([0] * 900 + [1] * 100)  # 900 legitimate, 100 fraud

print("Dataset Overview:")
print(f"  Total transactions: {len(y)}")
print(f"  Legitimate (class 0): {(y==0).sum()} ({(y==0).sum()/len(y):.1%})")
print(f"  Fraudulent (class 1): {(y==1).sum()} ({(y==1).sum()/len(y):.1%})")

We've created a dataset mimicking a common real-world scenario: imbalanced classes. Now let's see what happens with a regular train-test split:

# Step 2: Regular split (no stratification)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Count fraud cases in each set
train_fraud = (y_train == 1).sum()
test_fraud = (y_test == 1).sum()
train_fraud_pct = train_fraud / len(y_train)
test_fraud_pct = test_fraud / len(y_test)

print("\nWithout Stratification:")
print(f"  Training set: {train_fraud}/{len(y_train)} fraud = {train_fraud_pct:.1%}")
print(f"  Test set: {test_fraud}/{len(y_test)} fraud = {test_fraud_pct:.1%}")
print(f"  Difference from original 10%: {abs(test_fraud_pct - 0.10)*100:.1f} percentage points")

# Example output: Test set might have 8% or 12% fraud - not exactly 10%

Notice the test set might have 8% fraud instead of 10%, or perhaps 12%. This variation happens because random splitting doesn't consider class labels. With small minority classes, this can cause significant problems. Now let's use stratified splitting:

# Step 3: Stratified split (maintains proportions)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y  # <-- Key parameter
)

# Count fraud cases in each set
train_fraud = (y_train == 1).sum()
test_fraud = (y_test == 1).sum()
train_fraud_pct = train_fraud / len(y_train)
test_fraud_pct = test_fraud / len(y_test)

print("\nWith Stratification:")
print(f"  Training set: {train_fraud}/{len(y_train)} fraud = {train_fraud_pct:.1%}")
print(f"  Test set: {test_fraud}/{len(y_test)} fraud = {test_fraud_pct:.1%}")
print(f"  Perfect match with original 10%!")

# Output: Both sets will have exactly 10% fraud cases
# Training: 80/800 = 10.0%
# Test: 20/200 = 10.0%

Perfect! With stratify=y, both training and test sets maintain exactly 10% fraud cases. This ensures your model trains on representative data and your evaluation is fair. Stratified splitting is especially critical when the minority class is small - imagine having only 50 fraud cases out of 10,000 transactions. A random split might give your test set zero fraud cases, making evaluation impossible!

Stratification also applies to cross-validation. Let's use StratifiedKFold to ensure each of the 5 folds maintains our 90-10 class distribution:

# Step 1: Create a stratified K-fold splitter
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression

# Create stratified 5-fold splitter
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

print("Setting up Stratified 5-Fold Cross-Validation")
print("Each fold will maintain 90-10 class distribution")

The shuffle=True parameter randomizes the data before splitting, while n_splits=5 creates 5 folds. Now let's run cross-validation:

# Step 2: Perform stratified cross-validation
model = LogisticRegression(max_iter=200, random_state=42)
scores = cross_val_score(model, X, y, cv=skf)

print("\nStratified 5-Fold CV Results:")
for i, score in enumerate(scores, 1):
    print(f"  Fold {i}: {score:.2%}")

print(f"\nSummary:")
print(f"  Mean accuracy: {scores.mean():.2%}")
print(f"  Std deviation: {scores.std():.2%}")
print("\n✓ Each fold maintained exact 90-10 split")

With stratified cross-validation, every single fold had exactly 180 legitimate and 20 fraudulent transactions in its test set. This consistency ensures reliable evaluation, especially for imbalanced datasets where random splits could create misleading results.

Common Split Ratios

from sklearn.datasets import load_breast_cancer

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split

# Load data
cancer = load_breast_cancer()
X, y = cancer.data, cancer.target

# Split 75/25
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.25, random_state=42
)

print(f"Training set: {len(X_train)} samples")  # 426
print(f"Test set: {len(X_test)} samples")       # 143

from sklearn.datasets import load_wine
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

wine = load_wine()
X, y = wine.data, wine.target

model = RandomForestClassifier(random_state=42)
scores = cross_val_score(model, X, y, cv=10)

print("All fold scores:", [f"{s:.2%}" for s in scores])
print(f"\nMean: {scores.mean():.2%}")
print(f"Min:  {scores.min():.2%}")
print(f"Max:  {scores.max():.2%}")

from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

# Load and split data
digits = load_digits()
X, y = digits.data, digits.target

# First split: 80/20
X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)
# Second split: 75/25 of remaining = 60/20 overall
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42
)

# Tune max_depth using validation set
best_depth = 3
best_val_score = 0

for depth in range(3, 11):
    model = DecisionTreeClassifier(max_depth=depth, random_state=42)
    model.fit(X_train, y_train)
    val_score = model.score(X_val, y_val)
    print(f"max_depth={depth}: validation = {val_score:.2%}")
    
    if val_score > best_val_score:
        best_val_score = val_score
        best_depth = depth

# Final evaluation
final_model = DecisionTreeClassifier(max_depth=best_depth, random_state=42)
final_model.fit(X_train, y_train)
test_score = final_model.score(X_test, y_test)
print(f"\nBest max_depth: {best_depth}")
print(f"Final test accuracy: {test_score:.2%}")

import numpy as np
from sklearn.model_selection import train_test_split

# Create imbalanced data
np.random.seed(42)
X = np.random.randn(500, 3)
y = np.array([0] * 400 + [1] * 100)  # 80/20 split

# Stratified split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

train_ratio = (y_train == 1).sum() / len(y_train)
test_ratio = (y_test == 1).sum() / len(y_test)

print(f"Original class 1 ratio: 20.0%")
print(f"Train class 1 ratio: {train_ratio:.1%}")
print(f"Test class 1 ratio: {test_ratio:.1%}")
# Both should be 20.0%

Every prediction error in machine learning can be decomposed into bias and variance. Understanding this tradeoff is key to building models that generalize well to new data and helps explain why simple models sometimes outperform complex ones.

Understanding Bias

Bias is the error from overly simplistic assumptions in the model. A high-bias model doesn't capture the true complexity of the data. It's like assuming all relationships are linear when they're actually curved - the model will consistently miss the mark.

Bias Example: Creating Data with Quadratic Relationship

import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error, r2_score

np.random.seed(42)
X = np.linspace(0, 10, 100).reshape(-1, 1)

# True relationship is quadratic (parabola)
y_true = 0.5 * X.ravel()**2  # y = 0.5 * x^2
y = y_true + np.random.randn(100) * 5  # Add random noise

print(f"True relationship: y = 0.5 * x² (parabola)")

Model 1: HIGH BIAS - Simple Linear Model

# Linear model tries to fit a straight line to curved data!
linear_model = LinearRegression()
linear_model.fit(X, y)
linear_pred = linear_model.predict(X)

linear_r2 = r2_score(y, linear_pred)
linear_mse = mean_squared_error(y, linear_pred)

print(f"Model equation: y = {linear_model.coef_[0]:.2f} * x + {linear_model.intercept_:.2f}")
print(f"R² Score: {linear_r2:.3f}")
print(f"MSE: {linear_mse:.2f}")

Model 2: LOWER BIAS - Polynomial Model

# Polynomial model can capture the quadratic relationship
poly_model = make_pipeline(
    PolynomialFeatures(degree=2),  # Creates x, x² features
    LinearRegression()
)
poly_model.fit(X, y)
poly_pred = poly_model.predict(X)

poly_r2 = r2_score(y, poly_pred)
poly_mse = mean_squared_error(y, poly_pred)

print(f"R² Score: {poly_r2:.3f}")
print(f"MSE: {poly_mse:.2f}")

Model 3: TOO COMPLEX - High Degree Polynomial

# Very complex model - degree=15
poly15_model = make_pipeline(
    PolynomialFeatures(degree=15),  # Very complex!
    LinearRegression()
)
poly15_model.fit(X, y)
poly15_pred = poly15_model.predict(X)

poly15_r2 = r2_score(y, poly15_pred)
poly15_mse = mean_squared_error(y, poly15_pred)

print(f"R² Score: {poly15_r2:.3f}")
print(f"MSE: {poly15_mse:.2f}")
print("Warning: Extremely high training performance suggests overfitting!")

Summary Comparison

print("COMPARISON SUMMARY")
print(f"{'Model':<25} {'R² Score':<12} {'Diagnosis'}")
print(f"{'Linear (degree=1)':<25} {linear_r2:<12.3f} High Bias - UNDERFITTING")
print(f"{'Polynomial (degree=2)':<25} {poly_r2:<12.3f} Good Fit")
print(f"{'Polynomial (degree=15)':<25} {poly15_r2:<12.3f} High Variance - OVERFITTING")

Understanding Variance

Variance is the error from sensitivity to small fluctuations in the training data. A high-variance model essentially memorizes the training data, including its noise. It performs well on training data but poorly on new data because it learned patterns that were just random noise.

Step 1: Import Libraries & Generate Data

# High Variance Example: Overly complex decision tree

from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import numpy as np

# Generate regression data with some noise
np.random.seed(42)
X = np.linspace(0, 10, 100).reshape(-1, 1)
y = 0.5 * X.ravel()**2 + np.random.randn(100) * 5

print("Dataset:")
print(f"  Samples: {len(X)}")
print(f"  True pattern: Quadratic with random noise")

Step 2: Split Data for Generalization Testing

# Split data to measure generalization
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

print(f"\nSplit: {len(X_train)} train, {len(X_test)} test")

Step 3: Model 1 - High Variance (Unlimited Depth Tree)

# Model 1: HIGH VARIANCE - Unlimited depth tree
print("\nModel 1: Decision Tree (NO depth limit) - HIGH VARIANCE")
print("="*60)

# Tree can grow as deep as needed - will memorize every detail!
deep_tree = DecisionTreeRegressor(random_state=42)  # No max_depth!
deep_tree.fit(X_train, y_train)

# Evaluate on both sets
train_pred = deep_tree.predict(X_train)
test_pred = deep_tree.predict(X_test)

train_r2 = r2_score(y_train, train_pred)
test_r2 = r2_score(y_test, test_pred)
train_mse = mean_squared_error(y_train, train_pred)
test_mse = mean_squared_error(y_test, test_pred)

print(f"\nTraining Performance:")
print(f"  R² Score: {train_r2:.3f}")
print(f"  MSE: {train_mse:.2f}")

print(f"\nTest Performance:")
print(f"  R² Score: {test_r2:.3f}")
print(f"  MSE: {test_mse:.2f}")

print(f"\nPerformance Gap:")
print(f"  R² difference: {train_r2 - test_r2:.3f}")
print(f"  MSE ratio: {test_mse / train_mse:.1f}x worse on test")

print(f"\n  PROBLEM: Perfect training, poor test = OVERFITTING!")
print(f"Tree depth: {deep_tree.get_depth()} (very deep!)")
print(f"Tree leaves: {deep_tree.get_n_leaves()} (too many!)")                              
print("The tree memorized training noise instead of learning patterns.")

Step 4: Model 2 - Balanced (Limited Depth Tree)

# Model 2: BALANCED - Limited depth tree
print("\n" + "="*60)
print("Model 2: Decision Tree (max_depth=3) - BALANCED")
print("="*60)

shallow_tree = DecisionTreeRegressor(max_depth=3, random_state=42)
shallow_tree.fit(X_train, y_train)

train_pred_shallow = shallow_tree.predict(X_train)
test_pred_shallow = shallow_tree.predict(X_test)

train_r2_shallow = r2_score(y_train, train_pred_shallow)
test_r2_shallow = r2_score(y_test, test_pred_shallow)
train_mse_shallow = mean_squared_error(y_train, train_pred_shallow)
test_mse_shallow = mean_squared_error(y_test, test_pred_shallow)

print(f"\nTraining Performance:")
print(f"  R² Score: {train_r2_shallow:.3f}")
print(f"  MSE: {train_mse_shallow:.2f}")

print(f"\nTest Performance:")
print(f"  R² Score: {test_r2_shallow:.3f}")
print(f"  MSE: {test_mse_shallow:.2f}")

print(f"\nPerformance Gap:")
print(f"  R² difference: {train_r2_shallow - test_r2_shallow:.3f}")
print(f"  MSE ratio: {test_mse_shallow / train_mse_shallow:.1f}x")

print(f"\n✓ BETTER: Training and test scores are closer!")
print(f"Tree depth: {shallow_tree.get_depth()}")
print(f"Tree leaves: {shallow_tree.get_n_leaves()}")
print("The tree learned general patterns without memorizing noise.")

Step 5: Comparison Table

# Comparison table
print("\n" + "="*60)
print("VARIANCE COMPARISON")
print("="*60)
print(f"{'Model':<20} {'Train R²':<12} {'Test R²':<12} {'Gap':<10} {'Issue'}")
print("-" * 60)
print(f"{'Deep Tree':<20} {train_r2:<12.3f} {test_r2:<12.3f} {train_r2-test_r2:<10.3f} High Variance")
print(f"{'Shallow Tree':<20} {train_r2_shallow:<12.3f} {test_r2_shallow:<12.3f} {train_r2_shallow-test_r2_shallow:<10.3f} Balanced")

print("\nKey Lesson:")
print("  A large gap between train and test performance = High Variance")
print("  The model is too sensitive to training data specifics.")
print("  Solution: Reduce complexity (shallower tree, pruning, regularization)")

This is a textbook case of high variance. The deep tree achieves near-perfect training accuracy because it can create a split for almost every training point. With enough depth, it essentially memorizes the training data. But memorization isn't learning - it captures random noise as if it were meaningful patterns.

When the deep tree encounters test data, it fails because the test data has different noise. The tree learned rules like "if x=3.241 and y=15.7, then predict 16.2" - a rule that's uselessly specific. It's like memorizing specific practice problems instead of understanding the underlying concepts. This is high variance - small changes in training data would produce very different trees.

The shallow tree (max_depth=3) trades some training accuracy for better generalization. It can't memorize individual points, so it's forced to find broader patterns. The 3-level tree might learn "if x < 5, predict low values; if x >= 5, predict high values" - a simpler rule that works on new data. The gap between training and test performance is much smaller, indicating good generalization.

The tree depth directly controls model complexity. Depth 1 = two predictions (high bias), depth 20 = millions of possible predictions (high variance). Depth 3-5 often hits the sweet spot, learning enough to capture real patterns without memorizing noise. This is the bias-variance tradeoff in action.

The Tradeoff

Here's the fundamental challenge: reducing bias typically increases variance, and reducing variance typically increases bias. The goal is to find the sweet spot that minimizes total error (bias² + variance + irreducible noise).

# Finding the sweet spot: varying model complexity
from sklearn.tree import DecisionTreeRegressor
import numpy as np

# Test different max_depth values
depths = [1, 2, 3, 5, 10, 20, None]
results = []

for depth in depths:
    model = DecisionTreeRegressor(max_depth=depth, random_state=42)
    model.fit(X_train, y_train)
    
    train_r2 = model.score(X_train, y_train)
    test_r2 = model.score(X_test, y_test)
    gap = train_r2 - test_r2
    
    results.append({
        "depth": depth if depth else "None",
        "train_r2": train_r2,
        "test_r2": test_r2,
        "gap": gap
    })
    
# Print results
print(f"{'Depth':>8} | {'Train R²':>10} | {'Test R²':>10} | {'Gap':>8}")
print("-" * 45)
for r in results:
    print(f"{str(r['depth']):>8} | {r['train_r2']:>10.3f} | {r['test_r2']:>10.3f} | {r['gap']:>8.3f}")

# depth=3 or depth=5 likely gives best test performance
# Too shallow = high bias, too deep = high variance

import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split

# Generate curved data
np.random.seed(42)
X = np.linspace(-3, 3, 100).reshape(-1, 1)
y = X.ravel()**2 + np.random.randn(100) * 0.5

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# High bias: linear model
linear = LinearRegression()
linear.fit(X_train, y_train)
print("Linear (high bias):")
print(f"  Train: {linear.score(X_train, y_train):.3f}")
print(f"  Test:  {linear.score(X_test, y_test):.3f}")

# High variance: degree 10 polynomial
poly10 = make_pipeline(PolynomialFeatures(10), LinearRegression())
poly10.fit(X_train, y_train)
print("\nPolynomial deg=10 (high variance):")
print(f"  Train: {poly10.score(X_train, y_train):.3f}")
print(f"  Test:  {poly10.score(X_test, y_test):.3f}")

from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split

cancer = load_breast_cancer()
X, y = cancer.data, cancer.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

best_depth = 1
best_test_score = 0

print(f"{'Depth':>6} | {'Train':>8} | {'Test':>8}")
print("-" * 30)

for depth in range(1, 21):
    model = DecisionTreeClassifier(max_depth=depth, random_state=42)
    model.fit(X_train, y_train)
    train_score = model.score(X_train, y_train)
    test_score = model.score(X_test, y_test)
    
    print(f"{depth:>6} | {train_score:>8.3f} | {test_score:>8.3f}")
    
    if test_score > best_test_score:
        best_test_score = test_score
        best_depth = depth

print(f"\nOptimal depth: {best_depth} with test score: {best_test_score:.3f}")

def diagnose_model(train_score, test_score, threshold=0.75, gap_threshold=0.1):
    """
    Diagnose bias/variance based on train and test scores.
    """
    gap = train_score - test_score
    
    if train_score < threshold and test_score < threshold:
        return "High Bias (Underfitting)"
    elif train_score > 0.9 and gap > gap_threshold:
        return "High Variance (Overfitting)"
    else:
        return "Good Fit"

# Test cases
print(diagnose_model(0.60, 0.58))  # High Bias
print(diagnose_model(0.99, 0.65))  # High Variance
print(diagnose_model(0.92, 0.89))  # Good Fit

The two most common problems in machine learning are overfitting (model too complex) and underfitting (model too simple). Learning to diagnose and fix these issues is a crucial skill that separates beginners from experienced practitioners.

What is Underfitting?

Underfitting occurs when your model is too simple to capture the underlying patterns in the data. It's like trying to fit a straight line to data that clearly follows a curve. The model performs poorly on both training and test data because it hasn't learned enough from the examples.

# Demonstrating Underfitting
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
import numpy as np

# Complex data: cubic relationship
np.random.seed(42)
X = np.linspace(-3, 3, 100).reshape(-1, 1)
y = X.ravel()**3 - 2*X.ravel() + np.random.randn(100) * 3

# Underfitting: linear model for cubic data
linear = LinearRegression()
linear.fit(X, y)
train_score = linear.score(X, y)
print(f"Linear model R²: {train_score:.3f}")  # ~0.85 - missing the curve

# Better: cubic polynomial
cubic = make_pipeline(PolynomialFeatures(3), LinearRegression())
cubic.fit(X, y)
cubic_score = cubic.score(X, y)
print(f"Cubic model R²: {cubic_score:.3f}")   # ~0.98 - captures the pattern

Causes of Underfitting

• Model too simple: Using linear regression for non-linear data
• Insufficient features: Missing important predictive variables
• Too much regularization: Over-penalizing model complexity
• Training stopped too early: Not enough epochs for neural networks

What is Overfitting?

Overfitting is the opposite problem - your model is too complex and has essentially memorized the training data, including its random noise. It performs amazingly on training data but fails miserably on new, unseen data. This is the more common and dangerous problem in practice.

# Demonstrating Overfitting
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Overfitting: unlimited depth tree
overfit_tree = DecisionTreeRegressor(random_state=42)  # No constraints
overfit_tree.fit(X_train, y_train)

train_score = overfit_tree.score(X_train, y_train)
test_score = overfit_tree.score(X_test, y_test)

print(f"Training R²: {train_score:.3f}")  # 1.000 - perfect!
print(f"Test R²: {test_score:.3f}")       # ~0.60 - terrible!
print(f"Gap: {train_score - test_score:.3f}")  # ~0.40 - overfitting!

# Better: constrained tree
good_tree = DecisionTreeRegressor(max_depth=4, random_state=42)
good_tree.fit(X_train, y_train)

print(f"\nConstrained tree:")
print(f"Training R²: {good_tree.score(X_train, y_train):.3f}")
print(f"Test R²: {good_tree.score(X_test, y_test):.3f}")

Causes of Overfitting

• Model too complex: Too many parameters relative to data size
• Insufficient training data: Not enough examples to learn from
• Training too long: Neural networks memorize after too many epochs
• No regularization: No penalty for model complexity
• Noisy data: Model learns the noise as if it were signal

Detecting Overfitting and Underfitting

The key diagnostic tool is comparing training and test performance. Think of it like a student - if they ace every practice problem but bomb the real exam, they memorized answers instead of learning concepts. If they struggle with both, they didn't study enough. Here's a systematic approach:

Step 1: Import Required Libraries

# Step 1: Import required libraries
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
import numpy as np

print("Setting up diagnostic framework...")

Step 2: Create Diagnostic Function

# Step 2: Create diagnostic function
def diagnose_fit(model, X_train, X_test, y_train, y_test):
    """Diagnose if model is underfitting, overfitting, or just right"""
    
    # Train the model
    model.fit(X_train, y_train)
    
    # Get scores on both sets
    train_score = model.score(X_train, y_train)
    test_score = model.score(X_test, y_test)
    gap = train_score - test_score
    
    # Display raw scores
    print("\nPerformance Metrics:")
    print(f"  Training score: {train_score:.2%}")
    print(f"  Test score: {test_score:.2%}")
    print(f"  Gap (Train-Test): {gap:.2%}")
    
    return train_score, test_score, gap

Step 3: Add Interpretation Logic

# Step 3: Add interpretation logic
def interpret_diagnosis(train_score, test_score, gap):
    """Interpret the scores and provide remedies"""
    
    print("\n" + "="*60)
    
    # Case 1: Both scores are low = UNDERFITTING
    if train_score < 0.70 and test_score < 0.70:
        print("DIAGNOSIS: UNDERFITTING (High Bias)")
        print("-" * 60)
        print("Problem: Model is too simple to capture patterns")
        print("\nRemedies:")
        print("  1. Use a more complex model (e.g., Random Forest vs Linear)")
        print("  2. Add more features or polynomial features")
        print("  3. Reduce regularization strength")
        print("  4. Train for more epochs (neural networks)")
    
    # Case 2: High training, low test, large gap = OVERFITTING
    elif train_score > 0.90 and gap > 0.15:
        print("DIAGNOSIS: OVERFITTING (High Variance)")
        print("-" * 60)
        print("Problem: Model memorized training data")
        print("\nRemedies:")
        print("  1. Add regularization (Ridge, Lasso, dropout)")
        print("  2. Simplify model (lower depth, fewer parameters)")
        print("  3. Get more training data")
        print("  4. Use cross-validation for hyperparameter tuning")
        print("  5. Apply early stopping")
    
    # Case 3: Good performance with small gap = GOOD FIT
    else:
        print("DIAGNOSIS: GOOD FIT ✓")
        print("-" * 60)
        print("Model generalizes well!")
        print("\nNext steps:")
        print("  1. Validate on additional holdout data")
        print("  2. Test edge cases and error analysis")
        print("  3. Consider deploying to production")
    
    print("="*60)

Step 4: Load Data and Prepare

# Step 4: Test with example scenarios
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier

# Load data and split
cancer = load_breast_cancer()
X, y = cancer.data, cancer.target
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

print(f"Dataset: Breast Cancer (569 samples, 30 features)")
print(f"Training: {len(X_train)} samples, Test: {len(X_test)} samples")

Step 5: Test Scenario 1 - Underfitting

print("SCENARIO 1: Underfitting Model")
print("="*60)
# Too simple model for this data
simple_model = LogisticRegression(max_iter=10)  # Not enough iterations
train_s, test_s, gap_s = diagnose_fit(simple_model, X_train, X_test, y_train, y_test)
interpret_diagnosis(train_s, test_s, gap_s)

Step 6: Test Scenario 2 - Overfitting

print("\n\nSCENARIO 2: Overfitting Model")
print("="*60)
# Too complex model
complex_model = DecisionTreeClassifier(random_state=42)  # Unlimited depth!
train_c, test_c, gap_c = diagnose_fit(complex_model, X_train, X_test, y_train, y_test)
interpret_diagnosis(train_c, test_c, gap_c)

Step 7: Test Scenario 3 - Good Fit

print("\n\nSCENARIO 3: Well-Tuned Model")
print("="*60)
# Just right
balanced_model = RandomForestClassifier(n_estimators=100, max_depth=10, random_state=42)
train_b, test_b, gap_b = diagnose_fit(balanced_model, X_train, X_test, y_train, y_test)
interpret_diagnosis(train_b, test_b, gap_b)

print("\n\nSUMMARY:")
print(f"Simple model gap: {gap_s:.2%} (underfitting)")
print(f"Complex model gap: {gap_c:.2%} (overfitting)")
print(f"Balanced model gap: {gap_b:.2%} (good fit)")

This diagnostic approach gives you actionable insights. When you see underfitting (both scores low), you know the model lacks capacity. When you see overfitting (large gap), you know it's too flexible. The beauty is this works for ANY supervised learning model - just plug it in and diagnose!

Techniques to Prevent Overfitting

There are several battle-tested techniques to combat overfitting:

Regularization is one of the most powerful tools to prevent overfitting. It works by penalizing model complexity during training - essentially telling the model "you can fit the data, but not TOO perfectly." Let's see this in action:

# Step 1: Create high-dimensional polynomial features
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression, Ridge
import numpy as np

np.random.seed(42)

# Simple data: y = 2x + noise
X = np.linspace(0, 10, 50).reshape(-1, 1)
y = 2 * X.ravel() + np.random.randn(50) * 2

# Create overly complex features (degree 15 = 15 polynomial terms!)
poly = PolynomialFeatures(degree=15)
X_poly = poly.fit_transform(X)

print(f"Original features: {X.shape[1]}")
print(f"After polynomial expansion: {X_poly.shape[1]} features")
print("This many features will easily overfit only 50 data points!")

We've intentionally created a recipe for disaster: 15 polynomial features from just 50 data points. A simple linear model will overfit badly. Let's split and train without regularization first:

# Step 2: Train without regularization
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X_poly, y, test_size=0.3, random_state=42
)

# Regular linear regression (no penalty for complexity)
unregularized = LinearRegression()
unregularized.fit(X_train, y_train)

train_r2 = unregularized.score(X_train, y_train)
test_r2 = unregularized.score(X_test, y_test)

print("\nWithout Regularization:")
print("="*50)
print(f"  Training R²: {train_r2:.3f}")
print(f"  Test R²: {test_r2:.3f}")
print(f"  Gap: {train_r2 - test_r2:.3f}")
print(f"\n  Max coefficient: {np.abs(unregularized.coef_).max():.2f}")
print(f"  Total weight magnitude: {np.abs(unregularized.coef_).sum():.2f}")
print("\n⚠ Problem: Very large coefficients indicate overfitting!")

The huge gap and enormous coefficients are red flags. The model is using those 15 features to twist and turn in crazy ways to hit every training point. Now let's apply Ridge regularization, which penalizes large coefficients:

# Step 3: Train with Ridge regularization
# Ridge adds penalty: Loss = MSE + alpha * sum(coefficients²)
# Larger alpha = stronger penalty = simpler model

regularized = Ridge(alpha=1.0)  # alpha controls penalty strength
regularized.fit(X_train, y_train)

train_r2_reg = regularized.score(X_train, y_train)
test_r2_reg = regularized.score(X_test, y_test)

print("\nWith Ridge Regularization (alpha=1.0):")
print("="*50)
print(f"  Training R²: {train_r2_reg:.3f}")
print(f"  Test R²: {test_r2_reg:.3f}")
print(f"  Gap: {train_r2_reg - test_r2_reg:.3f}")
print(f"\n  Max coefficient: {np.abs(regularized.coef_).max():.2f}")
print(f"  Total weight magnitude: {np.abs(regularized.coef_).sum():.2f}")
print("\n✓ Much smaller coefficients = simpler, more robust model!")

# Step 4: Compare side-by-side
print("\n" + "="*60)
print("REGULARIZATION COMPARISON")
print("="*60)
print(f"{'Metric':<30} {'No Regularization':<20} {'Ridge (alpha=1.0)'}")
print("-" * 60)
print(f"{'Training R²':<30} {train_r2:<20.3f} {train_r2_reg:.3f}")
print(f"{'Test R²':<30} {test_r2:<20.3f} {test_r2_reg:.3f}")
print(f"{'Gap':<30} {train_r2-test_r2:<20.3f} {train_r2_reg-test_r2_reg:.3f}")
print(f"{'Max |coefficient|':<30} {np.abs(unregularized.coef_).max():<20.2f} {np.abs(regularized.coef_).max():.2f}")

print("\nKey Insight:")
print("  Ridge sacrificed a bit of training performance but IMPROVED test performance.")
print("  Smaller coefficients = the model uses features more cautiously.")
print("  Result: Better generalization to unseen data!")

Notice Ridge achieved lower training R² (0.95 instead of 0.99) but HIGHER test R² (0.85 instead of 0.60). That's the magic of regularization - it prevents the model from going crazy trying to fit every tiny detail in the training data. The alpha parameter controls how strong this penalty is: alpha=0 means no penalty (regular linear regression), alpha=1000 means extremely strong penalty (model becomes almost constant).

Early stopping is another elegant solution, especially for iterative algorithms like gradient boosting or neural networks. The idea: train the model while monitoring validation performance, and stop when validation performance stops improving - even if training could continue. This prevents the model from memorizing training data during later iterations.

# Step 1: Setup for early stopping demonstration
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import train_test_split
import numpy as np

# Generate regression data
np.random.seed(42)
X = np.random.randn(200, 10)
y = X[:, 0] * 2 + X[:, 1] * -1 + np.random.randn(200) * 0.5

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=42
)

print("Data prepared for early stopping demo")
print(f"  Training samples: {len(X_train)}")
print(f"  Test samples: {len(X_test)}")

Gradient Boosting builds trees sequentially - each new tree tries to fix the mistakes of previous trees. Without early stopping, it might build 500 trees and start overfitting around tree 250. Let's train without early stopping first:

# Step 2: Train without early stopping
print("\nTraining WITHOUT early stopping:")
print("="*60)

gb_no_stop = GradientBoostingRegressor(
    n_estimators=500,  # Build 500 trees no matter what
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)

gb_no_stop.fit(X_train, y_train)

train_score_no = gb_no_stop.score(X_train, y_train)
test_score_no = gb_no_stop.score(X_test, y_test)

print(f"  Trees built: {gb_no_stop.n_estimators}")
print(f"  Training R²: {train_score_no:.3f}")
print(f"  Test R²: {test_score_no:.3f}")
print(f"  Gap: {train_score_no - test_score_no:.3f}")
print("\n  Issue: Built all 500 trees even if later ones hurt generalization")

Now let's use early stopping. We'll reserve 20% of training data as a validation set (via validation_fraction) and stop if validation score doesn't improve for 10 consecutive rounds:

# Step 3: Train with early stopping
print("\nTraining WITH early stopping:")
print("="*60)

gb_early = GradientBoostingRegressor(
    n_estimators=500,  # Maximum trees, but will likely stop earlier
    learning_rate=0.1,
    max_depth=3,
    validation_fraction=0.2,  # Use 20% of training data for validation
    n_iter_no_change=10,      # Stop if no improvement for 10 rounds
    tol=0.001,                # Minimum improvement threshold
    random_state=42
)

gb_early.fit(X_train, y_train)

train_score_early = gb_early.score(X_train, y_train)
test_score_early = gb_early.score(X_test, y_test)

print(f"  Trees built: {gb_early.n_estimators_} (stopped early!)")
print(f"  Training R²: {train_score_early:.3f}")
print(f"  Test R²: {test_score_early:.3f}")
print(f"  Gap: {train_score_early - test_score_early:.3f}")
print(f"\n  ✓ Stopped at tree {gb_early.n_estimators_} instead of 500")
print("  This prevented overfitting in later iterations!")

# Step 4: Compare results
print("\n" + "="*60)
print("EARLY STOPPING COMPARISON")
print("="*60)
print(f"{'Metric':<25} {'Without Early Stop':<22} {'With Early Stop'}")
print("-" * 60)
print(f"{'Trees built':<25} {gb_no_stop.n_estimators:<22} {gb_early.n_estimators_}")
print(f"{'Training R²':<25} {train_score_no:<22.3f} {train_score_early:.3f}")
print(f"{'Test R²':<25} {test_score_no:<22.3f} {test_score_early:.3f}")
print(f"{'Gap':<25} {train_score_no-test_score_no:<22.3f} {train_score_early-test_score_early:.3f}")

trees_saved = gb_no_stop.n_estimators - gb_early.n_estimators_
print(f"\nEfficiency gain: Saved {trees_saved} unnecessary trees!")
print(f"Improved test score by: {test_score_early - test_score_no:.3f}")
print("\nKey lesson: More training isn't always better. Stop when validation peaks!")

Early stopping is beautiful because it's automatic - you don't need to guess the right number of trees or epochs. The algorithm watches validation performance and stops when it sees diminishing returns. This technique is essential for deep learning (where training can take days) and ensemble methods like gradient boosting and XGBoost.

Visual Summary

from sklearn.datasets import load_iris
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split

iris = load_iris()
X, y = iris.data, iris.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Unlimited depth - prone to overfitting
tree = DecisionTreeClassifier(random_state=42)
tree.fit(X_train, y_train)

print(f"Training accuracy: {tree.score(X_train, y_train):.2%}")  # 100%
print(f"Test accuracy: {tree.score(X_test, y_test):.2%}")
print(f"Gap: {tree.score(X_train, y_train) - tree.score(X_test, y_test):.2%}")

import numpy as np
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import train_test_split

# Generate data
np.random.seed(42)
X = np.linspace(0, 1, 50).reshape(-1, 1)
y = np.sin(2 * np.pi * X).ravel() + np.random.randn(50) * 0.1

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create polynomial features
poly = PolynomialFeatures(degree=10)
X_train_poly = poly.fit_transform(X_train)
X_test_poly = poly.transform(X_test)

# Overfitting: no regularization
lr = LinearRegression()
lr.fit(X_train_poly, y_train)
print("LinearRegression (no regularization):")
print(f"  Train R²: {lr.score(X_train_poly, y_train):.3f}")
print(f"  Test R²: {lr.score(X_test_poly, y_test):.3f}")

# Fixed: with regularization
ridge = Ridge(alpha=0.01)
ridge.fit(X_train_poly, y_train)
print("\nRidge (with regularization):")
print(f"  Train R²: {ridge.score(X_train_poly, y_train):.3f}")
print(f"  Test R²: {ridge.score(X_test_poly, y_test):.3f}")

from sklearn.datasets import load_digits
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import learning_curve
import numpy as np

digits = load_digits()
X, y = digits.data, digits.target

# Calculate learning curve
train_sizes, train_scores, test_scores = learning_curve(
    DecisionTreeClassifier(random_state=42),
    X, y,
    train_sizes=np.linspace(0.1, 1.0, 10),
    cv=5,
    n_jobs=-1
)

# Print results
print(f"{'Train Size':>12} | {'Train Mean':>12} | {'Test Mean':>12} | {'Gap':>8}")
print("-" * 50)
for size, train, test in zip(train_sizes, train_scores.mean(axis=1), test_scores.mean(axis=1)):
    print(f"{size:>12} | {train:>12.3f} | {test:>12.3f} | {train-test:>8.3f}")

# Gap decreases as training size increases!

from sklearn.datasets import load_diabetes
from sklearn.linear_model import Ridge
from sklearn.model_selection import train_test_split

diabetes = load_diabetes()
X, y = diabetes.data, diabetes.target

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

alphas = [0.001, 0.01, 0.1, 1, 10, 100]
best_alpha = 0.001
best_test_score = 0

print(f"{'Alpha':>8} | {'Train R²':>10} | {'Test R²':>10}")
print("-" * 35)

for alpha in alphas:
    model = Ridge(alpha=alpha)
    model.fit(X_train, y_train)
    train_score = model.score(X_train, y_train)
    test_score = model.score(X_test, y_test)
    print(f"{alpha:>8} | {train_score:>10.4f} | {test_score:>10.4f}")
    
    if test_score > best_test_score:
        best_test_score = test_score
        best_alpha = alpha

print(f"\nBest alpha: {best_alpha} with test R²: {best_test_score:.4f}")

Experiment with model complexity to see how it affects bias and variance. Adjust the slider to change model complexity and observe how training and test scores change.