The Machine Learning Workflow

Before writing any code, you must clearly understand the problem. A poorly defined problem leads to wasted effort and failed projects.

# Problem Statement Template
problem = {
    "objective": "Predict customer churn within 30 days",
    "type": "Binary Classification",
    "target_variable": "churned (0 or 1)",
    "success_metric": "F1-score >= 0.85",
    "constraints": {
        "latency": "< 100ms per prediction",
        "interpretability": "Must explain top 3 factors",
        "refresh": "Retrain weekly"
    }
}

Data is the fuel for ML. The quality and quantity of your data directly impacts model performance. Garbage in, garbage out!

Loading Data with pandas

import pandas as pd

# From CSV file
df = pd.read_csv('customers.csv')

# From Excel file
df = pd.read_excel('sales_data.xlsx', sheet_name='2024')

# From SQL database
import sqlite3
conn = sqlite3.connect('database.db')
df = pd.read_sql('SELECT * FROM customers', conn)

# From API (JSON)
import requests
response = requests.get('https://api.example.com/data')
df = pd.DataFrame(response.json())

# Quick data overview
print(f"Shape: {df.shape}")  # (rows, columns)
print(f"Columns: {df.columns.tolist()}")
df.head()  # First 5 rows

This is where you spend 60-80% of your time! Real-world data is messy. You need to clean, transform, and prepare it before feeding it to ML algorithms.

Common Data Issues

# Check for missing values
print(df.isnull().sum())

# Drop rows with missing values
df_clean = df.dropna()

# Fill with mean/median
df['age'].fillna(df['age'].median(), inplace=True)

# Fill with most frequent (mode)
df['category'].fillna(df['category'].mode()[0], inplace=True)

# Forward fill (time series)
df['price'].fillna(method='ffill', inplace=True)

# Remove duplicates
df = df.drop_duplicates()

# Find duplicates
duplicates = df[df.duplicated()]
print(f"Found {len(duplicates)} duplicates")

# Detect outliers using IQR
Q1 = df['price'].quantile(0.25)
Q3 = df['price'].quantile(0.75)
IQR = Q3 - Q1
outliers = df[(df['price'] < Q1 - 1.5*IQR) | 
              (df['price'] > Q3 + 1.5*IQR)]

# Remove outliers
df = df[~df.index.isin(outliers.index)]

Train-Test Split

from sklearn.model_selection import train_test_split

# Features (X) and target (y)
X = df.drop('target', axis=1)
y = df['target']

# Split: 80% train, 20% test
X_train, X_test, y_train, y_test = train_test_split(
    X, y, 
    test_size=0.2,      # 20% for testing
    random_state=42,    # Reproducibility
    stratify=y          # Keep class proportions (for classification)
)

print(f"Training set: {X_train.shape[0]} samples")
print(f"Test set: {X_test.shape[0]} samples")

Feature engineering is the art of creating and selecting the right input variables. Good features can make a simple model outperform a complex one!

from sklearn.preprocessing import StandardScaler, MinMaxScaler

# StandardScaler: mean=0, std=1
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)  # Use same scaler!

# MinMaxScaler: range [0, 1]
minmax = MinMaxScaler()
X_train_norm = minmax.fit_transform(X_train)
X_test_norm = minmax.transform(X_test)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
import pandas as pd

# Label Encoding (ordinal categories)
le = LabelEncoder()
df['size_encoded'] = le.fit_transform(df['size'])
# S=0, M=1, L=2

# One-Hot Encoding (nominal categories)
df_encoded = pd.get_dummies(df, columns=['color'])
# Creates: color_red, color_blue, color_green

# Or using sklearn
ohe = OneHotEncoder(sparse=False)
encoded = ohe.fit_transform(df[['color']])

Creating New Features

Feature engineering is where data science becomes an art! Creating smart new features from existing data can dramatically improve your model's performance — sometimes more than changing the algorithm itself.

# Feature creation examples
df['age_group'] = pd.cut(df['age'], bins=[0, 18, 35, 50, 100], 
                         labels=['teen', 'young', 'middle', 'senior'])

# Date features
df['signup_date'] = pd.to_datetime(df['signup_date'])
df['signup_year'] = df['signup_date'].dt.year
df['signup_month'] = df['signup_date'].dt.month
df['signup_dayofweek'] = df['signup_date'].dt.dayofweek
df['days_since_signup'] = (pd.Timestamp.now() - df['signup_date']).dt.days

# Interaction features
df['price_per_sqft'] = df['price'] / df['sqft']
df['total_spend'] = df['quantity'] * df['unit_price']

# Log transform for skewed data
import numpy as np
df['log_income'] = np.log1p(df['income'])  # log(1+x) handles zeros

Now comes the exciting part - training your ML model! Start simple, then iterate to more complex models if needed.

The Basic Training Pattern

from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

# Step 1: Choose a model
model = RandomForestClassifier(n_estimators=100, random_state=42)

# Step 2: Train (fit) on training data
model.fit(X_train, y_train)

# Step 3: Make predictions
y_pred = model.predict(X_test)

# Step 4: Get probability scores (if needed)
y_proba = model.predict_proba(X_test)[:, 1]  # Probability of class 1

Hyperparameter Tuning

Hyperparameters are settings you choose before training (unlike model parameters learned during training). Finding the best combination can significantly boost your model's performance!

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV

# Define parameter grid
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [None, 10, 20, 30],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}

# Grid Search (tries all combinations)
grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,              # 5-fold cross-validation
    scoring='f1',      # Optimize for F1-score
    n_jobs=-1          # Use all CPU cores
)

grid_search.fit(X_train, y_train)

# Best parameters and score
print(f"Best params: {grid_search.best_params_}")
print(f"Best CV score: {grid_search.best_score_:.4f}")

# Use best model
best_model = grid_search.best_estimator_

Cross-Validation

from sklearn.model_selection import cross_val_score

# 5-fold cross-validation
scores = cross_val_score(
    model, X_train, y_train, 
    cv=5, 
    scoring='accuracy'
)

print(f"Scores: {scores}")
print(f"Mean: {scores.mean():.4f}")
print(f"Std: {scores.std():.4f}")

# Interpretation:
# Mean = expected performance
# Low std = stable model (good!)
# High std = model overfitting (bad!)

# Other CV strategies
from sklearn.model_selection import (
    StratifiedKFold,  # Preserves class balance
    LeaveOneOut,      # N folds for N samples
    TimeSeriesSplit   # For time series data
)

# Stratified K-Fold (recommended for classification)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=skf)

# Time Series Split (respects temporal order)
tscv = TimeSeriesSplit(n_splits=5)
scores = cross_val_score(model, X, y, cv=tscv)

How do you know if your model is good? Evaluation metrics tell you how well your model performs on unseen data.

from sklearn.metrics import (
    accuracy_score, precision_score, 
    recall_score, f1_score, 
    confusion_matrix, classification_report
)

# Basic metrics
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)

print(f"Accuracy: {accuracy:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall: {recall:.4f}")
print(f"F1-Score: {f1:.4f}")

# Detailed report
print(classification_report(y_test, y_pred))

from sklearn.metrics import (
    mean_squared_error, mean_absolute_error,
    r2_score, root_mean_squared_error
)

# Calculate metrics
mse = mean_squared_error(y_test, y_pred)
rmse = root_mean_squared_error(y_test, y_pred)
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print(f"MSE: {mse:.4f}")
print(f"RMSE: {rmse:.4f}")
print(f"MAE: {mae:.4f}")
print(f"R-squared: {r2:.4f}")

Understanding the Confusion Matrix

from sklearn.metrics import confusion_matrix
import seaborn as sns
import matplotlib.pyplot as plt

# Create confusion matrix
cm = confusion_matrix(y_test, y_pred)

# Visualize
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Predicted No', 'Predicted Yes'],
            yticklabels=['Actual No', 'Actual Yes'])
plt.title('Confusion Matrix')
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()

A model that stays on your laptop creates zero value. Deployment puts your model into production where it can make real predictions!

import joblib
import pickle

# Save with joblib (recommended)
joblib.dump(model, 'model.joblib')

# Load model
loaded_model = joblib.load('model.joblib')

# Make prediction
prediction = loaded_model.predict(new_data)

# Flask API example
from flask import Flask, request
import joblib

app = Flask(__name__)
model = joblib.load('model.joblib')

@app.route('/predict', methods=['POST'])
def predict():
    data = request.json
    prediction = model.predict([data['features']])
    return {'prediction': int(prediction[0])}