6c:T898,import lightgbm as lgb from sklearn.metrics import roc_auc_score, precision_score, recall_score import numpy as np def multi_metric_eval(y_pred, dtrain): """Custom evaluator for LightGBM with multiple metrics.""" y_true = dtrain.get_label() proba = 1.0 / (1.0 + np.exp(-y_pred)) # sigmoid auc = roc_auc_score(y_true, proba) preds = (proba > 0.5).astype(int) precision = precision_score(y_true, preds, zero_division=0) recall = recall_score(y_true, preds, zero_division=0) # Return (metric_name, value, is_higher_better) # For multiple metrics, return list return [ ('custom_auc', auc, True), ('custom_precision', precision, True), ('custom_recall', recall, True) ] # LightGBM training with comprehensive monitoring params = { 'objective': 'binary', 'metric': ['binary_logloss', 'auc'], 'learning_rate': 0.05, 'num_leaves': 31, 'max_depth': -1, 'subsample': 0.8, 'colsample_bytree': 0.8, 'verbosity': 1, } train_data = lgb.Dataset(X_train, label=y_train) val_data = lgb.Dataset(X_val, label=y_val, reference=train_data) # Callback for detailed logging def logging_callback(env): """Custom callback for enhanced monitoring.""" if env.iteration % 10 == 0: results = env.evaluation_result_list msg = f"Iter {env.iteration}: " msg += " | ".join([f"{name}: {value:.4f}" for name, _, value, _ in results]) print(msg) return None evals_result = {} model = lgb.train( params, train_data, num_boost_round=1000, valid_sets=[train_data, val_data], valid_names=['train', 'validation'], feval=multi_metric_eval, callbacks=[ lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=10), lgb.record_evaluation(evals_result), logging_callback ] ) # Analyze training curves val_logloss = evals_result['validation']['binary_logloss'] val_auc = evals_result['validation']['auc'] best_iter = model.best_iteration print(f" Best iteration: {best_iter}") print(f"Best validation logloss: {min(val_logloss):.4f}") print(f"Best validation AUC: {max(val_auc):.4f}")6d:T6d5,import xgboost as xgb from sklearn.metrics import roc_auc_score, precision_score, recall_score import numpy as np # Custom evaluation function for multiple metrics def multi_metric_eval(predt, dtrain): """Custom evaluator tracking multiple business metrics.""" labels = dtrain.get_label() proba = 1.0 / (1.0 + np.exp(-predt)) # sigmoid for binary # Calculate multiple metrics auc = roc_auc_score(labels, proba) preds = (proba > 0.5).astype(int) precision = precision_score(labels, preds, zero_division=0) recall = recall_score(labels, preds, zero_division=0) # Return list of (metric_name, value) tuples return [ ('auc', auc), ('precision', precision), ('recall', recall) ] # Training with detailed monitoring params = { 'objective': 'binary:logistic', 'eval_metric': ['logloss', 'error', 'auc'], 'learning_rate': 0.05, 'max_depth': 6, 'subsample': 0.8, 'colsample_bytree': 0.8, } dtrain = xgb.DMatrix(X_train, label=y_train) dval = xgb.DMatrix(X_val, label=y_val) # Enable verbose evaluation evals = [(dtrain, 'train'), (dval, 'validation')] evals_result = {} # Dictionary to store evaluation results model = xgb.train( params, dtrain, num_boost_round=1000, evals=evals, evals_result=evals_result, # Stores all metrics per iteration custom_metric=multi_metric_eval, early_stopping_rounds=50, verbose_eval=10 # Print every 10 rounds ) # Access training history train_logloss = evals_result['train']['logloss'] val_logloss = evals_result['validation']['logloss'] val_auc = evals_result['validation']['auc'] print(f"Best iteration: {model.best_iteration}") print(f"Best validation AUC: {max(val_auc):.4f}")6e:T7a0,from catboost import CatBoostClassifier, Pool from sklearn.metrics import roc_auc_score, precision_score, recall_score import numpy as np class MultiMetricCallback: """Custom callback for tracking multiple metrics in CatBoost.""" def __init__(self): self.metrics_history = { 'auc': [], 'precision': [], 'recall': [] } def after_iteration(self, info): """Called after each boosting iteration.""" # info contains: iteration, validation metrics, etc. iteration = info.iteration if iteration % 10 == 0: print(f"Iteration {iteration}: " f"Logloss = {info.metrics['validation']['Logloss'][-1]:.4f}") return True # Return False to stop training # CatBoost with comprehensive monitoring model = CatBoostClassifier( iterations=1000, learning_rate=0.05, depth=6, loss_function='Logloss', eval_metric='AUC', custom_metric=['Precision', 'Recall', 'F1', 'Logloss'], early_stopping_rounds=50, verbose=10, # Print every 10 iterations random_seed=42 ) train_pool = Pool(X_train, y_train, cat_features=cat_cols) val_pool = Pool(X_val, y_val, cat_features=cat_cols) # Train with detailed monitoring model.fit( train_pool, eval_set=val_pool, use_best_model=True, plot=True # Enable interactive plotting (Jupyter) ) # Access comprehensive training history evals_result = model.get_evals_result() val_metrics = evals_result['validation'] print(f" Training Summary:") print(f"Best iteration: {model.get_best_iteration()}") print(f"Final validation Logloss: {val_metrics['Logloss'][-1]:.4f}") print(f"Final validation AUC: {val_metrics['AUC'][-1]:.4f}") print(f"Final validation Precision: {val_metrics['Precision'][-1]:.4f}") print(f"Final validation Recall: {val_metrics['Recall'][-1]:.4f}") print(f"Final validation F1: {val_metrics['F1'][-1]:.4f}")6f:T1252,import wandb import lightgbm as lgb import numpy as np from sklearn.metrics import roc_auc_score, precision_recall_curve # Initialize W&B run wandb.init( project="gradient-boosting-production", name="lgb_classification_v1", config={ "model_type": "lightgbm", "task": "binary_classification", } ) class WandbCallback: """LightGBM callback for Weights & Biases integration.""" def __init__(self, log_every_n=10, track_feature_importance=True): self.log_every_n = log_every_n self.track_feature_importance = track_feature_importance self.iteration_times = [] self.last_time = None def __call__(self, env): """Called after each LightGBM iteration.""" import time # Track iteration timing current_time = time.time() if self.last_time is not None: self.iteration_times.append(current_time - self.last_time) self.last_time = current_time if env.iteration % self.log_every_n != 0: return # Log all evaluation metrics metrics = {} for name, _, value, is_higher_better in env.evaluation_result_list: metrics[name] = value # Calculate derived metrics train_metrics = [m for m in metrics if 'train' in m.lower()] val_metrics = [m for m in metrics if 'valid' in m.lower()] if train_metrics and val_metrics: # Calculate generalization gap train_key = [k for k in metrics.keys() if 'train' in k and 'logloss' in k.lower()] val_key = [k for k in metrics.keys() if 'valid' in k and 'logloss' in k.lower()] if train_key and val_key: gap = metrics[val_key[0]] - metrics[train_key[0]] metrics['generalization_gap'] = gap # Log to W&B wandb.log(metrics, step=env.iteration) # Log iteration speed if len(self.iteration_times) >= self.log_every_n: avg_time = np.mean(self.iteration_times[-self.log_every_n:]) wandb.log({'seconds_per_iteration': avg_time}, step=env.iteration) def train_with_wandb_monitoring(X_train, y_train, X_val, y_val, params, cat_features=None): """Complete LightGBM training with W&B monitoring.""" # Log hyperparameters wandb.config.update(params) wandb.config.update({ 'n_train_samples': len(X_train), 'n_val_samples': len(X_val), 'n_features': X_train.shape[1], 'n_cat_features': len(cat_features) if cat_features else 0 }) train_data = lgb.Dataset( X_train, label=y_train, categorical_feature=cat_features ) val_data = lgb.Dataset( X_val, label=y_val, reference=train_data, categorical_feature=cat_features ) # Train with callbacks model = lgb.train( params, train_data, num_boost_round=params.get('num_boost_round', 1000), valid_sets=[train_data, val_data], valid_names=['train', 'validation'], callbacks=[ lgb.early_stopping(stopping_rounds=50), WandbCallback(log_every_n=5) ] ) # Log final model metrics wandb.summary['best_iteration'] = model.best_iteration wandb.summary['best_score'] = model.best_score # Log feature importance as bar chart importance_df = pd.DataFrame({ 'feature': model.feature_name(), 'importance': model.feature_importance(importance_type='gain') }).sort_values('importance', ascending=False) wandb.log({ 'feature_importance': wandb.plot.bar( wandb.Table(dataframe=importance_df.head(20)), 'feature', 'importance', title='Top 20 Feature Importances' ) }) # Log precision-recall curve on validation set val_proba = model.predict(X_val) precision, recall, thresholds = precision_recall_curve(y_val, val_proba) wandb.log({ 'pr_curve': wandb.plot.pr_curve( y_val, np.column_stack([1-val_proba, val_proba]), labels=['Negative', 'Positive'] ) }) # Save model artifact model.save_model('model.txt') wandb.save('model.txt') return model # Example usage params = { 'objective': 'binary', 'metric': ['binary_logloss', 'auc'], 'learning_rate': 0.05, 'num_leaves': 31, 'subsample': 0.8, 'num_boost_round': 1000 } model = train_with_wandb_monitoring( X_train, y_train, X_val, y_val, params, cat_features=['category', 'region'] ) wandb.finish()70:Tff3,import mlflow import mlflow.xgboost import xgboost as xgb import numpy as np from datetime import datetime # Configure MLflow tracking mlflow.set_tracking_uri("http://mlflow-server:5000") mlflow.set_experiment("gradient_boosting_classification") class MLflowCallback(xgb.callback.TrainingCallback): """Custom XGBoost callback for MLflow integration.""" def __init__(self, log_every_n=10): self.log_every_n = log_every_n self.best_val_metric = float('inf') def after_iteration(self, model, epoch, evals_log): """Log metrics after each iteration.""" for data_name, metric_dict in evals_log.items(): for metric_name, metric_values in metric_dict.items(): value = metric_values[-1] if epoch % self.log_every_n == 0: mlflow.log_metric( f"{data_name}_{metric_name}", value, step=epoch ) # Track best validation metric for alerts val_logloss = evals_log.get('validation', {}).get('logloss', []) if val_logloss: current_val = val_logloss[-1] if current_val < self.best_val_metric: self.best_val_metric = current_val mlflow.log_metric("best_val_logloss", current_val, step=epoch) # Detect overfitting: train-val gap analysis train_logloss = evals_log.get('train', {}).get('logloss', []) if train_logloss and val_logloss: gap = val_logloss[-1] - train_logloss[-1] mlflow.log_metric("train_val_gap", gap, step=epoch) # Alert if gap is growing rapidly if gap > 0.1: # Threshold for significant overfitting mlflow.set_tag("overfitting_warning", f"Epoch {epoch}") return False # Continue training def train_with_mlflow_monitoring(X_train, y_train, X_val, y_val, params): """Complete training pipeline with MLflow monitoring.""" with mlflow.start_run(run_name=f"xgb_{datetime.now().strftime('%Y%m%d_%H%M%S')}"): # Log hyperparameters mlflow.log_params(params) mlflow.log_param("train_samples", len(X_train)) mlflow.log_param("val_samples", len(X_val)) mlflow.log_param("n_features", X_train.shape[1]) # Log dataset info mlflow.set_tag("model_type", "xgboost") mlflow.set_tag("task", "binary_classification") dtrain = xgb.DMatrix(X_train, label=y_train) dval = xgb.DMatrix(X_val, label=y_val) evals = [(dtrain, 'train'), (dval, 'validation')] # Train with monitoring callback model = xgb.train( params, dtrain, num_boost_round=params.get('num_boost_round', 1000), evals=evals, early_stopping_rounds=50, callbacks=[MLflowCallback(log_every_n=5)], verbose_eval=False ) # Log final metrics mlflow.log_metric("final_best_iteration", model.best_iteration) mlflow.log_metric("final_best_score", model.best_score) # Log feature importance importance = model.get_score(importance_type='gain') for feature, score in importance.items(): mlflow.log_metric(f"importance_{feature}", score) # Log model artifact mlflow.xgboost.log_model(model, "model") # Log training artifacts artifacts_path = f"./artifacts/{mlflow.active_run().info.run_id}" model.save_model(f"{artifacts_path}/model.json") mlflow.log_artifact(f"{artifacts_path}/model.json") return model # Example usage params = { 'objective': 'binary:logistic', 'eval_metric': ['logloss', 'auc'], 'learning_rate': 0.05, 'max_depth': 6, 'subsample': 0.8, 'num_boost_round': 1000 } model = train_with_mlflow_monitoring(X_train, y_train, X_val, y_val, params)71:T1fd1,import numpy as np from dataclasses import dataclass from typing import List, Tuple, Optional from enum import Enum class TrainingPathology(Enum): HEALTHY = "healthy" OVERFITTING = "overfitting" UNDERFITTING = "underfitting" SLOW_CONVERGENCE = "slow_convergence" OSCILLATING = "oscillating" CATASTROPHIC_SPIKE = "catastrophic_spike" @dataclass class TrainingDiagnostics: """Comprehensive training diagnostics.""" pathology: TrainingPathology severity: str # 'mild', 'moderate', 'severe' current_gap: float gap_trend: str # 'stable', 'increasing', 'decreasing' recommendation: str class TrainingMonitor: """ Real-time training diagnostics for gradient boosting. Analyzes training curves to detect pathologies and provide actionable recommendations for remediation. """ def __init__( self, window_size: int = 50, overfitting_threshold: float = 0.1, oscillation_threshold: float = 0.05, convergence_threshold: float = 0.001 ): self.window_size = window_size self.overfitting_threshold = overfitting_threshold self.oscillation_threshold = oscillation_threshold self.convergence_threshold = convergence_threshold self.train_losses: List[float] = [] self.val_losses: List[float] = [] self.gaps: List[float] = [] def update(self, train_loss: float, val_loss: float) -> TrainingDiagnostics: """Update with new iteration metrics and return diagnostics.""" self.train_losses.append(train_loss) self.val_losses.append(val_loss) self.gaps.append(val_loss - train_loss) return self._diagnose() def _diagnose(self) -> TrainingDiagnostics: """Analyze training history and diagnose pathologies.""" if len(self.val_losses) < self.window_size: return TrainingDiagnostics( pathology=TrainingPathology.HEALTHY, severity="n/a", current_gap=self.gaps[-1] if self.gaps else 0, gap_trend="insufficient_data", recommendation="Continue training; insufficient data for diagnosis" ) # Extract recent windows recent_train = np.array(self.train_losses[-self.window_size:]) recent_val = np.array(self.val_losses[-self.window_size:]) recent_gaps = np.array(self.gaps[-self.window_size:]) # Calculate trends using linear regression x = np.arange(self.window_size) train_trend = self._linear_trend(x, recent_train) val_trend = self._linear_trend(x, recent_val) gap_trend = self._linear_trend(x, recent_gaps) # Calculate variability val_std = np.std(recent_val) val_mean = np.mean(recent_val) coefficient_of_variation = val_std / val_mean if val_mean != 0 else 0 # Current state current_gap = self.gaps[-1] # Check for catastrophic spike (sudden increase in validation loss) if len(self.val_losses) > self.window_size: recent_increase = recent_val[-1] - recent_val[0] if recent_increase > 3 * val_std: return TrainingDiagnostics( pathology=TrainingPathology.CATASTROPHIC_SPIKE, severity="severe", current_gap=current_gap, gap_trend="spike", recommendation="STOP TRAINING. Check for data corruption, numerical " "instability, or extreme outliers. Reduce learning rate." ) # Check for oscillating loss if coefficient_of_variation > self.oscillation_threshold: return TrainingDiagnostics( pathology=TrainingPathology.OSCILLATING, severity="moderate" if coefficient_of_variation < 0.1 else "severe", current_gap=current_gap, gap_trend="oscillating", recommendation="Reduce learning rate by 50%. Consider larger validation " "set or data cleaning for label noise." ) # Check for overfitting if train_trend < 0 and val_trend > 0 and gap_trend > 0: severity = "mild" if current_gap < 0.05 else "moderate" if current_gap < 0.1 else "severe" return TrainingDiagnostics( pathology=TrainingPathology.OVERFITTING, severity=severity, current_gap=current_gap, gap_trend="increasing", recommendation=f"Enable early stopping. Current optimal iteration " f"is approximately {self._find_best_iteration()} rounds ago. " f"Consider: reduce max_depth, increase reg_lambda, " f"decrease learning_rate, enable subsampling." ) # Check for underfitting if abs(train_trend) < self.convergence_threshold and abs(val_trend) < self.convergence_threshold: if recent_val.mean() > 0.5: # Assuming normalized loss return TrainingDiagnostics( pathology=TrainingPathology.UNDERFITTING, severity="moderate", current_gap=current_gap, gap_trend="stable", recommendation="Model has insufficient capacity. Increase max_depth, " "num_leaves, or num_boost_round. Consider higher " "learning rate or more complex features." ) # Check for slow convergence if train_trend < 0 and abs(train_trend) < self.convergence_threshold / 10: return TrainingDiagnostics( pathology=TrainingPathology.SLOW_CONVERGENCE, severity="mild", current_gap=current_gap, gap_trend="stable" if abs(gap_trend) < 0.01 else "decreasing", recommendation="Training is progressing but slowly. Consider " "increasing learning rate by 2-3x or using " "learning rate warmup." ) # Healthy training gap_status = "stable" if abs(gap_trend) < 0.01 else "increasing" if gap_trend > 0 else "decreasing" return TrainingDiagnostics( pathology=TrainingPathology.HEALTHY, severity="n/a", current_gap=current_gap, gap_trend=gap_status, recommendation="Training is progressing healthily. Continue monitoring." ) def _linear_trend(self, x: np.ndarray, y: np.ndarray) -> float: """Calculate linear trend (slope) using least squares.""" n = len(x) slope = (n * np.sum(x * y) - np.sum(x) * np.sum(y)) / (n * np.sum(x**2) - np.sum(x)**2) return slope def _find_best_iteration(self) -> int: """Find iteration with best validation loss.""" if not self.val_losses: return 0 return len(self.val_losses) - 1 - np.argmin(self.val_losses[::-1]) # Usage example monitor = TrainingMonitor(window_size=50) # During training loop for iteration in range(1000): train_loss = evals_result['train']['logloss'][iteration] val_loss = evals_result['validation']['logloss'][iteration] diagnostics = monitor.update(train_loss, val_loss) if iteration % 50 == 0: print(f"Iteration {iteration}: {diagnostics.pathology.value}") print(f" Gap: {diagnostics.current_gap:.4f} ({diagnostics.gap_trend})") print(f" Recommendation: {diagnostics.recommendation}") # Automatic intervention if diagnostics.pathology == TrainingPathology.OVERFITTING: if diagnostics.severity == 'severe': print("Severe overfitting detected! Stopping training.") break72:T22e8,import matplotlib.pyplot as plt import numpy as np from matplotlib.animation import FuncAnimation from typing import Dict, List import pandas as pd class TrainingVisualizer: """ Comprehensive training visualization for gradient boosting. Provides real-time and post-hoc visualization of training dynamics, including loss curves, generalization gaps, and feature importance evolution. """ def __init__(self, figsize=(16, 10)): self.figsize = figsize def plot_training_curves( self, evals_result: Dict, metric: str = 'logloss', best_iteration: int = None, title: str = 'Training Curves' ): """ Plot training and validation curves with annotated diagnostics. Args: evals_result: Dictionary with 'train' and 'validation' keys metric: Name of metric to plot best_iteration: Mark the best iteration title: Plot title """ fig, axes = plt.subplots(2, 2, figsize=self.figsize) train_metric = evals_result['train'][metric] val_metric = evals_result['validation'][metric] iterations = np.arange(len(train_metric)) # === Panel 1: Raw Training Curves === ax1 = axes[0, 0] ax1.plot(iterations, train_metric, label='Training', color='#2196F3', linewidth=1.5) ax1.plot(iterations, val_metric, label='Validation', color='#FF5722', linewidth=1.5) if best_iteration is not None: ax1.axvline(x=best_iteration, color='green', linestyle='--', linewidth=2, label=f'Best Iter ({best_iteration})') ax1.scatter([best_iteration], [val_metric[best_iteration]], color='green', s=100, zorder=5) ax1.set_xlabel('Boosting Iterations') ax1.set_ylabel(metric.upper()) ax1.set_title(f'{title} - Loss Curves') ax1.legend(loc='upper right') ax1.grid(True, alpha=0.3) # === Panel 2: Generalization Gap === ax2 = axes[0, 1] gap = np.array(val_metric) - np.array(train_metric) # Color by severity colors = ['#4CAF50' if g < 0.05 else '#FF9800' if g < 0.1 else '#F44336' for g in gap] ax2.scatter(iterations, gap, c=colors, s=5, alpha=0.7) ax2.axhline(y=0, color='gray', linestyle='-', linewidth=0.5) ax2.axhline(y=0.05, color='#FF9800', linestyle='--', linewidth=1, label='Mild Overfitting Threshold') ax2.axhline(y=0.1, color='#F44336', linestyle='--', linewidth=1, label='Severe Overfitting Threshold') # Add trend line z = np.polyfit(iterations, gap, 3) p = np.poly1d(z) ax2.plot(iterations, p(iterations), color='purple', linewidth=2, label='Trend') ax2.set_xlabel('Boosting Iterations') ax2.set_ylabel('Generalization Gap (Val - Train)') ax2.set_title('Generalization Gap Analysis') ax2.legend(loc='upper left') ax2.grid(True, alpha=0.3) # === Panel 3: Rate of Improvement === ax3 = axes[1, 0] window = 20 # Calculate rolling improvement rate val_diff = np.diff(val_metric) rolling_improvement = pd.Series(val_diff).rolling(window=window).mean() ax3.fill_between( range(1, len(val_diff) + 1), rolling_improvement, 0, where=(rolling_improvement < 0), color='#4CAF50', alpha=0.3, label='Improving' ) ax3.fill_between( range(1, len(val_diff) + 1), rolling_improvement, 0, where=(rolling_improvement >= 0), color='#F44336', alpha=0.3, label='Degrading' ) ax3.plot(range(1, len(val_diff) + 1), rolling_improvement, color='black', linewidth=1) ax3.axhline(y=0, color='gray', linestyle='-', linewidth=1) ax3.set_xlabel('Boosting Iterations') ax3.set_ylabel(f'Rolling {window}-iter Δ{metric.upper()}') ax3.set_title('Rate of Improvement (Rolling Average)') ax3.legend() ax3.grid(True, alpha=0.3) # === Panel 4: Log-scale View (Zoomed) === ax4 = axes[1, 1] ax4.semilogy(iterations, train_metric, label='Training', color='#2196F3', linewidth=1.5) ax4.semilogy(iterations, val_metric, label='Validation', color='#FF5722', linewidth=1.5) if best_iteration is not None: ax4.axvline(x=best_iteration, color='green', linestyle='--', linewidth=2, label=f'Best ({best_iteration})') ax4.set_xlabel('Boosting Iterations') ax4.set_ylabel(f'{metric.upper()} (log scale)') ax4.set_title('Loss Curves (Logarithmic Scale)') ax4.legend(loc='upper right') ax4.grid(True, alpha=0.3, which='both') plt.tight_layout() return fig def plot_feature_importance_evolution( self, importance_history: List[Dict[str, float]], top_k: int = 10, iterations: List[int] = None ): """ Visualize how feature importance evolves during training. Args: importance_history: List of {feature: importance} dicts per checkpoint top_k: Number of top features to display iterations: Corresponding iteration numbers """ if iterations is None: iterations = list(range(len(importance_history))) # Get top-k features by final importance final_importance = importance_history[-1] top_features = sorted( final_importance.keys(), key=lambda x: final_importance[x], reverse=True )[:top_k] fig, ax = plt.subplots(figsize=(12, 6)) for feature in top_features: values = [h.get(feature, 0) for h in importance_history] ax.plot(iterations, values, label=feature, linewidth=2) ax.set_xlabel('Boosting Iterations') ax.set_ylabel('Feature Importance (Gain)') ax.set_title('Feature Importance Evolution During Training') ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left') ax.grid(True, alpha=0.3) plt.tight_layout() return fig def plot_multi_metric_dashboard( self, evals_result: Dict, metrics: List[str], best_iteration: int = None ): """ Create a dashboard showing multiple metrics simultaneously. """ n_metrics = len(metrics) fig, axes = plt.subplots( (n_metrics + 1) // 2, 2, figsize=(14, 4 * ((n_metrics + 1) // 2)) ) axes = axes.flatten() for idx, metric in enumerate(metrics): ax = axes[idx] train_values = evals_result['train'].get(metric, []) val_values = evals_result['validation'].get(metric, []) if not train_values or not val_values: ax.text(0.5, 0.5, f'Metric "{metric}" not found', ha='center', va='center', transform=ax.transAxes) continue iterations = range(len(train_values)) ax.plot(iterations, train_values, label='Train', linewidth=1.5) ax.plot(iterations, val_values, label='Validation', linewidth=1.5) if best_iteration is not None: ax.axvline(x=best_iteration, color='green', linestyle='--', alpha=0.7) ax.set_title(metric.upper()) ax.set_xlabel('Iterations') ax.legend() ax.grid(True, alpha=0.3) # Hide unused subplots for idx in range(len(metrics), len(axes)): axes[idx].set_visible(False) plt.suptitle('Multi-Metric Training Dashboard', fontsize=14, fontweight='bold') plt.tight_layout() return fig # Usage example visualizer = TrainingVisualizer() # After training fig = visualizer.plot_training_curves( evals_result, metric='logloss', best_iteration=model.best_iteration, title='XGBoost Binary Classification' ) plt.savefig('training_curves.png', dpi=150, bbox_inches='tight') # Multi-metric view fig2 = visualizer.plot_multi_metric_dashboard( evals_result, metrics=['logloss', 'auc', 'error'], best_iteration=model.best_iteration ) plt.savefig('multi_metric_dashboard.png', dpi=150, bbox_inches='tight')65:{"json":{"id":"seed_d4c62301bae4676b1d37d90b914e9d95","slug":"boosting-best-practices","category":"Machine Learning","topic":"Boosting Best Practices","title":"Boosting Best Practices","description":"Master the art of training, tuning, and deploying gradient boosting models in production. Learn training monitoring techniques, early stopping strategies, cross-validation methodologies, class imbalance handling, and production deployment best practices for XGBoost, LightGBM, and CatBoost.","difficulty":"ADVANCED","estimatedMinutes":120,"pages":[{"id":"seed_0cfac23065070fd5a2a40b9d073376a7","title":"Training Monitoring","order":0,"content":[{"id":"opening-hook","title":"The Art of Watching Models Learn","content":[{"data":{"content":"In gradient boosting, training is not a black box—it is a **dynamic, observable process** where each iteration reveals critical information about model behavior. Unlike deep learning where training can take days and feedback loops are slow, boosting algorithms provide **iteration-by-iteration visibility** into loss reduction, generalization performance, and convergence dynamics.\n\nMastering training monitoring transforms you from a passive model trainer into an **active diagnostic practitioner** who can detect overfitting before it becomes catastrophic, identify convergence issues before wasting computational resources, and fine-tune hyperparameters based on real-time training signals."},"type":"text"},{"data":{"icon":"Target","title":"What You Will Learn","content":"By the end of this page, you will understand how to implement comprehensive training monitoring for XGBoost, LightGBM, and CatBoost. You'll master the interpretation of training curves, learn to identify pathological training patterns, and develop the diagnostic intuition that separates production-grade ML practitioners from amateur model builders.","variant":"info"},"type":"callout"}]},{"id":"fundamentals-training-monitoring","title":"Fundamentals of Training Monitoring","content":[{"data":{"content":"Training monitoring in gradient boosting encompasses the systematic observation and analysis of all signals generated during the iterative model building process. Unlike batch learning algorithms that produce a single pass through the data, boosting constructs models **sequentially**, with each new weak learner addressing residual errors from previous iterations.\n\nThis sequential nature creates a unique opportunity: we can observe **how the ensemble evolves** and make informed decisions about when to stop, how to adjust, and whether the training process is healthy."},"type":"text"},{"data":{"items":["**Loss Trajectory Analysis** — Track how training and validation loss decrease over iterations, identifying healthy convergence patterns.","**Generalization Gap Monitoring** — Measure the divergence between training and validation metrics to detect overfitting in real-time.","**Convergence Diagnostics** — Determine whether the model is still learning meaningful patterns or has reached diminishing returns.","**Resource Utilization Tracking** — Monitor computational efficiency including memory usage, training speed, and iteration times.","**Feature Importance Evolution** — Observe how feature importance scores shift as more trees are added to the ensemble."],"title":{"icon":"Target","text":"Core Monitoring Objectives","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"},{"data":{"content":"**The Training Curve as Diagnostic Tool:**\n\nThe **training curve** plots a performance metric against the number of boosting iterations. For gradient boosting, this curve reveals fundamental truths about model behavior:\n\n$$\\mathcal{L}_{train}(t) = \\frac{1}{n} \\sum_{i=1}^{n} L(y_i, F_t(x_i))$$\n\n$$\\mathcal{L}_{val}(t) = \\frac{1}{m} \\sum_{j=1}^{m} L(y_j, F_t(x_j))$$\n\nWhere $F_t(x)$ is the ensemble prediction after $t$ boosting rounds, and $L$ is the loss function. The relationship between these two curves tells a story:\n\n- **Both decreasing together**: Healthy learning, model is generalizing\n- **Training decreasing, validation stagnant**: Early signs of overfitting\n- **Training decreasing, validation increasing**: Active overfitting, stop immediately\n- **Both stagnant from early iterations**: Learning rate too low or model capacity insufficient"},"type":"text"},{"data":{"icon":"Lightbulb","title":"The Validation Set is Your Oracle","content":"In boosting, the validation set serves as your real-time oracle for generalization. Unlike the training set, which the model actively memorizes, the validation set provides unbiased estimates of out-of-sample performance. Every decision about stopping, regularization, or hyperparameter adjustment should be informed by validation metrics.","variant":"tip"},"type":"callout"}],"titleType":"number","titleNumber":1},{"id":"metrics-to-monitor","title":"Metrics to Monitor","content":[{"data":{"content":"Effective training monitoring requires tracking multiple complementary metrics. Each metric provides a different lens into model behavior, and together they paint a comprehensive picture of the training process."},"type":"text"},{"data":{"rows":[["Binary Classification","Log Loss / Binary Cross-Entropy","AUC-ROC, AUC-PR, Accuracy","Val loss increasing, AUC plateauing"],["Multi-class Classification","Multi-class Log Loss","Accuracy, Macro F1, Per-class AUC","Class-specific degradation"],["Regression","MSE / RMSE","MAE, R², MAPE","Val RMSE increasing, heteroscedastic residuals"],["Ranking (LTR)","Lambda Loss, Pairwise Loss","NDCG, MAP, MRR","NDCG@k divergence across k values"],["Survival Analysis","Cox Partial Likelihood","C-index, IBS","C-index stagnation on validation"]],"caption":"Essential Training Metrics by Task Type","headers":["Task Type","Primary Loss","Auxiliary Metrics","Warning Signals"]},"type":"table"},{"data":{"content":"**Primary Loss vs. Evaluation Metrics:**\n\nA critical distinction exists between the **optimization objective** (what the algorithm minimizes) and **evaluation metrics** (what we actually care about). XGBoost, LightGBM, and CatBoost optimize a differentiable loss function, but we often evaluate using non-differentiable or ranking-based metrics:"},"type":"text"},{"data":{"rows":[["Log Loss","Precision@k","Loss may improve while precision degrades"],["Squared Error","MAPE","Outliers dominate loss but not MAPE"],["Binary Log Loss","Cost-sensitive F1","Threshold-dependent, loss-agnostic"]],"headers":["Optimization Objective","Business Metric","Potential Mismatch"]},"type":"table"},{"data":{"content":"This mismatch means you must monitor **both** the optimization loss and your actual evaluation metrics—they can diverge, especially as models overfit."},"type":"text"},{"data":{"code":{"python_lgb":"$6c","python_xgb":"$6d","python_catboost":"$6e"},"fileName":"multi_metric_monitoring.py","languages":[{"label":"Python (XGBoost)","value":"python_xgb"},{"label":"Python (LightGBM)","value":"python_lgb"},{"label":"Python (CatBoost)","value":"python_catboost"}]},"type":"code"},{"data":{"icon":"AlertCircle","title":"Metric Volatility in Early Iterations","content":"During the first 10-50 iterations, metrics can be highly volatile. This is normal—the ensemble hasn't yet built sufficient capacity to make stable predictions. Avoid making decisions based on early-iteration metrics; wait until the training curve shows a clear trend before interpreting patterns.","variant":"warning"},"type":"callout"}],"titleType":"number","titleNumber":2},{"id":"realtime-monitoring-infrastructure","title":"Real-Time Monitoring Infrastructure","content":[{"data":{"content":"Production-grade training monitoring extends beyond printing metrics to the console. Modern ML infrastructure demands **structured logging**, **real-time dashboards**, and **alerting systems** that enable practitioners to monitor training runs remotely and at scale.\n\nThe monitoring infrastructure should support:\n\n1. **Persistent Storage**: Training metrics must be stored for post-hoc analysis, reproducibility, and comparison across experiments\n2. **Real-Time Visualization**: Interactive dashboards that update as training progresses\n3. **Automated Alerting**: Notifications when training exhibits pathological behavior\n4. **Multi-Run Comparison**: Side-by-side comparison of different hyperparameter configurations"},"type":"text"},{"data":{"code":{"python_wandb":"$6f","python_mlflow":"$70"},"fileName":"production_monitoring.py","languages":[{"label":"Python (MLflow)","value":"python_mlflow"},{"label":"Python (Weights & Biases)","value":"python_wandb"}]},"type":"code"},{"data":{"icon":"Info","title":"Monitoring Infrastructure Selection","content":"For production systems, choose your monitoring infrastructure based on team requirements. MLflow excels in on-premise deployments with strong model registry integration. Weights & Biases provides superior visualization and collaboration features for distributed teams. Both integrate seamlessly with all major boosting libraries.","variant":"note"},"type":"callout"}],"titleType":"number","titleNumber":3},{"id":"diagnosing-training-pathologies","title":"Diagnosing Training Pathologies","content":[{"data":{"content":"Training monitoring is only valuable if you can interpret the signals and diagnose problems. This section covers the most common pathological patterns you'll encounter in gradient boosting training and how to address them."},"type":"text"},{"data":{"items":["**Classic Overfitting** — Training loss continues decreasing while validation loss increases. The generalization gap grows unbounded.","**Underfitting / Insufficient Capacity** — Both training and validation loss plateau at high values. The model cannot capture the underlying patterns.","**Slow Convergence** — Loss decreases extremely slowly, requiring thousands of iterations to reach acceptable performance.","**Oscillating Validation Loss** — Validation loss fluctuates significantly between iterations, indicating instability.","**Catastrophic Forgetting** — Validation loss suddenly spikes after many iterations of stable improvement.","**Data Distribution Shift** — Validation loss degrades differently across different validation subsets."],"title":{"icon":"AlertCircle","text":"Common Training Pathologies","type":"icon","color":"destructive","variant":"outline-circle"},"variant":"glass"},"type":"list"},{"data":{"rows":[["Classic Overfitting","Train ↓, Val ↑, Gap ↑","Too many trees, too deep trees, insufficient regularization","Early stopping, reduce depth, increase regularization, subsample"],["Underfitting","Train ↔, Val ↔ (both high)","Too few trees, too shallow depth, learning rate too low","Increase trees, increase depth, increase learning rate"],["Slow Convergence","Both ↓ very slowly","Learning rate too low, insufficient model capacity","Increase learning rate, use learning rate schedules"],["Oscillating Loss","Val zigzag pattern","Learning rate too high, small validation set, label noise","Reduce learning rate, larger validation set, clean data"],["Catastrophic Spike","Sudden Val ↑ after stability","Numerical instability, extreme outliers, data corruption","Check for data issues, gradient clipping, reduce learning rate"]],"caption":"Training Pathology Diagnosis and Treatment","headers":["Pathology","Visual Signature","Root Causes","Remediation"]},"type":"table"},{"data":{"content":"**The Generalization Gap as Primary Diagnostic:**\n\nThe **generalization gap** $\\Delta(t) = \\mathcal{L}_{val}(t) - \\mathcal{L}_{train}(t)$ is the single most informative diagnostic signal:\n\n- **$\\Delta(t) \\approx 0$ and stable**: Model is generalizing well\n- **$\\Delta(t) > 0$ and growing**: Overfitting in progress\n- **$\\Delta(t)$ growing then plateauing**: Overfitting has saturated; early stopping effective\n- **$\\Delta(t)$ oscillating**: Unstable training; regularization needed\n\nMonitor the **rate of change** of the gap, not just its absolute value. A slowly growing gap over 1000 iterations is less concerning than a rapidly growing gap over 100 iterations."},"type":"text"},{"data":{"code":{"python":"$71"},"fileName":"pathology_detection.py","languages":[{"label":"Python","value":"python"}]},"type":"code"},{"data":{"icon":"Lightbulb","title":"Proactive vs. Reactive Monitoring","content":"The best monitoring is proactive—detecting issues before they become severe. Configure your monitoring system to alert when the generalization gap growth rate exceeds a threshold, not just when the gap itself crosses a limit. Early intervention saves computational resources and produces better models.","variant":"tip"},"type":"callout"}],"titleType":"number","titleNumber":4},{"id":"visualizing-training-progress","title":"Visualizing Training Progress","content":[{"data":{"content":"Visualization transforms abstract metrics into actionable insights. Effective training visualization should be **real-time**, **comparative**, and **informative**—showing not just where you are, but where you're going and why."},"type":"text"},{"data":{"code":{"python":"$72"},"fileName":"training_visualization.py","languages":[{"label":"Python (Matplotlib)","value":"python"}]},"type":"code"},{"data":{"icon":"Info","title":"Interactive Visualization in Production","content":"For production training runs, static plots are insufficient. Use interactive dashboards (Plotly, Bokeh) or integrate with monitoring platforms (Grafana, DataDog) that support real-time updates. This enables intervention during training rather than post-hoc analysis only.","variant":"note"},"type":"callout"}],"titleType":"number","titleNumber":5},{"id":"summary-best-practices","title":"Summary and Best Practices","content":[{"data":{"content":"Effective training monitoring is the foundation of production-grade gradient boosting. It transforms model training from a trial-and-error process into a disciplined, observable, and reproducible engineering practice."},"type":"text"},{"data":{"type":"ordered","items":["**Always use a validation set** — Never train without real-time generalization feedback. The validation set is your oracle for out-of-sample performance.","**Monitor multiple metrics** — Track both the optimization objective and your actual business metrics. They can diverge, especially during overfitting.","**Calculate the generalization gap** — The difference between validation and training loss is the most informative diagnostic signal.","**Watch the trend, not just the value** — A growing gap is more concerning than a large but stable gap.","**Log metrics persistently** — Use MLflow, Weights & Biases, or equivalent. Training curves should be reviewable for any historical run.","**Implement automated pathology detection** — Configure alerts for overfitting, oscillation, and catastrophic degradation.","**Visualize training dynamics** — Create dashboards that show loss curves, generalization gaps, and improvement rates.","**Track resource utilization** — Monitor training speed, memory usage, and per-iteration timing for efficiency optimization."],"title":{"icon":"CheckCircle","text":"Training Monitoring Best Practices","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"},{"data":{"icon":"CheckCircle","title":"Page Complete","content":"You now understand how to implement comprehensive training monitoring for gradient boosting models. You can track essential metrics, diagnose training pathologies, and build infrastructure for real-time observability. Next, we'll explore early stopping—the technique that leverages your monitoring signals to automatically determine the optimal stopping point.","variant":"success"},"type":"callout"}],"titleType":"number","titleNumber":6}],"isLocked":false},{"id":"seed_9319321188b2e2866f475f7a2d580793","title":"Early Stopping","order":1,"content":null,"isLocked":true},{"id":"seed_cc82cb532dc200977d249bf5f116ae94","title":"Cross-Validation for Boosting","order":2,"content":null,"isLocked":true},{"id":"seed_428b7a4e049d1d41fcebaa026fcd138b","title":"Handling Class Imbalance","order":3,"content":null,"isLocked":true},{"id":"seed_ef87b51f42063bf9d403fdf614938ce3","title":"Production Deployment","order":4,"content":null,"isLocked":true}]}}