45:"/_next/static/chunks/33yus346z0l_n.js" 46:"/_next/static/chunks/1r8ezocsukqp8.js" 47:"/_next/static/chunks/0iv56yrji_20j.js" 48:"/_next/static/chunks/0wa9m1-tphf9u.js" 49:"/_next/static/chunks/08cuqqn_f60pp.js" 4a:"/_next/static/chunks/3zdsl3f9b0--r.js" 4b:"/_next/static/chunks/2j22cxchii_xx.js" 4c:"/_next/static/chunks/3bgoyiso1rhn7.js" 4d:"/_next/static/chunks/01a-3hvnqu8wl.js" 4e:"/_next/static/chunks/3m1xjj5kw5fie.js" 4f:"/_next/static/chunks/0om3fkgfnicab.js" 50:"/_next/static/chunks/0gafyzg9j9yqj.js" 51:"/_next/static/chunks/20w4eaqlekefm.js" 52:"/_next/static/chunks/449--h5z1s6nq.js" 53:"/_next/static/chunks/3k2gyg36fnag_.js" 54:"/_next/static/chunks/18ggtgz2yf759.js" 55:"/_next/static/chunks/0hlb44ca7-4a2.js" 56:"/_next/static/chunks/253gnmj0-cj17.js" 57:"/_next/static/chunks/09nq57stwzkgf.js" 58:"/_next/static/chunks/082jo-0ajazgr.js" 59:"/_next/static/chunks/2ozcrjb_hh74t.js" 5a:"/_next/static/chunks/05i8en-x14ibi.js" 5b:"/_next/static/chunks/32izpn90gvm22.js" 5c:"/_next/static/chunks/0jbzid2v9xv3h.js" 5d:"/_next/static/chunks/0i3-eynrkvqi7.js" 5e:"/_next/static/chunks/1tn-0fivsxsv1.js" 5f:"/_next/static/chunks/34h36hyf_ggow.js" 60:"/_next/static/chunks/20329i6m_04t6.js" 61:"/_next/static/chunks/161uxd62cjny2.js" 62:"/_next/static/chunks/2k8pepwc4pxns.js" 63:I[593304,["$2b","$2c","$2d","$2e","$2f","$30","$45","$46","$47","$48","$49","$4a","$4b","$4c","$4d","$4e","$4f","$50","$51","$52","$53","$54","$55","$56","$57","$58","$59","$5a","$5b","$5c","$5d","$5e","$5f","$60","$61","$62"],"OwnedHydration"] 64:T82e,import numpy as np def gini_impurity(class_proportions): """ Calculate Gini impurity for a node. Args: class_proportions: Array of class proportions (must sum to 1) Returns: Gini impurity value in [0, 1 - 1/K] """ return 1 - np.sum(class_proportions ** 2) def entropy(class_proportions): """ Calculate entropy for a node. Args: class_proportions: Array of class proportions (must sum to 1) Returns: Entropy in bits, in [0, log2(K)] """ # Filter out zero proportions to avoid log(0) p = class_proportions[class_proportions > 0] return -np.sum(p * np.log2(p)) def impurity_decrease(parent_impurity, left_impurity, right_impurity, n_left, n_right): """ Calculate the impurity decrease (information gain) from a split. Args: parent_impurity: Impurity of the parent node left_impurity: Impurity of left child right_impurity: Impurity of right child n_left: Number of samples in left child n_right: Number of samples in right child Returns: Weighted impurity decrease """ n_total = n_left + n_right weighted_child_impurity = ( (n_left / n_total) * left_impurity + (n_right / n_total) * right_impurity ) return parent_impurity - weighted_child_impurity # Example: Binary classification # Node with 70% class 0, 30% class 1 p = np.array([0.7, 0.3]) print(f"Class proportions: {p}") print(f"Gini impurity: {gini_impurity(p):.4f}") # 0.42 print(f"Entropy: {entropy(p):.4f} bits") # 0.881 # Example: Impurity decrease from a split # Parent: 100 samples, 50/50 split → Gini = 0.5 # Left: 60 samples, 80% class 0 → Gini = 0.32 # Right: 40 samples, 10% class 0 → Gini = 0.18 parent_gini = gini_impurity(np.array([0.5, 0.5])) left_gini = gini_impurity(np.array([0.8, 0.2])) right_gini = gini_impurity(np.array([0.1, 0.9])) gain = impurity_decrease(parent_gini, left_gini, right_gini, 60, 40) print(f" Impurity decrease: {gain:.4f}") # 0.5 - 0.264 = 0.23665:T822,import numpy as np def variance_impurity(y): """ Calculate variance impurity for a regression node. Args: y: Array of target values in the node Returns: Variance of target values """ return np.var(y) def mae_impurity(y): """ Calculate MAE impurity for a regression node. Args: y: Array of target values in the node Returns: Mean absolute deviation from median """ return np.mean(np.abs(y - np.median(y))) def variance_reduction(y_parent, y_left, y_right): """ Calculate variance reduction from a regression split. Args: y_parent: Target values in parent node y_left: Target values in left child y_right: Target values in right child Returns: Variance reduction achieved by the split """ n_parent = len(y_parent) n_left = len(y_left) n_right = len(y_right) var_parent = variance_impurity(y_parent) var_left = variance_impurity(y_left) var_right = variance_impurity(y_right) weighted_child_variance = ( (n_left / n_parent) * var_left + (n_right / n_parent) * var_right ) return var_parent - weighted_child_variance # Example: Regression split # Parent node: 10 samples with values [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] y_parent = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) # Split at value 5: left = [1,2,3,4,5], right = [6,7,8,9,10] y_left = np.array([1, 2, 3, 4, 5]) y_right = np.array([6, 7, 8, 9, 10]) print(f"Parent variance: {variance_impurity(y_parent):.4f}") # 8.25 print(f"Left child variance: {variance_impurity(y_left):.4f}") # 2.0 print(f"Right child variance: {variance_impurity(y_right):.4f}") # 2.0 print(f"Variance reduction: {variance_reduction(y_parent, y_left, y_right):.4f}") # 6.25 # Verify using the mean difference formula n_l, n_r = len(y_left), len(y_right) mean_diff = np.mean(y_left) - np.mean(y_right) formula_result = (n_l * n_r / (n_l + n_r)**2) * mean_diff**2 print(f"Formula verification: {formula_result:.4f}") # 6.2566:T95e,import numpy as np from sklearn.tree import DecisionTreeClassifier from sklearn.datasets import make_classification # Create synthetic dataset X, y = make_classification( n_samples=1000, n_features=10, n_informative=5, n_redundant=2, n_clusters_per_class=2, random_state=42 ) # Train a decision tree tree = DecisionTreeClassifier(max_depth=5, random_state=42) tree.fit(X, y) # Access the built-in impurity-based importance importance_builtin = tree.feature_importances_ print("Built-in feature importances:") for i, imp in enumerate(importance_builtin): print(f" Feature {i}: {imp:.4f}") # Manual computation of feature importance (for educational purposes) def compute_mdi_importance(tree): """ Manually compute Mean Decrease in Impurity importance. Args: tree: Fitted sklearn DecisionTreeClassifier Returns: Array of feature importances """ tree_ = tree.tree_ n_features = tree_.n_features importance = np.zeros(n_features) for node_id in range(tree_.node_count): # Skip leaf nodes if tree_.children_left[node_id] == -1: continue # Get node properties feature = tree_.feature[node_id] n_samples = tree_.n_node_samples[node_id] impurity = tree_.impurity[node_id] # Get children properties left_child = tree_.children_left[node_id] right_child = tree_.children_right[node_id] n_left = tree_.n_node_samples[left_child] n_right = tree_.n_node_samples[right_child] impurity_left = tree_.impurity[left_child] impurity_right = tree_.impurity[right_child] # Compute weighted impurity decrease impurity_decrease = ( n_samples * impurity - n_left * impurity_left - n_right * impurity_right ) importance[feature] += impurity_decrease # Normalize to sum to 1 importance /= tree_.n_node_samples[0] # Divide by root node samples importance /= importance.sum() return importance importance_manual = compute_mdi_importance(tree) print(" Manual computation (verification):") for i, imp in enumerate(importance_manual): print(f" Feature {i}: {imp:.4f}") # Verify they match print(f" Importances match: {np.allclose(importance_builtin, importance_manual)}")67:T87b,import numpy as np import matplotlib.pyplot as plt from sklearn.ensemble import RandomForestClassifier from sklearn.datasets import make_classification # Create dataset with known informative features X, y = make_classification( n_samples=1000, n_features=10, n_informative=4, # Features 0-3 are truly informative n_redundant=2, # Features 4-5 are linear combinations n_repeated=0, n_classes=2, random_state=42 ) # Train Random Forest rf = RandomForestClassifier( n_estimators=100, max_depth=10, random_state=42, n_jobs=-1 ) rf.fit(X, y) # Get ensemble importance (averaged across trees) ensemble_importance = rf.feature_importances_ # Also compute individual tree importances to show variance tree_importances = np.array([tree.feature_importances_ for tree in rf.estimators_]) # Compute mean and standard deviation across trees importance_mean = tree_importances.mean(axis=0) importance_std = tree_importances.std(axis=0) # Display results print("Feature Importance (Random Forest):") print("=" * 55) print(f"{'Feature':<10} {'Importance':<12} {'Std Dev':<12} {'Status'}") print("-" * 55) # True informative features are 0-3 for i in range(10): status = "Informative" if i < 4 else ("Redundant" if i < 6 else "Noise") print(f"Feature {i:<2} {importance_mean[i]:>10.4f} {importance_std[i]:>10.4f} {status}") # Visualize importance with error bars fig, ax = plt.subplots(figsize=(10, 6)) x_pos = np.arange(10) ax.bar(x_pos, importance_mean, yerr=importance_std, capsize=4, color='steelblue', alpha=0.8) ax.set_xlabel('Feature Index') ax.set_ylabel('Mean Decrease in Impurity') ax.set_title('Random Forest Feature Importance with Standard Deviation') ax.set_xticks(x_pos) # Add feature type annotations for i, (mean, std) in enumerate(zip(importance_mean, importance_std)): if i < 4: ax.annotate('Informative', (i, mean + std + 0.01), ha='center', fontsize=8, color='green') elif i < 6: ax.annotate('Redundant', (i, mean + std + 0.01), ha='center', fontsize=8, color='orange') plt.tight_layout()68:Tbda,import numpy as np import pandas as pd from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from sklearn.model_selection import cross_val_score def analyze_feature_importance(X, y, feature_names=None, n_runs=5, is_classifier=True, **rf_params): """ Comprehensive impurity-based importance analysis with stability assessment. Args: X: Feature matrix y: Target vector feature_names: List of feature names n_runs: Number of random seed runs for stability is_classifier: True for classification, False for regression **rf_params: Additional Random Forest parameters Returns: DataFrame with importance statistics """ n_features = X.shape[1] if feature_names is None: feature_names = [f"Feature_{i}" for i in range(n_features)] # Collect importances across multiple runs all_importances = [] for run in range(n_runs): if is_classifier: rf = RandomForestClassifier(random_state=run, n_jobs=-1, **rf_params) else: rf = RandomForestRegressor(random_state=run, n_jobs=-1, **rf_params) rf.fit(X, y) all_importances.append(rf.feature_importances_) all_importances = np.array(all_importances) # Compute statistics results = pd.DataFrame({ 'feature': feature_names, 'mean_importance': all_importances.mean(axis=0), 'std_importance': all_importances.std(axis=0), 'min_importance': all_importances.min(axis=0), 'max_importance': all_importances.max(axis=0), }) # Compute coefficient of variation (stability metric) results['cv'] = results['std_importance'] / results['mean_importance'].replace(0, np.inf) # Compute mean rank across runs ranks = np.array([np.argsort(-imp) for imp in all_importances]) mean_ranks = ranks.mean(axis=0) results['mean_rank'] = mean_ranks # Sort by mean importance results = results.sort_values('mean_importance', ascending=False).reset_index(drop=True) return results # Example usage if __name__ == "__main__": from sklearn.datasets import make_classification X, y = make_classification( n_samples=1000, n_features=20, n_informative=10, n_redundant=5, random_state=42 ) feature_names = [f"feature_{i}" for i in range(20)] results = analyze_feature_importance( X, y, feature_names=feature_names, n_runs=10, n_estimators=100, max_depth=10 ) print("Feature Importance Analysis") print("=" * 80) print(results.to_string(index=False)) print() # Identify stable vs unstable features print(" Stability Analysis:") stable = results[results['cv'] < 0.2] unstable = results[results['cv'] >= 0.2] print(f" Stable features (CV < 0.2): {len(stable)}") print(f" Unstable features (CV >= 0.2): {len(unstable)}")44:["$","$L63",null,{"owner":"anonymous","state":{"mutations":[],"queries":[{"dehydratedAt":1792317794433,"state":{"data":{"json":{"id":"seed_650cf74b6e81d44f39e80cc287f6b75b","slug":"feature-importance","category":"Machine Learning","topic":"Bagging & Random Forests","title":"Feature Importance","description":"Master the art and science of feature importance in ensemble methods. Learn to quantify, compare, and interpret feature contributions using impurity-based, permutation-based, and drop-column methods while understanding their biases and limitations.","difficulty":"ADVANCED","estimatedMinutes":90,"pages":[{"id":"seed_ce10f972faaec2f14d6203a335fd01ab","title":"Impurity-based Importance","order":0,"content":[{"id":"opening-hook","title":"The Quest to Understand Model Decisions","content":[{"data":{"content":"When you train a Random Forest or Gradient Boosted ensemble and achieve remarkable predictive accuracy, a natural question emerges: **Which features are actually driving these predictions?** This question isn't merely academic curiosity—it's fundamental to model interpretability, feature engineering, domain validation, and ultimately building trust in machine learning systems.\n\nImpurity-based importance, also known as **Mean Decrease in Impurity (MDI)** or **Gini importance**, represents the most widely used method for answering this question in tree-based models. It's computed automatically during training, requires no additional computation, and provides intuitive rankings of feature significance. But beneath this elegant simplicity lies a sophisticated mathematical framework—and subtle biases that every practitioner must understand."},"type":"text"},{"data":{"icon":"Target","title":"What You Will Learn","content":"By the end of this page, you will understand: (1) The mathematical foundation of impurity-based importance, (2) How it's computed in decision trees and ensembles, (3) The relationship between Gini impurity, entropy, and importance scores, (4) Step-by-step calculation examples, and (5) The fundamental assumptions underlying this method.","variant":"info"},"type":"callout"}]},{"id":"intuition","title":"The Intuition Behind Impurity-based Importance","content":[{"data":{"content":"Consider how decision trees make predictions. At each internal node, the tree examines a single feature and asks: *\"Based on this feature's value, how should we partition the data?\"* The goal is to create child nodes that are **purer** than the parent—meaning samples within each child node are more homogeneous with respect to the target variable.\n\n**The key insight:** If a feature enables highly effective splits—creating much purer child nodes—then that feature must carry substantial predictive information. Features that consistently enable large impurity reductions across many splits are fundamentally **more important** for the model's decision-making process."},"type":"text"},{"data":{"icon":"Lightbulb","title":"The Core Principle","content":"Impurity-based importance quantifies importance as the **total reduction in impurity** that a feature provides across all nodes where it's used for splitting, weighted by the number of samples reaching those nodes. Features enabling large, well-populated impurity reductions rank higher.","variant":"tip"},"type":"callout"},{"data":{"content":"**Formalizing the intuition:**\n\nLet's define the components mathematically. For each split in a decision tree, we have:\n\n- **Parent node** with $N$ samples and impurity $I_{parent}$\n- **Left child** with $N_L$ samples and impurity $I_L$\n- **Right child** with $N_R$ samples and impurity $I_R$\n\nThe **impurity decrease** from this split is:\n\n$$\\Delta I = I_{parent} - \\frac{N_L}{N} I_L - \\frac{N_R}{N} I_R$$\n\nThis weighted average accounts for the fact that a split creating one very pure but tiny child and one large impure child isn't as valuable as a split creating two reasonably pure, well-balanced children."},"type":"text"},{"data":{"rows":[["Parent impurity","$$I_{parent}$","Impurity before split","[0, max]"],["Left child impurity","$$I_L$","Impurity of left partition","[0, max]"],["Right child impurity","$$I_R$","Impurity of right partition","[0, max]"],["Left sample fraction","$$N_L / N$","Proportion going left","(0, 1)"],["Right sample fraction","$$N_R / N$","Proportion going right","(0, 1)"],["Impurity decrease","$$\\Delta I$","Reduction achieved","[0, $I_{parent}$]"]],"caption":"Components of Impurity Decrease Calculation","headers":["Term","Symbol","Meaning","Range"]},"type":"table"}],"titleType":"number","titleNumber":1},{"id":"impurity-measures-classification","title":"Impurity Measures for Classification","content":[{"data":{"content":"For classification tasks, two impurity measures dominate: **Gini impurity** and **entropy**. Both quantify the \"disorder\" or \"heterogeneity\" of a node's class distribution, but they differ subtly in their mathematical properties and practical implications.\n\n**Gini Impurity:**\n\nGini impurity measures the probability that a randomly chosen sample would be incorrectly classified if labeled according to the distribution of labels in the node:\n\n$$I_{Gini}(t) = 1 - \\sum_{k=1}^{K} p_k^2 = \\sum_{k=1}^{K} p_k(1 - p_k)$$\n\nwhere $p_k$ is the proportion of samples in node $t$ belonging to class $k$.\n\n**Key properties of Gini impurity:**\n- **Range:** $[0, 1 - 1/K]$ where $K$ is the number of classes\n- **Minimum (0):** Pure node—all samples belong to one class\n- **Maximum ($1 - 1/K$):** Maximum impurity—samples uniformly distributed across classes\n- **Binary case:** Maximum of 0.5 when $p = 0.5$"},"type":"text"},{"data":{"icon":"Info","title":"Understanding the Gini Formula","content":"The term $p_k(1-p_k)$ represents the probability of selecting a sample from class $k$ and then a sample NOT from class $k$. Summing over all classes gives the total probability of a classification disagreement.","variant":"note"},"type":"callout"},{"data":{"content":"**Entropy (Information Gain):**\n\nEntropy originates from information theory and measures the expected information content (in bits) needed to describe the class of a random sample:\n\n$$I_{entropy}(t) = -\\sum_{k=1}^{K} p_k \\log_2(p_k)$$\n\nBy convention, $0 \\log 0 = 0$.\n\n**Key properties of entropy:**\n- **Range:** $[0, \\log_2 K]$\n- **Minimum (0):** Pure node—zero uncertainty\n- **Maximum ($\\log_2 K$):** Uniform distribution—maximum uncertainty\n- **Binary case:** Maximum of 1.0 when $p = 0.5$"},"type":"text"},{"data":{"rows":[["0.0","1.0","0.000","0.000"],["0.1","0.9","0.180","0.469"],["0.2","0.8","0.320","0.722"],["0.3","0.7","0.420","0.881"],["0.4","0.6","0.480","0.971"],["0.5","0.5","0.500","1.000"]],"caption":"Comparison of Gini and Entropy for Binary Classification","headers":["$$p_1$ (Class 1)","$$p_2$ (Class 2)","Gini Impurity","Entropy (bits)"]},"type":"table"},{"data":{"content":"**Practical behavior comparison:**\n\nBoth measures achieve their minimum at pure nodes and maximum at uniform distributions. However, entropy is slightly more sensitive to changes in skewed distributions, while Gini tends to favor creating one large pure node when possible.\n\nFor computing feature importance, **both measures typically produce highly correlated rankings**. The choice between them rarely impacts which features are deemed most important, though absolute importance values will differ due to the different scales."},"type":"text"},{"data":{"fileName":"impurity_measures.py","languages":[{"code":"$64","label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"}],"titleType":"number","titleNumber":2},{"id":"impurity-measures-regression","title":"Impurity Measures for Regression","content":[{"data":{"content":"For regression trees, we replace the class-based impurity measures with variance-based measures that quantify how spread out the target values are within a node.\n\n**Mean Squared Error (Variance):**\n\nThe most common regression impurity measure is the variance (equivalent to MSE from the node's mean prediction):\n\n$$I_{MSE}(t) = \\frac{1}{N_t} \\sum_{i \\in t} (y_i - \\bar{y}_t)^2 = Var(y_t)$$\n\nwhere $\\bar{y}_t$ is the mean target value of samples in node $t$.\n\n**Mean Absolute Error:**\n\nAn alternative measure using absolute deviations:\n\n$$I_{MAE}(t) = \\frac{1}{N_t} \\sum_{i \\in t} |y_i - median(y_t)|$$\n\nMAE is more robust to outliers but computationally more expensive due to median computation."},"type":"text"},{"data":{"icon":"Lightbulb","title":"The Regression Analogy","content":"In regression, 'purity' means target values are tightly clustered around a single value (low variance). A perfect split would create child nodes where every sample has exactly the target value equal to the node's prediction—zero variance.","variant":"tip"},"type":"callout"},{"data":{"content":"**Variance reduction calculation:**\n\nThe impurity decrease for regression is computed identically to classification, substituting variance for Gini/entropy:\n\n$$\\Delta I = Var(y_{parent}) - \\frac{N_L}{N} Var(y_L) - \\frac{N_R}{N} Var(y_R)$$\n\nThis quantity is also known as the **reduction in variance** or **variance reduction**.\n\n**A key mathematical fact:** The variance decrease from a split can be expressed in terms of the squared difference between child means:\n\n$$\\Delta I = \\frac{N_L \\cdot N_R}{(N_L + N_R)^2} (\\bar{y}_L - \\bar{y}_R)^2$$\n\nThis reveals that splits maximizing variance reduction are those that create children with maximally different mean predictions and balanced sample sizes."},"type":"text"},{"data":{"fileName":"regression_impurity.py","languages":[{"code":"$65","label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"}],"titleType":"number","titleNumber":3},{"id":"single-tree-importance","title":"Computing Feature Importance in a Single Tree","content":[{"data":{"content":"With impurity decrease defined for individual splits, we can now compute feature importance for an entire decision tree. The algorithm is elegant:\n\n**Algorithm: Mean Decrease in Impurity (MDI)**\n\n1. Initialize importance for each feature to 0\n2. Traverse every internal node in the tree\n3. For each node splitting on feature $j$:\n - Compute the weighted impurity decrease $N_t \\cdot \\Delta I_t$\n - Add this value to the importance of feature $j$\n4. Normalize all importances to sum to 1 (optional)"},"type":"text"},{"data":{"icon":"Info","title":"The Weighting Factor","content":"The impurity decrease is weighted by the number of samples $N_t$ reaching node $t$. This reflects the principle that splits affecting more samples should contribute more to feature importance. A split at the root (affecting all samples) contributes more than a deep split (affecting few samples).","variant":"note"},"type":"callout"},{"data":{"content":"**Formal definition:**\n\nFor a tree $T$ with a set of internal nodes $\\mathcal{N}$, the importance of feature $j$ is:\n\n$$Imp(j) = \\sum_{t \\in \\mathcal{N}: split\\_feature(t) = j} N_t \\cdot \\Delta I_t$$\n\nwhere $N_t$ is the number of samples reaching node $t$ and $\\Delta I_t$ is the impurity decrease at node $t$.\n\n**Normalization:**\n\nTo make importances interpretable as proportions (summing to 1), we normalize:\n\n$$Imp_{normalized}(j) = \\frac{Imp(j)}{\\sum_{k} Imp(k)}$$\n\nThis allows statements like \"feature X accounts for 30% of the total impurity reduction in this tree.\""},"type":"text"},{"data":{"left":[{"data":{"items":["**Computed during training** — No additional computational cost after model is trained","**Fast to retrieve** — Simply sum pre-computed values during tree traversal","**Intuitive interpretation** — Directly measures contribution to model's decision process","**Always available** — Works for any tree-based model"],"title":{"icon":"CheckCircle","text":"Advantages of MDI","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"}],"right":[{"data":{"items":["**Biased toward high-cardinality features** — More split opportunities inflates importance","**Doesn't reflect predictive power** — Measures in-sample, not generalization","**Correlated features share importance** — Can dilute importance across redundant features","**Tree structure dependent** — Different random seeds yield different rankings"],"title":{"icon":"AlertCircle","text":"Limitations to Consider","type":"icon","color":"destructive","variant":"outline-circle"},"variant":"accent"},"type":"list"}]},"type":"two-column"},{"data":{"fileName":"tree_feature_importance.py","languages":[{"code":"$66","label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"}],"titleType":"number","titleNumber":4},{"id":"random-forest-importance","title":"Feature Importance in Random Forests","content":[{"data":{"content":"In Random Forests and other ensemble methods, feature importance is computed by **averaging** the importances across all trees in the ensemble:\n\n$$Imp_{RF}(j) = \\frac{1}{B} \\sum_{b=1}^{B} Imp_{T_b}(j)$$\n\nwhere $B$ is the number of trees and $Imp_{T_b}(j)$ is the importance of feature $j$ in tree $b$.\n\n**Why averaging helps:**\n\nSingle-tree importances can be unstable—different random seeds produce different tree structures and thus different importance rankings. By averaging over many trees (each trained on bootstrap samples with random feature subsets), the ensemble importance smooths out this variance and provides more robust estimates."},"type":"text"},{"data":{"icon":"Lightbulb","title":"Variance Reduction in Importance Estimates","content":"Just as Random Forests reduce prediction variance through averaging, they also reduce variance in importance estimates. This makes ensemble importances more reliable than single-tree importances, especially when feature correlations exist.","variant":"tip"},"type":"callout"},{"data":{"content":"**The role of feature subsampling:**\n\nRandom Forests use a random subset of features at each split (typically $\\sqrt{p}$ for classification or $p/3$ for regression). This has important implications for feature importance:\n\n1. **Decorrelation effect:** Correlated features get opportunities to be selected independently across different trees, potentially sharing importance more equitably\n\n2. **Detection of weak features:** Features that would never be selected in a greedy tree (overshadowed by stronger features) can prove their worth when stronger features are excluded from consideration\n\n3. **Importance inflation for rare splitters:** Features that rarely appear (due to low max_features) get their importance concentrated in fewer splits, which can inflate or deflate their normalized importance"},"type":"text"},{"data":{"fileName":"random_forest_importance.py","languages":[{"code":"$67","label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"}],"titleType":"number","titleNumber":5},{"id":"mathematical-properties","title":"Mathematical Properties and Guarantees","content":[{"data":{"content":"Understanding the theoretical properties of impurity-based importance helps us interpret results correctly and recognize limitations.\n\n**Property 1: Non-negativity**\n\nImpurity-based importance is always non-negative: $Imp(j) \\geq 0$ for all features $j$. This follows from the fact that any split must reduce weighted impurity (or at least not increase it), so $\\Delta I_t \\geq 0$ for all nodes.\n\n**Property 2: Summation to unity**\n\nAfter normalization, importances sum to 1: $\\sum_j Imp(j) = 1$. This allows interpretation as proportional contributions to total impurity reduction.\n\n**Property 3: Unused features have zero importance**\n\nFeatures that never appear in any split have exactly zero importance. This can occur due to:\n- Feature redundancy (another feature is always preferred)\n- Feature subsampling in Random Forests (bad luck across all trees)\n- Early stopping (tree is shallow, not all features get used)"},"type":"text"},{"data":{"icon":"AlertCircle","title":"Zero Importance ≠ Irrelevance","content":"A feature with zero impurity-based importance may still be predictive! It might just be redundant with another feature that was chosen instead. Always verify with complementary methods like permutation importance.","variant":"warning"},"type":"callout"},{"data":{"content":"**Property 4: Bias toward features with more splits**\n\nThis is the critical limitation of impurity-based importance. Consider two features:\n\n- **Feature A:** Continuous, enables splits anywhere in its range\n- **Feature B:** Binary, enables only one possible split\n\nEven if Feature B is more predictive per-split, Feature A will likely accumulate more importance simply because it has more splitting opportunities. This bias is especially severe for:\n\n- High-cardinality categorical features (many distinct values)\n- Continuous features vs. binary features\n- Features with no missing values vs. features with missing values\n\n**Property 5: Sensitivity to tree depth**\n\nDeeper trees use more features and distribute importance more evenly. Shallow trees concentrate importance in root-proximal features. This means importance rankings can change dramatically based on `max_depth` settings."},"type":"text"},{"data":{"rows":[["max_depth=1","1","Single feature gets 100%","Baseline/sanity check"],["max_depth=3","7","Top 3-5 features dominate","Interpretable models"],["max_depth=10","~1000","Many features contribute","Balanced models"],["max_depth=None","~n_samples","Importance spread widely including noise","Maximum accuracy"]],"caption":"Impact of Tree Depth on Importance Distribution","headers":["Tree Depth","# Splits Used","Importance Concentration","Typical Use Case"]},"type":"table"}],"titleType":"number","titleNumber":6},{"id":"worked-example","title":"Worked Example: Step-by-Step Calculation","content":[{"data":{"content":"Let's walk through a complete example of computing impurity-based importance for a small decision tree.\n\n**Dataset:** 100 samples, 3 features, binary classification (50 positive, 50 negative)\n\n**Tree Structure:**\n\n```\n Root (n=100, 50+/50-)\n Split: Feature 0 ≤ 0.5\n / \n Node A (n=60, 45+/15-) Node B (n=40, 5+/35-)\n Split: Feature 1 ≤ 0.3 Split: Feature 2 ≤ 0.7\n / \\ / \n Leaf (40+/5-) Leaf (5+/10-) Leaf (4+/10-) Leaf (1+/25-)\n n=45 n=15 n=14 n=26\n```"},"type":"text"},{"data":{"content":"**Step 1: Compute impurities for all nodes**\n\nUsing Gini impurity: $I = 1 - p_+^2 - p_-^2$"},"type":"text"},{"data":{"rows":[["Root","100","50+/50- = (0.5, 0.5)","$$1 - 0.5^2 - 0.5^2 = 0.500$"],["A","60","45+/15- = (0.75, 0.25)","$$1 - 0.75^2 - 0.25^2 = 0.375$"],["B","40","5+/35- = (0.125, 0.875)","$$1 - 0.125^2 - 0.875^2 = 0.219$"],["A-Left","45","40+/5- = (0.889, 0.111)","$$1 - 0.889^2 - 0.111^2 = 0.198$"],["A-Right","15","5+/10- = (0.333, 0.667)","$$1 - 0.333^2 - 0.667^2 = 0.444$"],["B-Left","14","4+/10- = (0.286, 0.714)","$$1 - 0.286^2 - 0.714^2 = 0.408$"],["B-Right","26","1+/25- = (0.038, 0.962)","$$1 - 0.038^2 - 0.962^2 = 0.074$"]],"headers":["Node","Samples","Class Distribution","Gini Impurity"]},"type":"table"},{"data":{"content":"**Step 2: Compute impurity decrease for each split**\n\n**Root split (Feature 0):**\n$$\\Delta I_{root} = 0.500 - \\frac{60}{100}(0.375) - \\frac{40}{100}(0.219)$$\n$$= 0.500 - 0.225 - 0.088 = 0.187$$\n\nWeighted by samples: $100 \\times 0.187 = 18.7$\n\n**Node A split (Feature 1):**\n$$\\Delta I_A = 0.375 - \\frac{45}{60}(0.198) - \\frac{15}{60}(0.444)$$\n$$= 0.375 - 0.149 - 0.111 = 0.115$$\n\nWeighted by samples: $60 \\times 0.115 = 6.9$\n\n**Node B split (Feature 2):**\n$$\\Delta I_B = 0.219 - \\frac{14}{40}(0.408) - \\frac{26}{40}(0.074)$$\n$$= 0.219 - 0.143 - 0.048 = 0.028$$\n\nWeighted by samples: $40 \\times 0.028 = 1.1$"},"type":"text"},{"data":{"content":"**Step 3: Aggregate by feature**"},"type":"text"},{"data":{"rows":[["Feature 0","Root","18.7","$$18.7 / 26.7 = 0.700$"],["Feature 1","Node A","6.9","$$6.9 / 26.7 = 0.258$"],["Feature 2","Node B","1.1","$$1.1 / 26.7 = 0.042$"],["**Total**","—","**26.7**","**1.000**"]],"headers":["Feature","Nodes Used","Weighted Contribution","Normalized Importance"]},"type":"table"},{"data":{"content":"**Interpretation:** Feature 0 (used at the root) accounts for 70% of total impurity reduction, making it the most important feature by this metric. Feature 2 contributes only 4.2%, suggesting it provides minimal additional information beyond what Features 0 and 1 capture."},"type":"text"},{"data":{"icon":"Info","title":"Root Feature Dominance","content":"Features selected at the root tend to have highest importance because: (1) they affect all samples, and (2) the root has maximum impurity to reduce. This is often appropriate—the root feature typically IS most informative—but can exaggerate the gap between top features and others.","variant":"note"},"type":"callout"}],"titleType":"number","titleNumber":7},{"id":"implementation-best-practices","title":"Practical Implementation and Best Practices","content":[{"data":{"content":"When using impurity-based importance in practice, several considerations will help you extract reliable insights."},"type":"text"},{"data":{"items":["**Use ensembles over single trees** — Random Forest importance is more stable than single Decision Tree importance. The averaging reduces variance and the random feature selection helps correlated features share importance.","**Run multiple experiments** — Train several models with different random seeds and check consistency of top features. Truly important features should appear at the top across runs.","**Be cautious with high-cardinality features** — If you have features with many unique values (IDs, timestamps, ZIP codes), their importance will be artificially inflated. Consider binning or encoding strategies.","**Combine with permutation importance** — Use impurity-based importance for initial screening (it's fast), then validate findings with permutation importance (it measures true predictive contribution).","**Consider feature correlations** — Importance gets distributed among correlated features. If you have groups of related features, consider their combined importance rather than individual rankings.","**Beware of data leakage** — Features that leak target information will show extremely high importance. Suspiciously dominant features warrant investigation."],"title":{"icon":"Target","text":"Best Practices for Impurity-based Importance","type":"icon","color":"primary","variant":"outline-circle"},"variant":"glass"},"type":"list"},{"data":{"fileName":"importance_analysis.py","languages":[{"code":"$68","label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"}],"titleType":"number","titleNumber":8},{"id":"summary","title":"Summary: Impurity-based Importance","content":[{"data":{"content":"We've comprehensively examined impurity-based importance—the most common method for quantifying feature significance in tree-based models. Let's consolidate the key concepts:"},"type":"text"},{"data":{"type":"ordered","items":["**Impurity-based importance (MDI)** measures feature importance as the total weighted reduction in impurity across all splits using that feature.","**Gini impurity and entropy** are the primary measures for classification; **variance** is used for regression.","**The weighting factor** ($N_t$) ensures splits affecting more samples contribute proportionally more to importance.","**Random Forest importance** averages single-tree importances, providing more stable estimates through variance reduction.","**Key bias:** Features with more splitting opportunities (high cardinality) receive artificially inflated importance.","**Normalization** makes importances sum to 1, enabling interpretation as proportional contributions.","**Zero importance does not imply irrelevance**—redundant features may be predictive but never selected."],"title":{"icon":"CheckCircle","text":"Core Concepts","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"},{"data":{"content":"**What's next:**\n\nWhile impurity-based importance is convenient and fast, we've identified its fundamental limitation: it measures training-time behavior rather than true predictive contribution. The next page introduces **Permutation Importance**—a method that directly measures how each feature affects model predictions on held-out data, providing a more accurate picture of feature relevance for generalization."},"type":"text"},{"data":{"icon":"CheckCircle","title":"Page Complete","content":"You now understand the mathematical foundation, computation process, and practical considerations for impurity-based feature importance. You can correctly interpret MDI values, recognize their biases, and apply best practices for reliable feature analysis.","variant":"success"},"type":"callout"}],"titleType":"number","titleNumber":9}],"isLocked":false},{"id":"seed_02f98b069cefc31e4c0b227570f23b0b","title":"Permutation Importance","order":1,"content":null,"isLocked":true},{"id":"seed_a3d2cc9f29a4b681486ab2e469a5c07f","title":"Drop-Column Importance","order":2,"content":null,"isLocked":true},{"id":"seed_b55d0c9d2b705a96995dc4079b9b43a2","title":"Bias in Importance","order":3,"content":null,"isLocked":true},{"id":"seed_274a631bff1b01c8032fca70fb006813","title":"Interpretation Guidelines","order":4,"content":null,"isLocked":true}]}},"dataUpdateCount":1,"dataUpdatedAt":1792317794425,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"success","fetchStatus":"idle"},"queryKey":[["learningContent","getModule"],{"input":{"slug":"feature-importance"},"type":"query"}],"queryHash":"[[\"learningContent\",\"getModule\"],{\"input\":{\"slug\":\"feature-importance\"},\"type\":\"query\"}]"},{"dehydratedAt":1792317794434,"state":{"data":{"json":null},"dataUpdateCount":1,"dataUpdatedAt":1792317794431,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"success","fetchStatus":"idle"},"queryKey":[["learningContent","getProgress"],{"input":{"moduleId":"seed_650cf74b6e81d44f39e80cc287f6b75b"},"type":"query"}],"queryHash":"[[\"learningContent\",\"getProgress\"],{\"input\":{\"moduleId\":\"seed_650cf74b6e81d44f39e80cc287f6b75b\"},\"type\":\"query\"}]"},{"dehydratedAt":1792317794434,"state":{"data":{"json":null},"dataUpdateCount":1,"dataUpdatedAt":1792317794432,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"success","fetchStatus":"idle"},"queryKey":[["feedback","get"],{"input":{"contentType":"LEARNING_PAGE","contentId":"seed_02f98b069cefc31e4c0b227570f23b0b"},"type":"query"}],"queryHash":"[[\"feedback\",\"get\"],{\"input\":{\"contentId\":\"seed_02f98b069cefc31e4c0b227570f23b0b\",\"contentType\":\"LEARNING_PAGE\"},\"type\":\"query\"}]"},{"dehydratedAt":1792317794434,"state":{"data":{"json":null},"dataUpdateCount":1,"dataUpdatedAt":1792317794432,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"success","fetchStatus":"idle"},"queryKey":[["feedback","getUserLike"],{"input":{"contentType":"LEARNING_PAGE","contentId":"seed_02f98b069cefc31e4c0b227570f23b0b"},"type":"query"}],"queryHash":"[[\"feedback\",\"getUserLike\"],{\"input\":{\"contentId\":\"seed_02f98b069cefc31e4c0b227570f23b0b\",\"contentType\":\"LEARNING_PAGE\"},\"type\":\"query\"}]"},{"dehydratedAt":1792317794434,"state":{"data":{"json":[]},"dataUpdateCount":1,"dataUpdatedAt":1792317794432,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"success","fetchStatus":"idle"},"queryKey":[["highlights","getHighlights"],{"input":{"contentType":"LEARNING_PAGE","contentId":"seed_02f98b069cefc31e4c0b227570f23b0b"},"type":"query"}],"queryHash":"[[\"highlights\",\"getHighlights\"],{\"input\":{\"contentId\":\"seed_02f98b069cefc31e4c0b227570f23b0b\",\"contentType\":\"LEARNING_PAGE\"},\"type\":\"query\"}]"},{"dehydratedAt":1792317794434,"state":{"data":"$undefined","dataUpdateCount":0,"dataUpdatedAt":0,"error":null,"errorUpdateCount":0,"errorUpdatedAt":0,"fetchFailureCount":0,"fetchFailureReason":null,"fetchMeta":null,"isInvalidated":false,"status":"pending","fetchStatus":"fetching"},"queryKey":[["highlights","getPopularHighlights"],{"input":{"contentType":"LEARNING_PAGE","contentId":"seed_02f98b069cefc31e4c0b227570f23b0b","minCount":2},"type":"query"}],"queryHash":"[[\"highlights\",\"getPopularHighlights\"],{\"input\":{\"contentId\":\"seed_02f98b069cefc31e4c0b227570f23b0b\",\"contentType\":\"LEARNING_PAGE\",\"minCount\":2},\"type\":\"query\"}]","promise":"$@69"}]},"children":"$L6a"}]