79:T411,In polynomial regression, we expand a single predictor $x$ into a polynomial basis $(1, x, x^2, \ldots, x^d)$ and fit a linear model in these transformed features. This approach has a fundamental limitation: **polynomial bases are global**. Each basis function spans the entire range of $x$, meaning that the fitted value at any point depends on all the data—including data far away from that point. This global nature creates two severe problems: 1. **Runge's phenomenon**: High-degree polynomials oscillate wildly near the boundaries, producing poor fits despite low training error. 2. **Numerical instability**: The Vandermonde matrix $(x_i^j)$ becomes extremely ill-conditioned as degree increases, causing coefficient estimates to explode. Splines solve both problems by replacing global polynomial bases with **piecewise polynomial bases** that have **local support**—each basis function is non-zero only over a finite interval. The resulting fits adapt locally to the data while maintaining smoothness across the entire domain.7a:T433,import numpy as np import matplotlib.pyplot as plt def truncated_power(x, xi, d): """Compute (x - xi)_+^d""" return np.maximum(0, x - xi) ** d def build_truncated_power_design(x, knots, degree): """ Build design matrix for truncated power basis. For degree d with K knots: columns are [1, x, x^2, ..., x^d, (x-xi_1)_+^d, ..., (x-xi_K)_+^d] Total columns: d + 1 + K """ n = len(x) K = len(knots) # Polynomial part: 1, x, x^2, ..., x^d X_poly = np.column_stack([x ** j for j in range(degree + 1)]) # Truncated power part: (x - xi_j)_+^d for each knot X_trunc = np.column_stack([truncated_power(x, xi, degree) for xi in knots]) return np.hstack([X_poly, X_trunc]) # Example: Cubic splines with knots at 0.25, 0.5, 0.75 x = np.linspace(0, 1, 200) knots = [0.25, 0.5, 0.75] degree = 3 X = build_truncated_power_design(x, knots, degree) print(f"Design matrix shape: {X.shape}") # (200, 7) = 3+1 + 3 # Condition number reveals the problem! print(f"Condition number: {np.linalg.cond(X):.2e}")7b:T81a,import numpy as np def cox_de_boor(x, j, k, t): """ Compute B_{j,k}(x) using the Cox-de Boor recurrence. Parameters: ----------- x : float Evaluation point j : int Index of B-spline (0-indexed) k : int Degree of B-spline t : array Extended knot sequence Returns: -------- float : Value of B_{j,k}(x) """ # Base case: degree 0 (indicator function) if k == 0: if t[j] <= x < t[j + 1]: return 1.0 else: return 0.0 # Recursive case: degree k # Handle 0/0 convention: if denominator is 0, that term is 0 # First term: (x - t_j) / (t_{j+k} - t_j) * B_{j,k-1}(x) denom1 = t[j + k] - t[j] if denom1 == 0: term1 = 0.0 else: term1 = ((x - t[j]) / denom1) * cox_de_boor(x, j, k - 1, t) # Second term: (t_{j+k+1} - x) / (t_{j+k+1} - t_{j+1}) * B_{j+1,k-1}(x) denom2 = t[j + k + 1] - t[j + 1] if denom2 == 0: term2 = 0.0 else: term2 = ((t[j + k + 1] - x) / denom2) * cox_de_boor(x, j + 1, k - 1, t) return term1 + term2 def build_extended_knots(interior_knots, a, b, degree): """ Build extended knot sequence with repeated boundary knots. Parameters: ----------- interior_knots : array Interior knots xi_1, ..., xi_K a, b : float Domain boundaries degree : int Spline degree Returns: -------- array : Extended knot sequence """ left_boundary = [a] * (degree + 1) right_boundary = [b] * (degree + 1) return np.array(left_boundary + list(interior_knots) + right_boundary) # Example: Cubic B-splines (degree 3) with 3 interior knots interior_knots = [0.25, 0.5, 0.75] a, b = 0.0, 1.0 degree = 3 t = build_extended_knots(interior_knots, a, b, degree) print(f"Extended knot sequence ({len(t)} knots):") print(t) # Number of B-splines: n = len(t) - degree - 1 = K + degree + 1 n_basis = len(t) - degree - 1 print(f"Number of B-spline basis functions: {n_basis}")7c:T8f6,import numpy as np from scipy.interpolate import BSpline import matplotlib.pyplot as plt def build_bspline_design(x, interior_knots, degree, include_intercept=False): """ Build the B-spline design matrix using scipy. Parameters: ----------- x : array of shape (n,) Evaluation points interior_knots : array of shape (K,) Interior knots degree : int Spline degree (typically 3 for cubic) include_intercept : bool If True, add column of 1s (standard in statsmodels convention) Returns: -------- B : array of shape (n, K + degree + 1 + int(include_intercept)) B-spline design matrix """ # Build extended knot sequence t = np.concatenate([ np.repeat(x.min(), degree + 1), interior_knots, np.repeat(x.max(), degree + 1) ]) # Number of basis functions n_basis = len(t) - degree - 1 # Evaluate each basis function at all points B = np.zeros((len(x), n_basis)) for j in range(n_basis): # Create coefficient vector: 1 at position j, 0 elsewhere c = np.zeros(n_basis) c[j] = 1.0 # Create B-spline and evaluate spline = BSpline(t, c, degree, extrapolate=False) B[:, j] = spline(x) # Handle NaN at right boundary (open interval convention) B = np.nan_to_num(B, nan=0.0) if include_intercept: B = np.column_stack([np.ones(len(x)), B]) return B # Example: Generate data and fit B-spline regression np.random.seed(42) # True function: sinusoidal with noise n = 100 x = np.sort(np.random.uniform(0, 2*np.pi, n)) y_true = np.sin(x) + 0.5 * np.cos(2*x) y = y_true + np.random.normal(0, 0.3, n) # Define knots and build design matrix interior_knots = np.linspace(0.5, 2*np.pi - 0.5, 5) # 5 interior knots degree = 3 # Cubic B-splines B = build_bspline_design(x, interior_knots, degree) print(f"Design matrix shape: {B.shape}") # (100, 9) = 5 + 3 + 1 # Fit via OLS beta_hat = np.linalg.lstsq(B, y, rcond=None)[0] y_hat = B @ beta_hat # Check condition number - should be reasonable! print(f"Condition number: {np.linalg.cond(B):.2f}") # Compare with truncated power (import from earlier) # You'll find B-spline condition number is MUCH smaller7d:Tf72,import numpy as np from scipy.interpolate import BSpline, splrep import matplotlib.pyplot as plt # ===================================================== # STEP 1: Generate Synthetic Data # ===================================================== np.random.seed(42) # True function: combination of polynomial and periodic components def true_function(x): return 2 * np.sin(1.5 * x) + 0.5 * x - 0.02 * x**2 n = 150 x = np.sort(np.random.uniform(0, 10, n)) y_true = true_function(x) sigma = 0.8 y = y_true + np.random.normal(0, sigma, n) # ===================================================== # STEP 2: Define B-spline Basis Matrix Construction # ===================================================== def bspline_design_matrix(x, knots, degree): """Build design matrix for B-splines with given interior knots.""" # Extended knot sequence with repeated boundary knots t = np.concatenate([ np.repeat(x.min(), degree + 1), knots, np.repeat(x.max(), degree + 1) ]) n_basis = len(t) - degree - 1 n_points = len(x) B = np.zeros((n_points, n_basis)) for j in range(n_basis): c = np.zeros(n_basis) c[j] = 1.0 spl = BSpline(t, c, degree, extrapolate=False) B[:, j] = np.nan_to_num(spl(x), nan=0.0) return B, t # ===================================================== # STEP 3: Fit Models with Different Numbers of Knots # ===================================================== results = {} knot_counts = [3, 6, 10, 20] # Interior knots degree = 3 # Cubic splines x_grid = np.linspace(x.min(), x.max(), 500) for K in knot_counts: # Place knots at quantiles for uniform data density knots = np.percentile(x, np.linspace(0, 100, K + 2)[1:-1]) # Build design matrices for data and prediction grid B_train, t = bspline_design_matrix(x, knots, degree) B_pred, _ = bspline_design_matrix(x_grid, knots, degree) # Fit via OLS beta_hat = np.linalg.lstsq(B_train, y, rcond=None)[0] # Predictions y_hat_train = B_train @ beta_hat y_hat_pred = B_pred @ beta_hat # Compute metrics residuals = y - y_hat_train rss = np.sum(residuals ** 2) tss = np.sum((y - y.mean()) ** 2) r_squared = 1 - rss / tss # Degrees of freedom = number of basis functions df = B_train.shape[1] # Adjusted R² and AIC for model comparison n = len(y) adj_r_squared = 1 - (1 - r_squared) * (n - 1) / (n - df - 1) log_likelihood = -n/2 * np.log(2*np.pi) - n/2 * np.log(rss/n) - n/2 aic = 2 * df - 2 * log_likelihood results[K] = { 'knots': knots, 'beta': beta_hat, 'y_pred': y_hat_pred, 'r_squared': r_squared, 'adj_r_squared': adj_r_squared, 'aic': aic, 'df': df, 'condition': np.linalg.cond(B_train) } print(f"K={K:2d} knots: df={df:2d}, R²={r_squared:.4f}, " f"Adj R²={adj_r_squared:.4f}, AIC={aic:.1f}, " f"Cond={results[K]['condition']:.1f}") # ===================================================== # STEP 4: Visualize Results # ===================================================== fig, axes = plt.subplots(2, 2, figsize=(14, 10)) axes = axes.flatten() for ax, K in zip(axes, knot_counts): ax.scatter(x, y, alpha=0.5, s=20, label='Data') ax.plot(x_grid, true_function(x_grid), 'g--', lw=2, label='True function') ax.plot(x_grid, results[K]['y_pred'], 'r-', lw=2, label=f'B-spline fit (K={K})') # Mark knot positions for xi in results[K]['knots']: ax.axvline(xi, color='gray', linestyle=':', alpha=0.5, lw=0.8) ax.set_xlabel('x') ax.set_ylabel('y') ax.set_title(f'{K} Interior Knots: df={results[K]["df"]}, ' f'R²={results[K]["r_squared"]:.3f}') ax.legend(loc='upper right', fontsize=9) plt.tight_layout() plt.savefig('bspline_regression_comparison.png', dpi=150) plt.show()7e:T876,import numpy as np def de_boor(x, t, c, degree): """ Evaluate B-spline curve using de Boor's algorithm. Parameters: ----------- x : float Evaluation point t : array Knot sequence (extended) c : array Control points / coefficients degree : int Spline degree Returns: -------- float : Value of spline at x """ d = degree # Find the knot interval [t_r, t_{r+1}) containing x # The point x should be in [t[d], t[len(t)-d-1]) r = np.searchsorted(t, x, side='right') - 1 # Clamp to valid range r = max(d, min(r, len(t) - d - 2)) # Initialize: extract relevant coefficients # d_j = c[r-d+j] for j = 0, ..., d d_arr = np.array([c[r - d + j] for j in range(d + 1)], dtype=float) # Iterate: perform d rounds of convex combinations for k in range(1, d + 1): for j in range(d, k - 1, -1): # Right to left to allow in-place update # alpha = (x - t[r-d+k+j]) / (t[r+1+j] - t[r-d+k+j]) # But j is 0-indexed, so adjust: left_idx = r - d + j + k right_idx = r + 1 + j if t[right_idx] == t[left_idx]: alpha = 0.0 else: alpha = (x - t[left_idx]) / (t[right_idx] - t[left_idx]) d_arr[j] = (1 - alpha) * d_arr[j - 1] + alpha * d_arr[j] return d_arr[d] def evaluate_bspline_curve(x_vals, t, c, degree): """Vectorized evaluation of B-spline curve at multiple points.""" return np.array([de_boor(x, t, c, degree) for x in x_vals]) # Example: Verify de Boor matches scipy.BSpline from scipy.interpolate import BSpline # Set up a cubic spline with known coefficients t = np.array([0, 0, 0, 0, 0.3, 0.5, 0.7, 1, 1, 1, 1], dtype=float) c = np.array([1.0, 2.0, 1.5, 3.0, 2.5, 1.0, 2.0]) degree = 3 x_test = np.linspace(0, 1, 20) # scipy reference spl = BSpline(t, c, degree) y_scipy = spl(x_test) # Our implementation y_deboor = evaluate_bspline_curve(x_test, t, c, degree) # Check agreement print("Max difference from scipy:", np.max(np.abs(y_scipy - y_deboor)))71:{"json":{"id":"seed_ab6a1258d47178510b97446914ff6a5f","slug":"spline-regression","category":"Machine Learning","topic":"Spline Regression","title":"Spline Regression Methods","description":"Master the theory and practice of spline regression, from fundamental B-spline construction through natural splines, smoothing splines, and penalized spline methods. Learn to model nonlinear relationships with flexible, interpretable basis function expansions.","difficulty":"INTERMEDIATE","estimatedMinutes":90,"pages":[{"id":"seed_baf710dedf209853358c5a0b82fd849c","title":"Basis Splines (B-splines): The Foundation of Modern Spline Regression","order":0,"content":[{"id":"opening-motivation","title":"Beyond Polynomial Regression: The Need for Splines","content":[{"data":{"content":"$79"},"type":"text"},{"data":{"icon":"Target","title":"What You Will Learn","content":"By the end of this page, you will understand the mathematical construction of B-splines via the Cox-de Boor recurrence, their remarkable properties (local support, partition of unity, convex hull containment), and how to use them as basis functions for flexible regression modeling. You will see why B-splines are the preferred representation for spline regression across statistics, machine learning, and computer graphics.","variant":"info"},"type":"callout"}]},{"id":"what-is-a-spline","title":"What is a Spline?","content":[{"data":{"content":"Before diving into B-splines specifically, we need to understand what a spline is in general.\n\n**Definition (Spline Function):**\n\nA *spline of degree $d$* on the interval $[a, b]$ with *knots* $a = \\xi_0 < \\xi_1 < \\xi_2 < \\cdots < \\xi_K < \\xi_{K+1} = b$ is a function $s: [a, b] \\to \\mathbb{R}$ such that:\n\n1. On each subinterval $[\\xi_j, \\xi_{j+1})$, the function $s$ is a polynomial of degree at most $d$.\n2. The function $s$ has continuous derivatives up to order $d-1$ at each interior knot $\\xi_1, \\ldots, \\xi_K$.\n\nThe knots $\\xi_1, \\ldots, \\xi_K$ are the points where the polynomial pieces join. The smoothness requirement ensures that, despite being piecewise, the spline appears smooth to the eye—no visible corners or jumps up to the $(d-1)$-th derivative."},"type":"text"},{"data":{"icon":"BookOpen","title":"Historical Note: The Physical Spline","content":"The term 'spline' comes from drafting. Before computers, engineers and ship designers used thin flexible strips of wood or metal (called splines) held in place by heavy lead weights (called 'ducks'). The strip naturally assumed the shape that minimized bending energy—which turns out to be a cubic spline! This physical origin explains why cubic splines (degree 3) are the most common: they minimize curvature while allowing sufficient flexibility.","variant":"note"},"type":"callout"},{"data":{"content":"**Degrees of Freedom of a Spline Space:**\n\nHow many parameters are needed to specify a spline of degree $d$ with $K$ interior knots?\n\nConsider: We have $K+1$ subintervals, each carrying a polynomial of degree $d$, which has $d+1$ coefficients. This gives $(K+1)(d+1)$ free parameters. However, at each of the $K$ interior knots, we impose $d$ continuity conditions (continuity of the function itself and its first $d-1$ derivatives). Thus:\n\n$$\\dim(\\text{Spline Space}) = (K+1)(d+1) - Kd = K + d + 1$$\n\nThis is a fundamental formula. For **cubic splines** ($d = 3$) with $K$ interior knots, the dimension is $K + 4$. This matches: we need to specify $K$ knot positions plus 4 boundary conditions (or equivalently, choose from a $(K+4)$-dimensional basis)."},"type":"text"},{"data":{"rows":[["0","Step function","$$C^{-1}$ (discontinuous)","$$K$","$$K + 1$"],["1","Linear spline","$$C^0$ (continuous)","$$K$","$$K + 2$"],["2","Quadratic spline","$$C^1$ (smooth)","$$K$","$$K + 3$"],["3","Cubic spline","$$C^2$ (very smooth)","$$K$","$$K + 4$"],["$$d$","Degree-$d$ spline","$$C^{d-1}$","$$K$","$$K + d + 1$"]],"caption":"Spline Space Dimensions for Common Cases","headers":["Degree $d$","Name","Continuity","Interior Knots $K$","Dimension"]},"type":"table"},{"data":{"content":"**The Basis Function Approach:**\n\nSince the space of splines of degree $d$ with $K$ knots has dimension $K + d + 1$, we can represent any such spline as a linear combination of $K + d + 1$ basis functions:\n\n$$s(x) = \\sum_{j=1}^{K+d+1} \\beta_j B_j(x)$$\n\nThe question is: what basis should we use? The most natural-seeming basis—truncated power functions—turns out to be poorly suited for computation. Enter B-splines."},"type":"text"}],"titleType":"number","titleNumber":1},{"id":"truncated-power-basis","title":"The Truncated Power Basis (And Why We Avoid It)","content":[{"data":{"content":"The most straightforward basis for the space of degree-$d$ splines with knots $\\xi_1, \\ldots, \\xi_K$ is the **truncated power basis**:\n\n$$\\{1, x, x^2, \\ldots, x^d, (x - \\xi_1)_+^d, (x - \\xi_2)_+^d, \\ldots, (x - \\xi_K)_+^d\\}$$\n\nwhere the *truncated power function* is defined as:\n\n$$(x - \\xi)_+^d = \\begin{cases} (x - \\xi)^d & \\text{if } x > \\xi \\\\ 0 & \\text{if } x \\leq \\xi \\end{cases}$$\n\nThis basis is intuitive: the first $d+1$ functions are the standard polynomial basis, and each additional function 'kicks in' at its corresponding knot to allow a new polynomial piece."},"type":"text"},{"data":{"code":"$7a","fileName":"truncated_power_basis.py","languages":[{"label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"},{"data":{"icon":"AlertCircle","title":"Numerical Disaster: Ill-Conditioning","content":"The truncated power basis suffers from severe numerical ill-conditioning. The condition number of the design matrix grows extremely large as the number of knots increases. In double precision arithmetic (about 16 significant digits), a condition number of $10^{12}$ means you lose 12 digits of accuracy in your coefficient estimates! This makes the truncated power basis essentially unusable for practical spline regression with more than a handful of knots.","variant":"warning"},"type":"callout"},{"data":{"content":"**Why the Ill-Conditioning?**\n\nThe problem is that truncated power functions are nearly collinear. Consider two adjacent knots $\\xi_j$ and $\\xi_{j+1}$. The functions $(x - \\xi_j)_+^d$ and $(x - \\xi_{j+1})_+^d$ are highly correlated—both are zero below their respective knots, and both grow polynomially above them, differing only by a slight phase shift.\n\nMathematically, if the knots are closely spaced, the truncated power functions become nearly linearly dependent, causing the design matrix to be nearly singular. The closer the knots, the worse the conditioning.\n\n**The Solution: B-Splines**\n\nB-splines provide an alternative basis for the exact same spline space that has excellent numerical properties. Instead of functions that 'switch on' at knots, B-splines are bell-shaped functions with **compact support**—each is non-zero only over a finite interval spanning a few adjacent knots."},"type":"text"}],"titleType":"number","titleNumber":2},{"id":"bspline-definition","title":"B-spline Definition: The Cox-de Boor Recurrence","content":[{"data":{"content":"B-splines (Basis splines) are a numerically stable basis for the spline function space. Unlike the truncated power basis, B-splines have **compact support** and are defined via an elegant recursive formula.\n\n**Extended Knot Sequence:**\n\nTo construct B-splines of degree $d$ with $K$ interior knots $\\xi_1 < \\cdots < \\xi_K$ on $[a, b]$, we first define an **extended knot sequence** by adding $d+1$ copies of the boundary values:\n\n$$\\underbrace{t_0 = t_1 = \\cdots = t_d = a}_{d+1 \\text{ copies}}, \\quad t_{d+1} = \\xi_1, \\quad t_{d+2} = \\xi_2, \\quad \\ldots, \\quad t_{d+K} = \\xi_K, \\quad \\underbrace{t_{d+K+1} = \\cdots = t_{2d+K+1} = b}_{d+1 \\text{ copies}}$$\n\nThis extended sequence has $2d + K + 2$ knots in total. The repeated boundary knots ensure that the B-splines interpolate exactly at the endpoints."},"type":"text"},{"data":{"icon":"Lightbulb","title":"Why Repeat Boundary Knots?","content":"Repeating the boundary knots $d+1$ times is crucial. This multiplicity causes the B-splines to sum exactly to 1 over the entire interval $[a,b]$ (the partition of unity property) and ensures that the spline passes through the control points at the boundaries. Without this convention, B-splines would not form a valid basis for the natural spline space.","variant":"tip"},"type":"callout"},{"data":{"content":"**The Cox-de Boor Recurrence Formula:**\n\nB-splines are defined recursively. For notational convenience, let $B_{j,k}(x)$ denote the $j$-th B-spline of degree $k$ (so $k$ goes from 0 to $d$).\n\n**Base case (degree 0):**\n$$B_{j,0}(x) = \\begin{cases} 1 & \\text{if } t_j \\leq x < t_{j+1} \\\\ 0 & \\text{otherwise} \\end{cases}$$\n\nThis is simply the indicator function for the interval $[t_j, t_{j+1})$—a rectangular pulse.\n\n**Recursive case (degree $k \\geq 1$):**\n$$B_{j,k}(x) = \\frac{x - t_j}{t_{j+k} - t_j} B_{j,k-1}(x) + \\frac{t_{j+k+1} - x}{t_{j+k+1} - t_{j+1}} B_{j+1,k-1}(x)$$\n\nHere, we adopt the convention that $0/0 = 0$ whenever a denominator vanishes (which happens when consecutive knots coincide).\n\nThis recurrence is sometimes also written as:\n$$B_{j,k}(x) = \\omega_{j,k}(x) \\cdot B_{j,k-1}(x) + \\big(1 - \\omega_{j+1,k}(x)\\big) \\cdot B_{j+1,k-1}(x)$$\n\nwhere $\\omega_{j,k}(x) = \\frac{x - t_j}{t_{j+k} - t_j}$ is a local linear interpolation weight."},"type":"text"},{"data":{"code":"$7b","fileName":"cox_de_boor_recursion.py","languages":[{"label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"},{"data":{"content":"**Understanding the Recurrence Geometrically:**\n\nThe Cox-de Boor formula has a beautiful geometric interpretation:\n\n1. Start with degree-0 B-splines: rectangular pulses on each knot interval\n2. At each level, each B-spline is a weighted average of two degree-$(k-1)$ B-splines\n3. The weights are linear functions that transition smoothly across the knot span\n4. Each recursion step adds a degree of smoothness\n\nBy the time we reach degree $d$, each B-spline $B_{j,d}(x)$ is:\n- Non-zero only on the interval $[t_j, t_{j+d+1}]$ (local support)\n- A piecewise polynomial of degree $d$\n- $C^{d-1}$ continuous everywhere (smooth)\n- Non-negative everywhere\n- Bell-shaped (single maximum)"},"type":"text"}],"titleType":"number","titleNumber":3},{"id":"bspline-properties","title":"Fundamental Properties of B-splines","content":[{"data":{"content":"B-splines possess a remarkable collection of properties that make them ideal for statistical modeling. These properties are not just mathematically elegant—they translate directly into computational and interpretive advantages."},"type":"text"},{"data":{"items":["**Local Support:** Each B-spline $B_{j,d}(x)$ is non-zero only on the interval $[t_j, t_{j+d+1}]$, spanning exactly $d+1$ knot intervals. This means changes to one coefficient affect only a local portion of the curve.","**Non-negativity:** $B_{j,d}(x) \\geq 0$ for all $x$. This is immediate from the recurrence: the base case is non-negative, and the recurrence preserves non-negativity.","**Partition of Unity:** $\\sum_{j=0}^{n-1} B_{j,d}(x) = 1$ for all $x \\in [a, b]$, where $n = K + d + 1$ is the number of B-splines. This ensures that the coefficients can be interpreted as weights.","**Convex Hull Property:** If $s(x) = \\sum_j \\beta_j B_{j,d}(x)$ and $L \\leq \\beta_j \\leq U$ for all $j$, then $L \\leq s(x) \\leq U$ for all $x$. The spline curve lies within the convex hull of its control coefficients.","**Smoothness:** Degree-$d$ B-splines are $C^{d-1}$ continuous at simple knots. At a knot of multiplicity $m$, continuity is reduced to $C^{d-m}$. This gives fine control over smoothness.","**Linear Independence:** The B-splines $\\{B_{j,d}\\}_{j=0}^{n-1}$ are linearly independent, forming a basis for the spline space."],"title":{"icon":"Sparkles","text":"Essential B-spline Properties","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"},{"data":{"content":"**Proof of Partition of Unity:**\n\nThe partition of unity property is crucial and follows from induction on degree. For degree 0, the B-splines are indicator functions of disjoint intervals, so exactly one equals 1 at any point.\n\nFor degree $k$, suppose the property holds for degree $k-1$. Then:\n$$\\sum_j B_{j,k}(x) = \\sum_j \\left[ \\omega_{j,k}(x) B_{j,k-1}(x) + (1-\\omega_{j+1,k}(x)) B_{j+1,k-1}(x) \\right]$$\n\nRe-indexing the second sum and using $\\omega_{j,k}(x) + (1-\\omega_{j,k}(x)) = 1$:\n$$= \\sum_j B_{j,k-1}(x) \\cdot \\left[ \\omega_{j,k}(x) + (1-\\omega_{j,k}(x)) \\right] = \\sum_j B_{j,k-1}(x) = 1$$\n\nby the induction hypothesis. The repeated boundary knots ensure this works at the domain boundaries."},"type":"text"},{"data":{"icon":"Lightbulb","title":"Why These Properties Matter for ML","content":"Local support means the design matrix for B-splines is banded (sparse), not dense. Partition of unity enables interpretable coefficients. Non-negativity with positive coefficients guarantees non-negative fitted values (useful for counts, densities). The convex hull property enables uncertainty visualization. These aren't abstract niceties—they're practical advantages.","variant":"tip"},"type":"callout"},{"data":{"rows":[["0","Discontinuous","1 interval","Rectangular pulse"],["1","$$C^0$ (continuous)","2 intervals","Triangular (tent function)"],["2","$$C^1$ (smooth)","3 intervals","Quadratic bump"],["3","$$C^2$ (very smooth)","4 intervals","Bell-shaped curve"],["$$d$","$$C^{d-1}$","$$d+1$ intervals","Smooth, bell-shaped"]],"caption":"B-spline Properties by Degree","headers":["Degree $d$","Continuity at Simple Knots","Support Width","Shape Characteristics"]},"type":"table"}],"titleType":"number","titleNumber":4},{"id":"bspline-in-practice","title":"Constructing the B-spline Design Matrix","content":[{"data":{"content":"For regression, we evaluate each B-spline basis function at each data point to form a **design matrix**. Given $n$ observations $x_1, \\ldots, x_n$ and $p = K + d + 1$ B-spline basis functions, the design matrix $\\mathbf{B}$ is:\n\n$$\\mathbf{B}_{i,j} = B_{j,d}(x_i) \\quad \\text{for } i = 1, \\ldots, n \\text{ and } j = 0, \\ldots, p-1$$\n\nThe spline regression model is then:\n$$y_i = \\sum_{j=0}^{p-1} \\beta_j B_{j,d}(x_i) + \\epsilon_i = \\mathbf{B}_i \\boldsymbol{\\beta} + \\epsilon_i$$\n\nIn matrix form: $\\mathbf{y} = \\mathbf{B} \\boldsymbol{\\beta} + \\boldsymbol{\\epsilon}$\n\nThis is simply linear regression with the B-spline design matrix! We can apply ordinary least squares, ridge regression, or any other linear model technique."},"type":"text"},{"data":{"code":"$7c","fileName":"bspline_design_matrix.py","languages":[{"label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"},{"data":{"icon":"CheckCircle","title":"Numerical Stability Achieved","content":"The condition number of the B-spline design matrix is typically on the order of $10^1$ to $10^2$ (small!), compared to $10^{12}$ or worse for the truncated power basis. This allows reliable coefficient estimation even with many knots. This is the primary computational reason to prefer B-splines.","variant":"success"},"type":"callout"},{"data":{"content":"**Using scipy.interpolate for B-splines:**\n\nIn practice, you should use optimized library implementations rather than the recursive formula. In Python, the `scipy.interpolate.BSpline` class provides efficient evaluation using a variant of the de Boor algorithm. For design matrix construction specifically, `patsy` and `statsmodels` provide the `bs()` function analogous to R's `splines::bs()` function:\n\n```python\nimport patsy\n\n# Create B-spline design with 5 interior knots, degree 3\nB = patsy.dmatrix(\"bs(x, df=8, degree=3, include_intercept=True) - 1\", \n data={\"x\": x})\n```\n\nThe `df` parameter specifies the degrees of freedom (= number of basis functions), which equals $K + d + 1$ for open-boundary B-splines."},"type":"text"}],"titleType":"number","titleNumber":5},{"id":"bspline-regression-example","title":"Complete B-spline Regression Example","content":[{"data":{"content":"Let's work through a complete example of B-spline regression, from data generation to model fitting to interpretation. We'll compare B-spline fits with different numbers of knots to illustrate the flexibility-smoothness tradeoff."},"type":"text"},{"data":{"code":"$7d","fileName":"bspline_regression_complete.py","languages":[{"label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"},{"data":{"content":"**Interpreting the Results:**"},"type":"text"},{"data":{"rows":[["3","7","0.89","0.88","Underfit: too rigid, misses curvature"],["6","10","0.92","0.91","Good balance: captures signal"],["10","14","0.93","0.92","Slightly more flexible"],["20","24","0.94","0.92","Overfit risk: wiggly, Adj R² stagnates"]],"headers":["Knots","df","R²","Adj R²","Interpretation"]},"type":"table"},{"data":{"content":"Note that $R^2$ always increases with more knots (more parameters = lower training error), but Adjusted $R^2$ penalizes complexity. Beyond a certain point, adding knots hurts generalization.\n\nThe **condition number** remains small (under 100) even with 20 knots—B-splines maintain numerical stability where the truncated power basis would have failed catastrophically."},"type":"text"},{"data":{"icon":"Info","title":"Knot Placement Strategy","content":"In this example, we placed knots at quantiles of the x values. This is a common default that ensures roughly equal numbers of observations between knots. Alternative strategies include equally-spaced knots (better for visualization) or adaptive knot placement based on residual structure (more advanced). We'll cover knot selection systematically in Page 4.","variant":"note"},"type":"callout"}],"titleType":"number","titleNumber":6},{"id":"de-boor-algorithm","title":"Efficient Evaluation: de Boor's Algorithm","content":[{"data":{"content":"The naive recursive evaluation of B-splines via Cox-de Boor is inefficient—it recomputes the same lower-degree B-splines multiple times. **de Boor's algorithm** provides a much faster approach for evaluating a B-spline curve $s(x) = \\sum_{j} \\beta_j B_{j,d}(x)$ at a given point $x$.\n\nThe key insight is that at any point $x$, at most $d+1$ B-splines are non-zero (due to local support). De Boor's algorithm exploits this by building up the value iteratively using only the non-zero B-splines.\n\n**Algorithm Sketch:**\n\n1. Find the knot interval containing $x$: find $r$ such that $t_r \\leq x < t_{r+1}$\n2. Initialize: $d_j^{[0]} = \\beta_{r-d+j}$ for $j = 0, \\ldots, d$ (the $d+1$ relevant coefficients)\n3. Iterate for $k = 1, \\ldots, d$:\n $$d_j^{[k]} = (1 - \\alpha_{j,k}) d_j^{[k-1]} + \\alpha_{j,k} d_{j+1}^{[k-1]}$$\n where $\\alpha_{j,k} = \\frac{x - t_{r-d+k+j}}{t_{r+1+j} - t_{r-d+k+j}}$\n4. Result: $s(x) = d_0^{[d]}$\n\nThis computes $s(x)$ in $O(d^2)$ operations, compared to $O(2^d)$ for naive recursion."},"type":"text"},{"data":{"code":"$7e","fileName":"de_boor_algorithm.py","languages":[{"label":"Python","value":"python"}],"showLineNumbers":true},"type":"code"},{"data":{"icon":"Lightbulb","title":"Practical Advice","content":"Unless you're implementing B-splines for a specialized system (embedded device, GPU kernel, etc.), use library implementations like scipy.interpolate.BSpline. These are highly optimized, handle edge cases correctly, and have been tested extensively. Understanding de Boor's algorithm is valuable for insight, but reinventing it is rarely necessary in ML practice.","variant":"tip"},"type":"callout"}],"titleType":"number","titleNumber":7},{"id":"connections-other-representations","title":"Connection to Other Spline Representations","content":[{"data":{"content":"B-splines are one of several ways to represent spline functions. Understanding the relationships between representations clarifies when each is appropriate."},"type":"text"},{"data":{"rows":[["Truncated Power","$$(1, x, \\ldots, x^d, (x-\\xi_j)_+^d)$","Intuitive, easy derivation","Numerically unstable"],["B-spline","Cox-de Boor basis","Numerically stable, local support","Complex construction"],["Natural Spline","Constrained B-spline","Reduces df by 4 for cubics","Boundary assumptions"],["Cardinal Spline","Interpolating basis","Direct knot value control","Less smooth at boundaries"],["Hermite","Value + derivative basis","Explicit derivative control","More parameters needed"]],"caption":"Comparison of Spline Representations","headers":["Representation","Basis Functions","Pros","Cons"]},"type":"table"},{"data":{"content":"**Key Relationships:**\n\n1. **Truncated Power ↔ B-spline**: Same spline space, different basis. Any truncated power spline can be written as a B-spline and vice versa—just with different coefficients. B-splines are preferred computationally.\n\n2. **B-spline → Natural Spline**: Natural cubic splines are B-splines with additional constraints (zero second derivative at endpoints). We'll cover these in detail on Page 2.\n\n3. **B-spline Derivatives**: The derivative of a degree-$d$ B-spline is a degree-$(d-1)$ B-spline:\n$$\\frac{d}{dx} B_{j,d}(x) = d \\left( \\frac{B_{j,d-1}(x)}{t_{j+d} - t_j} - \\frac{B_{j+1,d-1}(x)}{t_{j+d+1} - t_{j+1}} \\right)$$\n\nThis is important for penalized splines (Page 5).\n\n4. **B-spline Integrals**: Integration of B-splines leads to higher-degree B-splines. This is useful for computing penalty matrices."},"type":"text"},{"data":{"icon":"BookOpen","title":"The Name 'Basis Spline'","content":"The 'B' in B-spline stands for 'Basis.' Carl de Boor, who developed much of modern B-spline theory, chose this name because B-splines form a particularly convenient basis for the space of spline functions. The name does not stand for 'Bell' (though they are bell-shaped) or any other word.","variant":"info"},"type":"callout"}],"titleType":"number","titleNumber":8},{"id":"summary-looking-ahead","title":"Summary: B-splines as the Foundation","content":[{"data":{"content":"We've covered substantial ground in establishing B-splines as the foundation for spline regression. Let's consolidate the key ideas:"},"type":"text"},{"data":{"type":"ordered","items":["**Splines are piecewise polynomials** joined at knots with smoothness constraints. A degree-$d$ spline is $C^{d-1}$ continuous at simple knots.","**The spline space has dimension $K + d + 1$**, where $K$ is the number of interior knots and $d$ is the degree.","**B-splines are the numerically stable choice** for representing splines. The truncated power basis, though intuitive, suffers from catastrophic ill-conditioning.","**B-splines are defined recursively** via the Cox-de Boor formula, starting from indicator functions (degree 0) and building up through weighted averages.","**Local support is the key property**: Each B-spline is non-zero only over $d+1$ knot intervals. This enables sparse design matrices and efficient computation.","**Partition of unity ensures interpretability**: B-splines sum to 1 everywhere, making coefficients act as local weights.","**de Boor's algorithm** enables efficient $O(d^2)$ evaluation of B-spline curves at any point.","**Spline regression is just linear regression** with a B-spline design matrix. All standard techniques (OLS, ridge, inference) apply directly."],"title":{"icon":"CheckCircle","text":"Key Takeaways","type":"icon","color":"primary","variant":"outline-circle"},"variant":"accent"},"type":"list"},{"data":{"content":"**What's Next:**\n\nWith B-splines as our foundation, we can now explore more specialized spline constructions:\n\n- **Page 2: Natural Splines** — Constrained cubic splines that reduce degrees of freedom and improve behavior at boundaries\n- **Page 3: Smoothing Splines** — An optimization-based approach that avoids explicit knot selection entirely\n- **Page 4: Knot Selection** — Strategies for choosing the number and placement of knots\n- **Page 5: Penalized Splines** — Combining dense knot placement with penalty terms for the ultimate in flexibility"},"type":"text"},{"data":{"icon":"CheckCircle","title":"Page Complete","content":"You now understand B-splines: their mathematical definition via the Cox-de Boor recurrence, their remarkable properties (local support, partition of unity, numerical stability), and their application to regression modeling. This foundation underlies all modern spline-based methods in machine learning and statistics.","variant":"success"},"type":"callout"}],"titleType":"number","titleNumber":9}],"isLocked":false},{"id":"seed_c77bcf9dc30bf73ac93dad1753e6c11f","title":"Natural Splines: Optimal Boundary Behavior","order":1,"content":null,"isLocked":true},{"id":"seed_9fb1e34712ba8dd5f1de915db3ca1d2c","title":"Smoothing Splines: The Optimization Framework","order":2,"content":null,"isLocked":true},{"id":"seed_b26995ab647e7d9151911fb299645954","title":"Knot Selection: Strategies for Optimal Spline Flexibility","order":3,"content":null,"isLocked":true},{"id":"seed_bd1a97174963a61038945c7aba819739","title":"Penalized Splines (P-splines): The Best of Both Worlds","order":4,"content":null,"isLocked":true}]}}