{ "site": "quants.wiki", "section": "Covariance and volatility estimation", "url": "https://quants.wiki/estimation/", "reviewed": "2026-08-27", "license": "CC BY 4.0", "changes": "https://quants.wiki/changes.json", "tables": [ { "title": "Marchenko-Pastur eigenvalue bounds and the condition number by N/T", "intro": "For N independent series of T observations each with unit variance, the eigenvalues of the sample correlation matrix do not converge to 1. They spread across the interval [(1-sqrt(c))^2, (1+sqrt(c))^2] with c = N/T, and the condition number of the matrix is the ratio of the endpoints. The true matrix is the identity, so every value below is pure estimation noise.", "columns": [ "c = N/T", "Smallest eigenvalue", "Largest eigenvalue", "Condition number", "Example" ], "rows": [ [ "0.05", "0.602786", "1.497214", "2.4838", "50 assets, 1000 observations" ], [ "0.10", "0.467544", "1.732456", "3.7054", "24 assets, 240 observations" ], [ "0.20", "0.305573", "2.094427", "6.8541", "50 assets, 250 observations" ], [ "0.50", "0.085786", "2.914214", "33.9706", "50 assets, 100 observations" ], [ "0.80", "0.011146", "3.588854", "321.9969", "200 assets, 250 observations" ], [ "0.90", "0.002633", "3.797367", "1441.9993", "225 assets, 250 observations" ], [ "0.95", "0.000641", "3.899359", "6081.9998", "238 assets, 250 observations" ], [ "0.99", "0.000025", "3.979975", "158402.0000", "248 assets, 250 observations" ], [ "1.00 and above", "0", "-", "infinite, matrix singular", "rank at most T-1" ] ] }, { "title": "Ten daily OHLC bars used for every volatility estimator below", "intro": "Ten consecutive daily bars, with a previous close of 100.00 before the first. All five estimators in this section are computed from exactly these numbers, annualised at 252 trading days, so their disagreement is a property of the estimators and not of the data.", "columns": [ "Day", "Open", "High", "Low", "Close", "ln(C/C prev)", "ln(H/L)", "ln(C/O)" ], "rows": [ [ "1", "100.30", "101.80", "99.90", "101.40", "+0.013903", "0.018840", "+0.010907" ], [ "2", "101.60", "102.50", "100.70", "100.90", "-0.004943", "0.017717", "-0.006914" ], [ "3", "100.50", "101.20", "99.10", "99.60", "-0.012968", "0.020969", "-0.008996" ], [ "4", "99.20", "100.40", "98.30", "98.80", "-0.008065", "0.021138", "-0.004040" ], [ "5", "98.50", "100.10", "98.20", "99.90", "+0.011072", "0.019163", "+0.014113" ], [ "6", "100.20", "101.90", "99.80", "101.50", "+0.015889", "0.020824", "+0.012891" ], [ "7", "101.80", "103.20", "101.30", "102.60", "+0.010779", "0.018582", "+0.007828" ], [ "8", "102.30", "102.90", "100.80", "101.10", "-0.014728", "0.020619", "-0.011800" ], [ "9", "100.90", "101.60", "99.60", "100.30", "-0.007944", "0.019881", "-0.005964" ], [ "10", "100.60", "102.40", "100.20", "102.10", "+0.017787", "0.021719", "+0.014800" ] ] }, { "title": "Six volatility estimators on the same ten bars", "intro": "All annualised at 252 and expressed in volatility points. The spread from lowest to highest is 1.41 points on identical data, which is the size of the definitional disagreement a reader should expect before any sampling error is considered.", "columns": [ "Estimator", "Annualised volatility, percent", "Uses", "Assumes" ], "rows": [ [ "Close-to-close, zero mean", "19.6906", "closes only", "zero drift; discards intraday information entirely" ], [ "Close-to-close, sample mean, divisor n-1", "20.4623", "closes only", "drift estimated from the same short window" ], [ "Parkinson 1980", "19.0522", "high and low", "zero drift, no opening jump, continuous monitoring of the extremes" ], [ "Garman-Klass 1980", "19.9407", "open, high, low, close", "zero drift, no opening jump; combines range and close-open" ], [ "Rogers-Satchell 1991", "19.5043", "open, high, low, close", "allows nonzero drift; still assumes no opening jump" ], [ "Yang-Zhang 2000", "19.8563", "open, high, low, close, previous close", "allows both drift and opening jumps; minimum-variance combination" ] ] }, { "title": "Precision of a volatility estimate by window length", "intro": "For iid normal returns the relative standard error of the sample standard deviation is approximately 1/sqrt(2n). The exact value is sqrt(1 - c4(n)^2) with c4 the gamma-function bias factor. Longer windows reduce this error and increase staleness bias when the true volatility is time-varying: that is the whole tradeoff, and it has no general solution.", "columns": [ "Window n", "1/sqrt(2n), approximate relative SE", "Exact SD(s)/sigma", "Relative SE, percent" ], "rows": [ [ "5", "0.316228", "0.341214", "34.12" ], [ "10", "0.223607", "0.232237", "23.22" ], [ "20", "0.158114", "0.161123", "16.11" ], [ "24 (base series)", "0.144338", "0.146621", "14.66" ], [ "60", "0.091287", "0.091861", "9.19" ], [ "120", "0.064550", "0.064752", "6.48" ], [ "252", "0.044544", "0.044610", "4.46" ], [ "504", "0.031497", "0.031520", "3.15" ] ] } ], "entries": [ { "id": "sample-covariance-conditioning", "term": "Why the sample covariance matrix fails as N approaches T", "definition": "The sample covariance matrix is an unbiased estimator of the true covariance matrix and is nonetheless useless in an optimiser when the number of assets is a material fraction of the number of observations. Unbiasedness of the matrix does not imply usability of its inverse, and it is the inverse the optimiser uses.", "formula": "S = (1/(T-1)) sum_t (r_t - rbar)(r_t - rbar)'. rank(S) <= min(N, T-1). S is singular whenever N >= T", "fields": [ { "label": "Rank", "value": "Demeaning costs one degree of freedom, so the rank of S is at most T-1. With N = 500 assets and T = 250 monthly observations the matrix has rank at most 249 and at least 251 zero eigenvalues, and no inverse exists", "mono": true }, { "label": "Worked, degrees of freedom", "value": "An N by N covariance matrix has N(N+1)/2 free parameters. At N = 100 that is 5,050 parameters. Estimating them from T = 240 monthly observations means 24,000 data points supporting 5,050 parameters, a ratio of 4.75 observations per parameter", "mono": true }, { "label": "Consequence for the inverse", "value": "Sigma inverse has eigenvalues that are the reciprocals of Sigma's. The smallest eigenvalue of S, which is the noisiest, becomes the largest eigenvalue of S inverse and dominates the optimiser's output", "mono": true }, { "label": "Where the optimiser puts the weight", "value": "The minimum-variance and tangency solutions both apply Sigma inverse to a vector, so they load most heavily on the direction of the smallest eigenvalue - the least reliably estimated direction in the data", "mono": true } ], "notes": [ "This is not a small-sample problem that goes away with more history. It is a ratio problem: doubling the asset count requires roughly quadrupling the observations to hold the parameter-per-observation ratio fixed, and history is finite and non-stationary.", "The commonly stated rule that T should exceed N is far too weak. At N/T = 0.5 - twice as many observations as assets - the condition number of a sample correlation matrix estimated from pure independent noise is already 34, meaning the optimiser sees a factor-of-34 spread in risk where the truth is a factor of 1.", "Adding assets to a portfolio makes the estimation problem worse faster than it makes the diversification better, past a point that depends entirely on T. This is the argument for factor models and for shrinkage, and it is quantitative rather than aesthetic." ], "source": "Marchenko and Pastur 1967 for the eigenvalue distribution" }, { "id": "marchenko-pastur", "term": "The Marchenko-Pastur bounds", "definition": "The limiting spectral distribution of a sample covariance matrix of independent unit-variance series. It gives the exact interval into which the sample eigenvalues spread when the truth is the identity matrix, which turns the question of how much of an eigenvalue spectrum is noise into arithmetic.", "formula": "With c = N/T fixed as both grow, the sample eigenvalues of the correlation matrix of independent unit-variance series fill [(1 - sqrt(c))^2, (1 + sqrt(c))^2]. Condition number = ((1 + sqrt(c))/(1 - sqrt(c)))^2", "fields": [ { "label": "Worked, c = 0.20", "value": "sqrt(0.20) = 0.447214. Lower edge (1 - 0.447214)^2 = 0.305573; upper edge (1 + 0.447214)^2 = 2.094427. Condition number 2.094427/0.305573 = 6.8541", "mono": true }, { "label": "Worked, c = 0.80", "value": "sqrt(0.80) = 0.894427. Edges 0.011146 and 3.588854. Condition number 321.9969 - and the true matrix is the identity, whose condition number is 1", "mono": true }, { "label": "Diagnostic use", "value": "Eigenvalues of a real correlation matrix that fall INSIDE the Marchenko-Pastur interval for the relevant c are indistinguishable from noise. Only those above the upper edge carry information under this null", "mono": true }, { "label": "Edge case", "value": "At c = 1 the lower edge is 0 exactly and the condition number is infinite. Beyond c = 1 a fraction 1 - 1/c of the eigenvalues are exactly zero", "mono": true } ], "notes": [ "The bounds are asymptotic in N and T jointly, so they are a guide rather than a test at small N. They are nonetheless the right null: they say what the eigenvalue spectrum of pure noise looks like, and most of a real equity correlation spectrum lies inside them.", "The result assumes the true covariance is the identity - independent series with equal variance. Real returns have a dominant market factor, so the largest empirical eigenvalue is usually far above the upper edge and is not noise. The bulk of the small eigenvalues typically is.", "Filtering by discarding eigenvalues inside the bulk and rebuilding the matrix is one form of regularisation. Shrinkage is another, and it is smoother because it does not require a discrete cut." ], "source": "Marchenko and Pastur 1967" }, { "id": "condition-number", "term": "Condition number, and what it costs", "definition": "The ratio of the largest to the smallest eigenvalue of a symmetric positive-definite matrix. It bounds how much a relative error in the inputs to a linear solve is amplified in the output, which is exactly the operation a mean-variance optimiser performs.", "formula": "kappa(Sigma) = lambda_max / lambda_min. Solving Sigma w = b, a relative perturbation of size eps in b produces a relative perturbation of at most kappa * eps in w", "fields": [ { "label": "Worked, three-asset matrix", "value": "Volatilities 16, 20 and 25 percent with correlations 0.60, 0.30 and 0.85. Eigenvalues of the covariance matrix: 0.10122125, 0.02324793, 0.00363082. kappa = 27.8783", "mono": true }, { "label": "Correlation matrix alone", "value": "Eigenvalues 2.195208, 0.720371, 0.084421, summing to 3 as they must. kappa = 26.0032", "mono": true }, { "label": "Amplification", "value": "At kappa = 27.88, a 1 percent relative error in the expected-return vector can produce a 27.88 percent relative error in the weight vector. At kappa = 322, typical of N/T = 0.8, it can produce a 322 percent error", "mono": true }, { "label": "Digits lost", "value": "A solve loses roughly log10(kappa) significant digits. kappa = 27.88 loses 1.4 digits; kappa = 158,402 at N/T = 0.99 loses 5.2 digits, which exceeds the precision of any return forecast", "mono": true } ], "notes": [ "The condition number is the single most useful diagnostic on a covariance matrix before it is inverted, and it is one line of code. A matrix with a condition number in the thousands should not be handed to an unconstrained optimiser regardless of how it was estimated.", "Constraints - long only, position limits, turnover limits - reduce the effective amplification by restricting the feasible set, which is why constrained optimisers are more stable than unconstrained ones even with a badly conditioned matrix. That is a robustness accident, not a solution.", "The condition number of the correlation matrix and of the covariance matrix differ, because the volatility scaling itself spreads the spectrum. Both are worth checking; the correlation one isolates the dependence structure." ] }, { "id": "correlation-matrix-psd-constraint", "term": "Pairwise correlations that cannot all be true at once", "definition": "A correlation matrix must be positive semi-definite. Correlations elicited pairwise - from three separate estimates, three separate desks, or three different sample windows - can be individually plausible and jointly impossible. For three assets the admissible range of the third correlation given the other two has a closed form.", "formula": "Given rho12 and rho13, positive semi-definiteness requires rho23 in [ rho12*rho13 - sqrt((1-rho12^2)(1-rho13^2)) , rho12*rho13 + sqrt((1-rho12^2)(1-rho13^2)) ]. Equivalently det = 1 + 2*rho12*rho13*rho23 - rho12^2 - rho13^2 - rho23^2 >= 0", "fields": [ { "label": "Worked, admissible range", "value": "rho12 = 0.60, rho13 = 0.30. Centre 0.60*0.30 = 0.180000. Half-width sqrt((1-0.36)(1-0.09)) = sqrt(0.64*0.91) = sqrt(0.5824) = 0.763151. Admissible rho23 in [-0.583151, 0.943151]", "mono": true }, { "label": "Determinant check", "value": "rho23 = 0.85: det = +0.13350000, valid. rho23 = 0.95: det = -0.01050000, invalid. rho23 = 0.98: det = -0.05760000, invalid", "mono": true }, { "label": "What invalid means", "value": "With rho12 = 0.60, rho13 = 0.30 and rho23 = 0.98 the eigenvalues are 2.297073, 0.736953 and -0.034026. The negative eigenvalue means there is a portfolio with negative variance - an arbitrage the optimiser will find and load into without limit", "mono": true }, { "label": "Minimum repair", "value": "Shrinking that matrix toward the identity restores positive semi-definiteness at delta > 0.03290611, the point where the smallest eigenvalue reaches exactly zero", "mono": true } ], "notes": [ "An unconstrained optimiser handed a matrix with a negative eigenvalue does not fail loudly. It returns a portfolio with an apparently negative variance and unbounded leverage in one direction, which looks like a spectacular result rather than an error.", "Estimating each pairwise correlation on the window where that pair has the most data - a common and reasonable-sounding practice - produces exactly this failure, because the pairs are then not estimated from a common sample. Estimate the whole matrix on a common window, or shrink afterwards.", "The three-asset closed form generalises only as the full positive-semi-definiteness condition; for N assets there is no simple per-entry admissible interval. Checking the smallest eigenvalue is the practical test at any N." ] }, { "id": "ledoit-wolf-shrinkage", "term": "Ledoit-Wolf shrinkage", "definition": "A convex combination of the sample covariance matrix and a structured target, with the weight chosen to minimise expected squared distance from the true matrix. The sample matrix is unbiased but high-variance; the target is biased but low-variance; the optimal blend has lower error than either.", "formula": "Sigma_hat = delta*F + (1-delta)*S, with F the shrinkage target. The optimal intensity is delta* = max(0, min(1, kappa/T)) with kappa = (pi - rho)/gamma, where pi is the sum of asymptotic variances of the entries of S, rho the sum of asymptotic covariances between the entries of F and of S, and gamma the squared Frobenius distance between F and the true matrix", "fields": [ { "label": "Common targets", "value": "Scaled identity F = (trace(S)/N) * I, from the 2004 well-conditioned estimator. Constant-correlation F, built from the sample volatilities and the average sample correlation, from the 2004 Honey I Shrunk paper. A single-factor model matrix, from Ledoit and Wolf 2003", "mono": true }, { "label": "Behaviour of delta*", "value": "delta* falls as T grows, since kappa/T shrinks, and rises with N since pi grows with the number of entries. In the limit of infinite T the shrinkage vanishes and the estimator reduces to S", "mono": true }, { "label": "Eigenvalue map, identity target", "value": "With F = m*I where m = trace(S)/N, the eigenvalues transform exactly as lambda_i -> (1-delta)*lambda_i + delta*m. Eigenvectors are unchanged", "mono": true }, { "label": "Worked", "value": "Correlation matrix with eigenvalues 2.297073, 0.736953, -0.034026 and m = 1. At delta = 0.20: 0.8*2.297073 + 0.2 = 2.037658; 0.8*0.736953 + 0.2 = 0.789563; 0.8*(-0.034026) + 0.2 = 0.172779. Condition number falls from negative-and-meaningless to 11.7934", "mono": true } ], "notes": [ "The shrinkage intensity is data-determined, not a tuning parameter, and that is the whole point of the result. Implementations that expose delta as a knob and default it to a round number have discarded the contribution and kept the form.", "Shrinkage toward the identity destroys the correlation structure in proportion to delta. Shrinking toward a constant-correlation or single-factor target preserves more of it at the same regularisation, and is preferable when the target is a defensible approximation of the truth.", "Shrinkage always biases the estimate. The claim is that it reduces total error, not that it reduces bias, and the claim holds only against the loss function the derivation minimises - expected squared Frobenius distance. If the downstream use is sensitive to a different functional of the matrix, the optimal delta for that use is different.", "Shrinkage cannot manufacture rank. It makes a singular matrix invertible by lifting the zero eigenvalues off zero, but the directions those eigenvalues span were never estimated from data and the shrunk matrix simply asserts a value for them." ], "source": "Ledoit and Wolf 2003 and 2004" }, { "id": "shrinkage-eigenvalue-map", "term": "Choosing shrinkage to hit a target condition number", "definition": "Because shrinkage toward a scaled identity acts on eigenvalues as an affine map, the intensity required to achieve any target condition number has a closed form. This gives a defensible alternative to a hand-picked delta when the goal is numerical stability rather than minimum Frobenius error.", "formula": "With eigenvalues lambda_1 >= ... >= lambda_N, target m and desired condition number k: delta = (lambda_1 - k*lambda_N) / ((lambda_1 - k*lambda_N) + m*(k-1)). The minimum delta for positive semi-definiteness is delta > -lambda_N/(m - lambda_N) when lambda_N < 0", "fields": [ { "label": "Worked inputs", "value": "Correlation matrix with eigenvalues 2.297073, 0.736953, -0.034026; m = 1 since a correlation matrix has trace N", "mono": true }, { "label": "Target k = 10", "value": "delta = (2.297073 - 10*(-0.034026))/((2.297073 + 0.340260) + 1*9) = 2.637333/11.637333 = 0.226627. Resulting eigenvalues 2.003121, 0.796567, 0.200312; condition number exactly 10.000000", "mono": true }, { "label": "Target k = 5", "value": "delta = 0.381494, eigenvalues 1.802247, 0.837304, 0.360449, condition number 5.000000", "mono": true }, { "label": "Minimum for validity", "value": "delta > 0.034026/(1 + 0.034026) = 0.03290611, at which the smallest eigenvalue is exactly zero and the matrix is positive semi-definite but still singular", "mono": true } ], "notes": [ "This is regularisation for a numerical objective, not for a statistical one. It will produce a well-conditioned matrix from arbitrarily bad inputs, which is exactly the risk: a stable optimiser fed a fabricated matrix returns a stable, fabricated portfolio.", "The affine eigenvalue map holds only for the scaled-identity target. For a constant-correlation or factor target the eigenvectors change too and there is no closed form for delta as a function of a target condition number.", "Where both are available, prefer the Ledoit-Wolf data-determined delta and check the resulting condition number as a diagnostic. Use the closed form above only when the resulting condition number is unacceptable, and record that the choice was made on numerical grounds." ] }, { "id": "ewma-variance", "term": "EWMA variance", "definition": "An exponentially weighted moving average of squared returns. It has one parameter, no long-run mean, and no estimation step, which makes it the cheapest defensible volatility forecast and also the least informative about mean reversion.", "formula": "sigma2_t = lambda * sigma2_{t-1} + (1 - lambda) * r_{t-1}^2, with 0 < lambda < 1. Weight on the observation k periods back is (1-lambda)*lambda^(k-1). Half-life = ln(0.5)/ln(lambda). Centre of mass = lambda/(1-lambda)", "fields": [ { "label": "Worked update", "value": "sigma2_{t-1} = 0.000400, so sigma_{t-1} = 2.0000 percent. r_{t-1} = -3.60 percent. At lambda = 0.94: sigma2_t = 0.94*0.000400 + 0.06*0.001296 = 0.000376 + 0.00007776 = 0.00045376, so sigma_t = 2.130164 percent", "mono": true }, { "label": "Same update, lambda = 0.97", "value": "0.97*0.000400 + 0.03*0.001296 = 0.000388 + 0.00003888 = 0.00042688, sigma_t = 2.066107 percent. The single 3.60 percent move moves the forecast by 13 basis points at lambda 0.94 and 7 at lambda 0.97", "mono": true }, { "label": "Half-life and mass", "value": "lambda 0.94: half-life 11.2023 periods, centre of mass 15.6667, newest weight 0.0600, first 20 weights sum to 0.709894. lambda 0.97: half-life 22.7566, centre of mass 32.3333, newest weight 0.0300, first 20 weights sum to 0.456206. lambda 0.99: half-life 68.9676, centre of mass 99.0000", "mono": true }, { "label": "Conventional values", "value": "lambda = 0.94 for daily data and 0.97 for monthly data are the RiskMetrics 1996 values, fitted to a specific dataset in that document and not a general result", "mono": true } ], "notes": [ "EWMA has no long-run variance to revert to, so its multi-period forecast is flat: the k-step-ahead forecast equals the one-step-ahead forecast for every k. That is a strong and usually false statement about term structure, and it is the main reason to prefer GARCH for anything beyond a one-period horizon.", "The recursion needs an initial value, and at lambda = 0.99 the initialisation still carries 82 percent of the weight after 20 periods. A short backtest of an EWMA model is measuring its seed as much as its data.", "Because the weights sum to 1 only in the infinite limit, a truncated implementation is biased downward unless the weights are renormalised. At lambda 0.94 and a 20-period truncation the weights sum to 0.709894, understating variance by 29 percent if left unrenormalised.", "Using r^2 rather than (r - mu)^2 assumes zero mean. Over daily horizons that is harmless; over monthly horizons with a strong drift it is not, and the estimator then includes the squared mean in the variance." ], "source": "RiskMetrics Technical Document, J.P. Morgan and Reuters 1996" }, { "id": "garch-11", "term": "GARCH(1,1)", "definition": "Variance as a weighted sum of a long-run level, the last squared return, and the last variance. It is EWMA with a constant added, and the constant is what gives it a long-run mean and a term structure.", "formula": "sigma2_t = omega + alpha*r_{t-1}^2 + beta*sigma2_{t-1}. Constraints: omega > 0, alpha >= 0, beta >= 0, and alpha + beta < 1 for covariance stationarity. Long-run variance = omega/(1 - alpha - beta). Persistence half-life = ln(0.5)/ln(alpha + beta)", "fields": [ { "label": "Worked parameters", "value": "omega = 0.000020, alpha = 0.080, beta = 0.900, so alpha + beta = 0.9800. Long-run variance = 0.000020/0.020 = 0.00100000, long-run volatility 3.162278 percent per period, 50.1996 percent annualised at 252", "mono": true }, { "label": "Persistence", "value": "Half-life = ln(0.5)/ln(0.98) = 34.3096 periods. A shock to variance takes 34 periods to decay halfway to the long-run level", "mono": true }, { "label": "Worked recursion", "value": "From sigma2 = 0.000400 and r = -3.60 percent: step 1 gives 0.000020 + 0.08*0.001296 + 0.9*0.000400 = 0.00048368 (2.1993 percent). Subsequent steps with no new return use sigma2_{t+1} = omega + (alpha+beta)*sigma2_t: 0.00049401, 0.00050413, 0.00051404, 0.00052376", "mono": true }, { "label": "Term structure", "value": "k-step forecast = LR + (alpha+beta)^(k-1) * (sigma2_{t+1} - LR). From the state above: k=1 gives 2.1993 percent, k=5 gives 2.2886, k=20 gives 2.5461, k=60 gives 2.9038, k=250 gives 3.1569, converging to the long-run 3.1623", "mono": true } ], "notes": [ "alpha + beta must be strictly below 1. At exactly 1 the process is IGARCH: no long-run variance exists, shocks are permanent, and the unconditional variance is infinite. Fitted equity GARCH models routinely return alpha + beta above 0.98, which means the long-run variance implied by the fit is a large extrapolation from a nearly non-stationary process.", "omega is the parameter that carries the long-run level and it is the hardest to estimate, because it is identified only by the slow mean reversion in the sample. A fit on two years of data pins alpha and beta and effectively guesses omega, and every long-horizon forecast is a function of omega.", "EWMA is the special case omega = 0, alpha = 1 - lambda, beta = lambda, so alpha + beta = 1 exactly. That is why EWMA has a flat term structure: it is a boundary case of GARCH with no mean to revert to.", "The one-step forecast of GARCH(1,1) and of a well-chosen EWMA are usually close. The models diverge at longer horizons, which is where the choice between them actually matters and where it is hardest to validate." ], "source": "Bollerslev 1986, generalising Engle 1982" }, { "id": "close-to-close-volatility", "term": "Close-to-close volatility, and its two divisors", "definition": "The standard deviation of log close-to-close returns, annualised. The zero-mean variant divides the sum of squares by n; the sample variant subtracts the mean and divides by n-1. On short windows they differ materially and the zero-mean version is the defensible one, because drift is not estimable over a short window.", "formula": "Zero mean: sigma = sqrt( (A/n) * sum_t ln(C_t/C_{t-1})^2 ). Sample mean: sigma = sqrt( A * sum_t (x_t - xbar)^2 / (n-1) ), with A the annualisation factor", "fields": [ { "label": "Worked, zero mean", "value": "Ten bars: sum of squared log returns = 0.0015385710. sigma = sqrt(252/10 * 0.0015385710) = sqrt(0.03877199) = 19.6906 percent", "mono": true }, { "label": "Worked, sample mean", "value": "Same data with the mean removed and divisor 9: sigma = 20.4623 percent. A difference of 0.77 volatility points on ten observations", "mono": true }, { "label": "Information discarded", "value": "Uses 10 of the 40 available prices. The intraday range, which on these bars averages 2.00 percent per day, contributes nothing", "mono": true }, { "label": "Annualisation factor", "value": "252 for trading-day returns, 365 for calendar-day returns, 12 for monthly. The factor is a convention and must be stated; the choice between 252 and 365 moves a volatility figure by 20 percent", "mono": true } ], "notes": [ "Close-to-close is the only one of the five estimators that captures overnight gaps in full, because a gap is entirely contained in the close-to-close return. That is its one advantage over the range estimators and it is not small for single names around events.", "It is also the least efficient: it uses one observation per day where the range estimators use two to four. On short windows the efficiency difference dominates every other consideration.", "A close-to-close volatility on a series with a strong trend and the sample-mean divisor removes the trend from the volatility. Whether that is desirable depends on whether the trend is expected to persist. State the choice." ] }, { "id": "parkinson-estimator", "term": "Parkinson estimator", "definition": "Volatility from the daily high-low range alone. The expected squared range of a driftless Brownian motion over one period is 4*ln(2) times its variance, which gives an unbiased variance estimator with a single constant.", "formula": "sigma^2 = (A / (4*ln(2)*n)) * sum_t ln(H_t/L_t)^2, with 1/(4*ln(2)) = 0.36067376", "fields": [ { "label": "Worked", "value": "Ten bars: sum of squared log ranges = 0.0039936811. sigma^2 = 252/10 * 0.36067376 * 0.0039936811 = 0.03629848. sigma = 19.0522 percent", "mono": true }, { "label": "Constant", "value": "4*ln(2) = 2.772589, so the reciprocal is 0.36067376. This is the only constant in the estimator and it comes from the expected range of a Brownian motion, not from a fit", "mono": true }, { "label": "Efficiency", "value": "Parkinson 1980 gives the theoretical efficiency gain over close-to-close as approximately 5 times under its stated assumptions - zero drift, no jumps, continuously observed extremes", "mono": true }, { "label": "Comparison", "value": "19.0522 percent against 19.6906 for close-to-close on the same bars, a difference of 0.64 volatility points", "mono": true } ], "notes": [ "The estimator is biased DOWNWARD on real data, always, for a reason that is structural rather than statistical: the true continuous high and low are never observed. A discretely sampled path has a smaller range than the continuous path it came from, so the observed range understates and so does the variance built from it.", "It ignores the open and the close entirely, which means it cannot see an overnight gap. On a stock that gaps 5 percent and then trades in a 1 percent range all day, Parkinson reports a quiet day.", "It assumes zero drift. A strong trend inflates the range without inflating the variance of the underlying diffusion, and Parkinson attributes all of it to volatility. Rogers-Satchell is the drift-tolerant alternative.", "Because the estimator uses only two prices per bar, the choice of what counts as the high and low - consolidated tape, primary exchange, regular hours only - changes the answer, and those definitions differ across data vendors." ], "source": "Parkinson 1980" }, { "id": "garman-klass-estimator", "term": "Garman-Klass estimator", "definition": "A combination of the high-low range and the close-open return, weighted to minimise variance among estimators built from those two quantities. More efficient than Parkinson and subject to the same two assumptions.", "formula": "sigma^2 = (A/n) * sum_t [ 0.5*ln(H_t/L_t)^2 - (2*ln(2) - 1)*ln(C_t/O_t)^2 ], with 2*ln(2) - 1 = 0.38629436", "fields": [ { "label": "Worked", "value": "Ten bars: sum of the bracketed terms = 0.0015779070. sigma^2 = 252/10 * 0.0015779070 = 0.03976326. sigma = 19.9407 percent", "mono": true }, { "label": "Constants", "value": "0.5 on the squared range and 0.38629436 = 2*ln(2) - 1 on the squared close-open. Both derive from the moments of a Brownian bridge, not from calibration", "mono": true }, { "label": "Efficiency", "value": "Garman and Klass 1980 report a theoretical efficiency gain over close-to-close of approximately 7.4 under their stated assumptions", "mono": true }, { "label": "Sign hazard", "value": "The second term is subtracted, so a single bar with a large close-open move and a narrow range contributes NEGATIVE variance. On a short window the total can go negative and the estimator returns no answer", "mono": true } ], "notes": [ "The negative-contribution hazard is real and not rare on daily bars of illiquid names, where a large close-open move with a narrow intraday range is exactly what a gap-then-drift day looks like. Implementations either floor the per-bar contribution at zero, which biases upward, or floor the total, which fails silently. State which.", "Like Parkinson it assumes zero drift and no opening jump. The close-open term is measured within the session, so an overnight gap is still invisible to it: the estimator sees the previous close nowhere in its formula.", "The efficiency gain is theoretical and conditional on the assumptions. On real data with discrete sampling of the extremes, the realised gain is smaller, and the estimator's downward range bias partially cancels its efficiency advantage." ], "source": "Garman and Klass 1980" }, { "id": "rogers-satchell-estimator", "term": "Rogers-Satchell estimator", "definition": "A range-based estimator that is unbiased in the presence of a nonzero drift. It replaces the raw squared range with products of the distances from the high and low to both the open and the close, which cancels the drift term exactly.", "formula": "sigma^2 = (A/n) * sum_t [ ln(H_t/C_t)*ln(H_t/O_t) + ln(L_t/C_t)*ln(L_t/O_t) ]", "fields": [ { "label": "Worked", "value": "Ten bars: sum of the bracketed terms = 0.0015095887. sigma^2 = 252/10 * 0.0015095887 = 0.03804164. sigma = 19.5043 percent", "mono": true }, { "label": "Non-negativity", "value": "Both products are non-negative for every valid bar, since H is at or above both O and C and L is at or below both. Unlike Garman-Klass the per-bar contribution can never be negative", "mono": true }, { "label": "Drift independence", "value": "The estimator is unbiased for any constant drift. Parkinson and Garman-Klass are unbiased only at zero drift and are inflated by trend", "mono": true }, { "label": "Comparison on the same bars", "value": "Rogers-Satchell 19.5043, Parkinson 19.0522, Garman-Klass 19.9407, close-to-close 19.6906. All four from the same forty prices", "mono": true } ], "notes": [ "Drift independence is bought with efficiency. Rogers-Satchell is less efficient than Garman-Klass at zero drift, which is the case Garman-Klass was optimised for, and more accurate when a trend is present. Which is preferable depends entirely on the asset and horizon.", "It still assumes no opening jump. The previous close does not appear in the formula, so an overnight gap contributes nothing, exactly as with Parkinson and Garman-Klass.", "It shares the discrete-sampling downward bias of all range estimators. Every estimator on this list that touches H or L understates volatility for the same reason, and the understatement grows with the sampling interval between ticks." ], "source": "Rogers and Satchell 1991" }, { "id": "yang-zhang-estimator", "term": "Yang-Zhang estimator", "definition": "A minimum-variance combination of the overnight return variance, the intraday close-open variance and the Rogers-Satchell variance. It is the only estimator in this list that handles both a nonzero drift and an opening jump, at the cost of two extra variance terms and a sample-size-dependent weight.", "formula": "sigma^2 = A * [ V_o + k*V_c + (1-k)*V_rs ], with V_o the sample variance of ln(O_t/C_{t-1}), V_c the sample variance of ln(C_t/O_t), both with divisor n-1, V_rs the per-period Rogers-Satchell mean, and k = 0.34/(1.34 + (n+1)/(n-1))", "fields": [ { "label": "Worked components", "value": "Ten bars: overnight returns have mean -0.000204 and variance 0.0000103083. Intraday close-open returns have mean 0.002283 and variance 0.0001147100. Rogers-Satchell per-period mean 0.0001509589", "mono": true }, { "label": "Worked weight", "value": "n = 10: k = 0.34/(1.34 + 11/9) = 0.34/(1.34 + 1.222222) = 0.34/2.562222 = 0.13269731", "mono": true }, { "label": "Worked total", "value": "252 * (0.0000103083 + 0.13269731*0.0001147100 + 0.86730269*0.0001509589) = 252 * 0.000156457 = 0.03942718. sigma = 19.8563 percent", "mono": true }, { "label": "Full comparison", "value": "On these ten bars: Parkinson 19.0522, Rogers-Satchell 19.5043, close-to-close zero-mean 19.6906, Yang-Zhang 19.8563, Garman-Klass 19.9407, close-to-close sample-mean 20.4623. Spread 1.41 volatility points", "mono": true } ], "notes": [ "The weight k depends on n, so the Yang-Zhang estimate of a fixed period changes when the window is extended even if no new bar falls inside the period. That is a property of the minimum-variance derivation, not a bug, but it makes the estimator non-comparable across window lengths.", "Two of the three components use a divisor of n-1 and a sample mean, so Yang-Zhang inherits the small-sample instability of the sample-mean close-to-close estimator. At n = 10 the overnight variance is estimated from nine degrees of freedom.", "It is the correct default when overnight gaps matter - single names, anything around earnings, anything with a meaningful non-trading period. It is unnecessary for a 24-hour market with no session boundary, where the overnight term is zero by construction.", "None of the six estimators is right and the others wrong. Each is unbiased under its own assumptions, and the 1.41-point spread on identical data is what the assumptions cost. Reporting a volatility without naming its estimator and its annualisation factor is not reproducible." ], "source": "Yang and Zhang 2000" }, { "id": "window-length-bias-variance", "term": "The bias-variance tradeoff in window length", "definition": "A longer estimation window lowers the sampling error of a volatility estimate and raises its staleness bias when the true volatility varies. The sampling error has a closed form; the staleness bias does not, because it depends on the unobserved dynamics of the true volatility.", "formula": "For iid normal returns, relative SE of the sample standard deviation is approximately 1/sqrt(2n); exactly, SD(s)/sigma = sqrt(1 - c4(n)^2) with c4(n) = sqrt(2/(n-1))*Gamma(n/2)/Gamma((n-1)/2)", "fields": [ { "label": "Worked, the base series", "value": "n = 24 monthly observations: 1/sqrt(48) = 0.144338, exact 0.146621. The annualised volatility of 7.2063 percent therefore carries a standard error of about 1.06 volatility points, a 95 percent interval of roughly 5.14 to 9.27 percent", "mono": true }, { "label": "Precision by window", "value": "n = 5 gives 34.12 percent relative SE; n = 20 gives 16.11; n = 60 gives 9.19; n = 252 gives 4.46; n = 504 gives 3.15. Halving the error requires quadrupling the window", "mono": true }, { "label": "Approximation quality", "value": "1/sqrt(2n) is within 2 percent of the exact value from n = 20 upward and within 8 percent at n = 5", "mono": true }, { "label": "The tradeoff, stated", "value": "A 20-day window has 16 percent estimation error and tracks a volatility regime change in about 20 days. A 252-day window has 4.5 percent estimation error and takes a year. Neither is correct; the choice is a horizon assumption", "mono": true } ], "notes": [ "There is no window length that is right in general, and reporting one without the horizon it was chosen for is reporting half a decision. An EWMA or GARCH specification replaces the discrete choice with a decay parameter, which is the same decision made continuously.", "The closed-form standard error assumes iid normal returns. Under volatility clustering the effective number of independent observations in a window is smaller than n, so the real standard error is larger than the table shows - the table is a lower bound.", "The two errors are not commensurable. Sampling error is symmetric and quantifiable; staleness bias is directional, largest exactly when it matters most - during a regime change - and cannot be estimated from the same window that suffers from it." ] } ] }