quantized-lab 0.8.0__py3-none-any.whl

quantized/calc/stats_dist.py

@@ -0,0 +1,196 @@
+"""Distribution fitting with goodness-of-fit + t-test power / sample size.
+
+ORIGIN_GAP_PLAN #28 — new capability beyond MATLAB parity, scipy (BSD) only
+(statsmodels not required: power uses the exact noncentral-t formulation).
+Validated in tests against published values (Cohen's power tables / G*Power)
+and closed-form MLE identities.
+
+Conventions: positive-support families (lognormal / weibull / gamma /
+exponential) are fitted with ``loc`` fixed at 0 — the 2-parameter forms
+instrument-data practice (and Origin) use. KS p-values are flagged
+approximate when parameters were estimated from the same sample
+(the Lilliefors situation).
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy import stats as sps
+
+__all__ = ["fit_distribution", "fit_distributions", "required_n", "t_test_power"]
+
+# family -> (scipy dist, fit kwargs, n free params, positive support required)
+_FAMILIES: dict[str, tuple[Any, dict[str, float], int, bool]] = {
+    "normal": (sps.norm, {}, 2, False),
+    "lognormal": (sps.lognorm, {"floc": 0.0}, 2, True),
+    "weibull": (sps.weibull_min, {"floc": 0.0}, 2, True),
+    "gamma": (sps.gamma, {"floc": 0.0}, 2, True),
+    "exponential": (sps.expon, {"floc": 0.0}, 1, True),
+}
+
+_KINDS = ("one-sample", "paired", "two-sample")
+
+
+def fit_distribution(x: NDArray[np.float64], dist: str = "normal") -> dict[str, Any]:
+    """MLE-fit one distribution family; return params + log-likelihood + AIC
+    + a KS goodness-of-fit test against the fitted distribution.
+
+    ``params`` holds scipy's (shape, loc, scale) plus friendly aliases
+    (normal: mu/sigma; lognormal: mu/sigma of ln x; weibull: shape/scale;
+    gamma: shape/scale; exponential: rate).
+    """
+    if dist not in _FAMILIES:
+        raise ValueError(f"dist must be one of {sorted(_FAMILIES)}, got {dist!r}")
+    xv = np.asarray(x, dtype=float).ravel()
+    xv = xv[np.isfinite(xv)]
+    if xv.size < 5:
+        raise ValueError("fit_distribution needs at least 5 finite observations")
+    family, fit_kw, n_params, positive = _FAMILIES[dist]
+    if positive and np.any(xv <= 0):
+        raise ValueError(f"{dist} requires strictly positive data")
+
+    fitted = family.fit(xv, **fit_kw)
+    loglike = float(np.sum(family.logpdf(xv, *fitted)))
+    aic = 2.0 * n_params - 2.0 * loglike
+    frozen = family(*fitted)
+    ks = sps.kstest(xv, frozen.cdf)
+
+    shapes = [float(v) for v in fitted[:-2]]
+    loc, scale = float(fitted[-2]), float(fitted[-1])
+    params: dict[str, float] = {"loc": loc, "scale": scale}
+    if shapes:
+        params["shape"] = shapes[0]
+    if dist == "normal":
+        params.update(mu=loc, sigma=scale)
+    elif dist == "lognormal":
+        params.update(mu=math.log(scale), sigma=shapes[0])
+    elif dist == "exponential":
+        params.update(rate=1.0 / scale)
+
+    return {
+        "dist": dist,
+        "params": params,
+        "loglike": loglike,
+        "aic": aic,
+        "n_params": n_params,
+        "ks_d": float(ks.statistic),
+        "ks_p": float(ks.pvalue),
+        "ks_p_approximate": True,  # params estimated from the same sample
+        "N": int(xv.size),
+    }
+
+
+def fit_distributions(
+    x: NDArray[np.float64], dists: list[str] | None = None
+) -> dict[str, Any]:
+    """Fit several families and rank them by AIC (lowest first).
+
+    Families whose support the data violates (or that fail to converge) are
+    reported under ``skipped`` with the reason, never silently dropped.
+    """
+    names = list(_FAMILIES) if dists is None else dists
+    fits: list[dict[str, Any]] = []
+    skipped: list[dict[str, str]] = []
+    for name in names:
+        try:
+            fits.append(fit_distribution(x, name))
+        except ValueError as exc:
+            skipped.append({"dist": name, "reason": str(exc)})
+        except Exception as exc:  # noqa: BLE001 — scipy fit convergence failures
+            skipped.append({"dist": name, "reason": f"fit failed: {exc}"})
+    if not fits:
+        raise ValueError("no distribution family could be fitted")
+    fits.sort(key=lambda f: f["aic"])
+    return {"fits": fits, "best": fits[0]["dist"], "skipped": skipped}
+
+
+def _power_from_n(kind: str, d: float, n: int, alpha: float, tails: int) -> float:
+    """Exact noncentral-t power for a t-test at the given per-group n."""
+    if kind == "two-sample":
+        df = 2 * n - 2
+        ncp = abs(d) * math.sqrt(n / 2.0)
+    else:  # one-sample / paired
+        df = n - 1
+        ncp = abs(d) * math.sqrt(n)
+    if df < 1:
+        return 0.0
+    if tails == 2:
+        tc = float(sps.t.ppf(1.0 - alpha / 2.0, df))
+        return float(1.0 - sps.nct.cdf(tc, df, ncp) + sps.nct.cdf(-tc, df, ncp))
+    tc = float(sps.t.ppf(1.0 - alpha, df))
+    return float(1.0 - sps.nct.cdf(tc, df, ncp))
+
+
+def t_test_power(
+    effect_size: float,
+    n: int,
+    *,
+    kind: str = "two-sample",
+    alpha: float = 0.05,
+    tails: int = 2,
+) -> dict[str, Any]:
+    """Power of a t-test at Cohen's d = ``effect_size`` and per-group ``n``.
+
+    Exact noncentral-t computation (what G*Power does): ncp = d·sqrt(n/2)
+    with df = 2n-2 for two-sample; ncp = d·sqrt(n), df = n-1 for
+    one-sample/paired. Reference: Cohen, *Statistical Power Analysis*, ch. 2.
+    """
+    if kind not in _KINDS:
+        raise ValueError(f"kind must be one of {_KINDS}, got {kind!r}")
+    if tails not in (1, 2):
+        raise ValueError("tails must be 1 or 2")
+    if n < 2:
+        raise ValueError("n must be >= 2")
+    if not 0 < alpha < 1:
+        raise ValueError("alpha must be in (0, 1)")
+    power = _power_from_n(kind, effect_size, n, alpha, tails)
+    return {
+        "power": power,
+        "effect_size": float(effect_size),
+        "n": int(n),
+        "kind": kind,
+        "alpha": alpha,
+        "tails": tails,
+    }
+
+
+def required_n(
+    effect_size: float,
+    power: float = 0.8,
+    *,
+    kind: str = "two-sample",
+    alpha: float = 0.05,
+    tails: int = 2,
+) -> dict[str, Any]:
+    """Smallest per-group n reaching the target power (doubling + bisection)."""
+    if kind not in _KINDS:
+        raise ValueError(f"kind must be one of {_KINDS}, got {kind!r}")
+    if not 0 < power < 1:
+        raise ValueError("power must be in (0, 1)")
+    if effect_size == 0:
+        raise ValueError("effect_size must be nonzero")
+    lo, hi = 2, 4
+    while _power_from_n(kind, effect_size, hi, alpha, tails) < power:
+        hi *= 2
+        if hi > 2**22:
+            raise ValueError("required n exceeds 4e6 — effect size too small")
+    while lo < hi:
+        mid = (lo + hi) // 2
+        if _power_from_n(kind, effect_size, mid, alpha, tails) >= power:
+            hi = mid
+        else:
+            lo = mid + 1
+    achieved = _power_from_n(kind, effect_size, lo, alpha, tails)
+    return {
+        "n": int(lo),
+        "achieved_power": achieved,
+        "target_power": power,
+        "effect_size": float(effect_size),
+        "kind": kind,
+        "alpha": alpha,
+        "tails": tails,
+    }

quantized/calc/stats_glm.py

@@ -0,0 +1,245 @@
+"""Generalized Linear Models: logistic and Poisson regression with inference.
+
+GAP_PLAN #30 (stats extra) — GLM via statsmodels (BSD-3). Result-dict shapes
+match ``stats_multivar.multiple_regression`` so downstream code (routes, report
+sheets) treats linear/logistic/Poisson alike. Rows with any non-finite value
+are dropped (listwise deletion). Requires ``pip install quantized[stats]``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+__all__ = ["logistic_regression", "poisson_regression"]
+
+_EPS = float(np.finfo(float).tiny)
+
+
+def _check_statsmodels() -> None:
+    """Raise a clear error if statsmodels is not installed."""
+    try:
+        import statsmodels  # noqa: F401
+    except ImportError as exc:
+        raise RuntimeError(
+            "GLM methods require statsmodels. Install with: pip install quantized[stats]"
+        ) from exc
+
+
+def _as_matrix(columns: list[NDArray[np.float64]] | NDArray[np.float64]) -> NDArray[np.float64]:
+    """Stack same-length 1-D columns into an (n, k) matrix (validates lengths)."""
+    if isinstance(columns, np.ndarray) and columns.ndim == 2:
+        return np.asarray(columns, dtype=float)
+    cols = [np.asarray(c, dtype=float).ravel() for c in columns]
+    if len(cols) < 1:
+        raise ValueError("need at least one column")
+    n = cols[0].size
+    if any(c.size != n for c in cols):
+        raise ValueError("all columns must have the same length")
+    return np.column_stack(cols)
+
+
+def _binomial_deviance(y: NDArray[np.float64], mu: NDArray[np.float64]) -> float:
+    """Bernoulli/binomial deviance: D = 2·Σ[y·log(y/μ) + (1-y)·log((1-y)/(1-μ))].
+
+    ``statsmodels``' discrete-choice ``LogitResults`` does not expose a
+    ``.deviance`` scalar (only per-observation ``resid_dev``), unlike the
+    ``genmod.GLM`` family models — compute it directly from the standard
+    formula (matches ``sm.GLM(..., family=Binomial()).deviance`` exactly).
+    """
+    with np.errstate(divide="ignore", invalid="ignore"):
+        t1 = np.where(y > 0, y * np.log(y / mu), 0.0)
+        t2 = np.where(y < 1, (1.0 - y) * np.log((1.0 - y) / (1.0 - mu)), 0.0)
+    return float(2.0 * np.sum(t1 + t2))
+
+
+def _poisson_deviance(y: NDArray[np.float64], mu: NDArray[np.float64]) -> float:
+    """Poisson deviance: D = 2·Σ[y·log(y/μ) - (y - μ)].
+
+    ``statsmodels``' discrete-choice ``PoissonResults`` does not expose a
+    ``.deviance`` scalar — compute it directly (matches
+    ``sm.GLM(..., family=Poisson()).deviance`` exactly).
+    """
+    with np.errstate(divide="ignore", invalid="ignore"):
+        term = np.where(y > 0, y * np.log(y / mu), 0.0)
+    return float(2.0 * np.sum(term - (y - mu)))
+
+
+def logistic_regression(
+    predictors: list[NDArray[np.float64]] | NDArray[np.float64],
+    y: NDArray[np.float64],
+    *,
+    alpha: float = 0.05,
+) -> dict[str, Any]:
+    """Logistic regression ``logit(p) = b0 + b1·x1 + … + bk·xk`` with inference.
+
+    ``y`` must be binary (0/1). ``predictors`` is a list of k same-length
+    columns (or an (n, k) matrix); the intercept is always included. Returns
+    coefficients (intercept first), standard errors, z-stats/p-values, 95%
+    confidence intervals, AIC, and deviance — same key vocabulary as
+    ``multiple_regression``. Rows containing any non-finite value are dropped
+    (listwise deletion).
+
+    Reference: logistic GLM via statsmodels ``Logit(...).fit(disp=0)``
+    (Newton-Raphson IRLS).
+    """
+    _check_statsmodels()
+    import statsmodels.api as sm
+
+    xmat0 = _as_matrix(predictors)
+    yv = np.asarray(y, dtype=float).ravel()
+    if yv.size != xmat0.shape[0]:
+        raise ValueError(f"y length {yv.size} != predictor length {xmat0.shape[0]}")
+
+    # Listwise NaN deletion
+    keep = np.isfinite(yv) & np.all(np.isfinite(xmat0), axis=1)
+    xmat0, yv = xmat0[keep], yv[keep]
+    n, k = xmat0.shape
+
+    if n < k + 2:
+        raise ValueError(f"need at least {k + 2} complete rows for {k} predictors")
+
+    # Check binary
+    unique_y = np.unique(yv)
+    is_binary = (
+        np.array_equal(unique_y, [0.0, 1.0])
+        or np.array_equal(unique_y, [0.0])
+        or np.array_equal(unique_y, [1.0])
+    )
+    if not is_binary:
+        raise ValueError("logistic regression requires binary y (0/1)")
+
+    # Add constant, fit
+    xmat = sm.add_constant(xmat0)
+    model = sm.Logit(yv, xmat)
+    result = model.fit(disp=0)
+
+    coeffs = np.asarray(result.params, dtype=float)
+    se = np.asarray(result.bse, dtype=float)
+    z_stats = np.asarray(result.tvalues, dtype=float)
+    p_values = np.asarray(result.pvalues, dtype=float)
+
+    # 95% CIs (z_crit ≈ 1.96 for alpha=0.05)
+    z_crit = 1.959964  # scipy.stats.norm.ppf(1 - alpha / 2)
+    ci_low = np.asarray(coeffs - z_crit * se, dtype=float)
+    ci_high = np.asarray(coeffs + z_crit * se, dtype=float)
+
+    # Predictions for pseudo-R²
+    y_pred = np.asarray(result.predict(xmat), dtype=float)
+
+    # McFadden's pseudo-R² = 1 - (LL_full / LL_null)
+    null_model = sm.Logit(yv, sm.add_constant(np.ones(n))).fit(disp=0)
+    ll_full = float(result.llf)
+    ll_null = float(null_model.llf)
+    # ll_null is a log-likelihood (log of a probability mass ≤ 1), so it is
+    # always ≤ 0 — guard the magnitude while preserving sign. `max(ll_null,
+    # _EPS)` (the previous form) picks the tiny *positive* _EPS whenever
+    # ll_null is negative (virtually always), which inverts the sign of the
+    # ratio and silently saturates pseudoR2 to 1.0 for every non-degenerate
+    # fit — verified against the McFadden pseudo-R² statsmodels itself
+    # reports (0.374 on the spector logistic reference case).
+    ll_null_safe = -max(abs(ll_null), _EPS)
+    pseudo_r2 = 1.0 - (ll_full / ll_null_safe)
+
+    return {
+        "coeffs": coeffs,  # intercept first
+        "se": se,
+        "zStats": z_stats,
+        "pValues": p_values,
+        "ciLow": ci_low,
+        "ciHigh": ci_high,
+        "pseudoR2": float(np.clip(pseudo_r2, 0.0, 1.0)),
+        "AIC": float(result.aic),
+        "deviance": _binomial_deviance(yv, y_pred),
+        "yPred": y_pred,
+        "N": n,
+        "alpha": alpha,
+    }
+
+
+def poisson_regression(
+    predictors: list[NDArray[np.float64]] | NDArray[np.float64],
+    y: NDArray[np.float64],
+    *,
+    alpha: float = 0.05,
+) -> dict[str, Any]:
+    """Poisson regression ``log(μ) = b0 + b1·x1 + … + bk·xk`` with inference.
+
+    ``y`` must be non-negative integers (counts). ``predictors`` is a list of
+    k same-length columns (or an (n, k) matrix); the intercept is always
+    included. Returns coefficients (intercept first), standard errors,
+    z-stats/p-values, 95% confidence intervals, AIC, and deviance — same key
+    vocabulary as ``multiple_regression``. Rows containing any non-finite
+    value are dropped (listwise deletion).
+
+    Reference: Poisson GLM via statsmodels ``Poisson(...).fit(disp=0)``
+    (Fisher scoring / IRLS).
+    """
+    _check_statsmodels()
+    import statsmodels.api as sm
+
+    xmat0 = _as_matrix(predictors)
+    yv = np.asarray(y, dtype=float).ravel()
+    if yv.size != xmat0.shape[0]:
+        raise ValueError(f"y length {yv.size} != predictor length {xmat0.shape[0]}")
+
+    # Listwise NaN deletion
+    keep = np.isfinite(yv) & np.all(np.isfinite(xmat0), axis=1)
+    xmat0, yv = xmat0[keep], yv[keep]
+    n, k = xmat0.shape
+
+    if n < k + 2:
+        raise ValueError(f"need at least {k + 2} complete rows for {k} predictors")
+
+    # Check non-negative counts
+    if np.any(yv < 0.0) or not np.allclose(yv, np.round(yv)):
+        raise ValueError("Poisson regression requires non-negative integer y")
+
+    # Add constant, fit
+    xmat = sm.add_constant(xmat0)
+    model = sm.Poisson(yv, xmat)
+    result = model.fit(disp=0)
+
+    coeffs = np.asarray(result.params, dtype=float)
+    se = np.asarray(result.bse, dtype=float)
+    z_stats = np.asarray(result.tvalues, dtype=float)
+    p_values = np.asarray(result.pvalues, dtype=float)
+
+    # 95% CIs
+    z_crit = 1.959964
+    ci_low = np.asarray(coeffs - z_crit * se, dtype=float)
+    ci_high = np.asarray(coeffs + z_crit * se, dtype=float)
+
+    # Predictions
+    y_pred = np.asarray(result.predict(xmat), dtype=float)
+
+    # McFadden's pseudo-R²
+    null_model = sm.Poisson(yv, sm.add_constant(np.ones(n))).fit(disp=0)
+    ll_full = float(result.llf)
+    ll_null = float(null_model.llf)
+    # ll_null is a log-likelihood (log of a probability mass ≤ 1), so it is
+    # always ≤ 0 — guard the magnitude while preserving sign. `max(ll_null,
+    # _EPS)` (the previous form) picks the tiny *positive* _EPS whenever
+    # ll_null is negative (virtually always), which inverts the sign of the
+    # ratio and silently saturates pseudoR2 to 1.0 for every non-degenerate
+    # fit — verified against the McFadden pseudo-R² statsmodels itself
+    # reports (0.374 on the spector logistic reference case).
+    ll_null_safe = -max(abs(ll_null), _EPS)
+    pseudo_r2 = 1.0 - (ll_full / ll_null_safe)
+
+    return {
+        "coeffs": coeffs,
+        "se": se,
+        "zStats": z_stats,
+        "pValues": p_values,
+        "ciLow": ci_low,
+        "ciHigh": ci_high,
+        "pseudoR2": float(np.clip(pseudo_r2, 0.0, 1.0)),
+        "AIC": float(result.aic),
+        "deviance": _poisson_deviance(yv, y_pred),
+        "yPred": y_pred,
+        "N": n,
+        "alpha": alpha,
+    }