tppis 0.1.0__py3-none-any.whl

tppis/__init__.py

@@ -0,0 +1,29 @@
+"""TPPIS: Truncated Preconditioned Profiled Independence Screening."""
+
+from tppis.datasets import (
+    make_example1,
+    make_example2,
+    make_example3,
+    make_example4,
+)
+from tppis.exceptions import TPPISError
+from tppis.metrics import screening_scores
+from tppis.screeners import FPSIS, FPSISBIC, PPIS, SIS, TPPIS, screen
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "FPSIS",
+    "FPSISBIC",
+    "PPIS",
+    "SIS",
+    "TPPIS",
+    "TPPISError",
+    "make_example1",
+    "make_example2",
+    "make_example3",
+    "make_example4",
+    "screen",
+    "screening_scores",
+    "__version__",
+]

tppis/_validation.py

@@ -0,0 +1,150 @@
+"""Input checking, standardization and tie-stable ranking.
+
+Standardization uses the population standard deviation (``ddof=0``), matching
+ambiguity A-5 of the specification and ``sklearn.preprocessing.StandardScaler``.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+from tppis.exceptions import ConstantFeatureWarning, TPPISError
+
+__all__ = [
+    "TPPISError",
+    "apply_standardize",
+    "as_float_arrays",
+    "check_literal",
+    "rank_by_abs",
+    "standardize",
+]
+
+
+def as_float_arrays(
+    X: ArrayLike,
+    y: ArrayLike,
+) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
+    """Cast ``X`` and ``y`` to contiguous float64 arrays with matching ``n``.
+
+    Parameters
+    ----------
+    X :
+        Predictor matrix of shape ``(n, p)``.
+    y :
+        Response vector of shape ``(n,)``.
+
+    Returns
+    -------
+    X, y
+        Contiguous ``float64`` copies.
+    """
+    X_arr = np.ascontiguousarray(np.asarray(X, dtype=np.float64))
+    y_arr = np.ascontiguousarray(np.asarray(y, dtype=np.float64).reshape(-1))
+    if X_arr.ndim != 2:
+        raise TPPISError(f"X must be 2-dimensional, got shape {X_arr.shape}.")
+    n, p = X_arr.shape
+    if n < 2:
+        raise TPPISError(f"n must be at least 2, got {n}.")
+    if p < 1:
+        raise TPPISError("X must have at least one column.")
+    if y_arr.shape[0] != n:
+        raise TPPISError(
+            f"X and y must share the first dimension, "
+            f"got n={n} and y.size={y_arr.size}."
+        )
+    if not np.isfinite(X_arr).all():
+        raise TPPISError("X contains non-finite values.")
+    if not np.isfinite(y_arr).all():
+        raise TPPISError("y contains non-finite values.")
+    return X_arr, y_arr
+
+
+def standardize(
+    X: NDArray[np.float64],
+    y: NDArray[np.float64],
+    *,
+    enabled: bool = True,
+) -> tuple[
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    NDArray[np.float64],
+    float,
+]:
+    """Center ``y`` and optionally column-standardize ``X`` with ``ddof=0``.
+
+    Parameters
+    ----------
+    X, y :
+        Validated arrays from :func:`as_float_arrays`.
+    enabled :
+        If False, ``X`` is returned unchanged and column scales are ones.
+
+    Returns
+    -------
+    X_std, y_c, x_mean, x_scale, y_mean
+    """
+    y_mean = float(y.mean())
+    y_c = y - y_mean
+    x_mean = X.mean(axis=0)
+    if not enabled:
+        return X, y_c, x_mean, np.ones(X.shape[1], dtype=np.float64), y_mean
+    x_scale = X.std(axis=0, ddof=0)
+    zero = x_scale <= np.finfo(np.float64).eps * max(X.shape)
+    if np.any(zero):
+        warnings.warn(
+            f"{int(zero.sum())} constant column(s) left unscaled.",
+            ConstantFeatureWarning,
+            stacklevel=2,
+        )
+        x_scale = x_scale.copy()
+        x_scale[zero] = 1.0
+    X_std = (X - x_mean) / x_scale
+    return X_std, y_c, x_mean, x_scale, y_mean
+
+
+def rank_by_abs(
+    omega: NDArray[np.float64],
+) -> NDArray[np.intp]:
+    """Descending ``|omega|`` order, ties broken by ascending column index (A-7).
+
+    Parameters
+    ----------
+    omega :
+        Marginal importance scores of length ``p``.
+
+    Returns
+    -------
+    order
+        Integer indices of shape ``(p,)``.
+    """
+    p = omega.shape[0]
+    keys = np.empty((p, 2), dtype=np.float64)
+    keys[:, 0] = -np.abs(omega)
+    keys[:, 1] = np.arange(p, dtype=np.float64)
+    return np.lexsort((keys[:, 1], keys[:, 0])).astype(np.intp)
+
+
+def apply_standardize(
+    X: ArrayLike,
+    x_mean: NDArray[np.float64],
+    x_scale: NDArray[np.float64],
+) -> NDArray[np.float64]:
+    """Apply a previously fitted column standardization to new ``X``."""
+    X_arr = np.ascontiguousarray(np.asarray(X, dtype=np.float64))
+    if X_arr.ndim != 2:
+        raise TPPISError(f"X must be 2-dimensional, got shape {X_arr.shape}.")
+    if X_arr.shape[1] != x_mean.shape[0]:
+        raise TPPISError(f"X has {X_arr.shape[1]} columns, expected {x_mean.shape[0]}.")
+    return (X_arr - x_mean) / x_scale
+
+
+def check_literal(name: str, value: Any, allowed: set[str]) -> str:
+    """Validate a string option against a closed set."""
+    if value not in allowed:
+        raise TPPISError(f"{name} must be one of {sorted(allowed)}, got {value!r}.")
+    return str(value)

tppis/criteria.py

@@ -0,0 +1,220 @@
+"""Refit and the BIC-type criterion (10).
+
+The coefficient inside the criterion is ordinary least squares of the original
+columns on the centered response. That is the residual whose value matches
+Tables 1–4. Equation (8) fits the transformed design instead, and that residual
+does not.
+
+The Gram matrices for ``k = 1, 2, ...`` are nested leading blocks, so an
+incremental Cholesky sweep produces every ``beta_hat(M_k)`` in ``O(K^3 / 3)``.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.linalg import solve_triangular
+
+from tppis._validation import check_literal
+from tppis.exceptions import BoundarySelectionWarning, TPPISError, TppisWarning
+
+# Floor for log(RSS). Combined with A-9 this keeps BIC finite.
+_RSS_FLOOR = 1e-300
+# Relative pivot tolerance for the incremental Cholesky path.
+_CHOL_EPS = 1e-12
+
+
+@dataclass(frozen=True)
+class CriterionSweep:
+    """BIC path over ``k = 1 .. K`` for one ``(d, alpha)`` pair."""
+
+    k: NDArray[np.intp]
+    bic: NDArray[np.float64]
+    rss: NDArray[np.float64]
+    best_k: int
+    best_bic: float
+    best_beta: NDArray[np.float64]
+    on_boundary: bool
+
+
+def bic_value(
+    rss: float,
+    k: int,
+    n: int,
+    p: int,
+    *,
+    kind: str = "paper",
+) -> float:
+    """BIC-type criterion (10).
+
+    Parameters
+    ----------
+    rss :
+        ``|| y - X(M_k) @ beta_hat(M_k) ||^2`` on the original scale.
+    k, n, p :
+        Model size, sample size, number of predictors.
+    kind :
+        ``"paper"`` is ``log(RSS) + (log(p)/n) * k * log(n)`` (A-4).
+        ``"rss_mean"`` replaces ``RSS`` with ``RSS / n``.
+
+    Returns
+    -------
+    bic
+        Scalar criterion value. Natural logarithm.
+    """
+    kind = check_literal("bic", kind, {"paper", "rss_mean"})
+    if rss < _RSS_FLOOR:
+        warnings.warn(
+            f"RSS={rss} was clamped at {_RSS_FLOOR} before taking the log.",
+            TppisWarning,
+            stacklevel=2,
+        )
+        rss = _RSS_FLOOR
+    scale = rss if kind == "paper" else rss / n
+    return float(np.log(scale) + (np.log(p) / n) * k * np.log(n))
+
+
+def default_k_max(p: int, n: int, n_retained: int, override: int | None) -> int:
+    """A-9 cap ``k_max = min(p, n_retained, n - 1)``.
+
+    ``k <= p - 1`` is not the constraint that keeps least squares valid.
+    After centering, the columns of ``X`` span at most ``n - 1`` dimensions,
+    so ``k >= n`` interpolates ``y`` and ``log(RSS)`` is unbounded below.
+    An override at or above ``n`` is rejected. An override above the retained
+    rank is still clipped to that rank.
+    """
+    cap = min(p, n_retained, n - 1)
+    if cap < 1:
+        raise TPPISError(
+            f"k_max collapsed to {cap}; n_retained={n_retained}, n={n}, p={p}."
+        )
+    if override is None:
+        return cap
+    if override < 1:
+        raise TPPISError(f"k_max must be at least 1, got {override}.")
+    if int(override) >= n:
+        raise TPPISError(
+            f"k_max={override} must be at most n-1={n - 1}. "
+            "A column-centered design has rank at most n-1, so k >= n makes "
+            "log(RSS) unbounded below. k <= p-1 is not that constraint."
+        )
+    return min(int(override), cap)
+
+
+def _chol_solve(
+    L: NDArray[np.float64], rhs: NDArray[np.float64]
+) -> NDArray[np.float64]:
+    z = solve_triangular(L, rhs, lower=True, check_finite=False)
+    beta = solve_triangular(L.T, z, lower=False, check_finite=False)
+    return np.asarray(beta, dtype=np.float64)
+
+
+def _min_norm(
+    gram: NDArray[np.float64], rhs: NDArray[np.float64]
+) -> NDArray[np.float64]:
+    """Minimum-norm least-squares solve of a possibly singular Gram block (A-9)."""
+    beta, *_ = np.linalg.lstsq(gram, rhs, rcond=None)
+    return np.asarray(beta, dtype=np.float64)
+
+
+def incremental_refit(
+    F: NDArray[np.float64],
+    rhs: NDArray[np.float64],
+    X_sel: NDArray[np.float64],
+    y: NDArray[np.float64],
+    p: int,
+    *,
+    kind: str = "paper",
+) -> CriterionSweep:
+    """Sweep ``k = 1 .. K`` with nested Cholesky updates of equation (8).
+
+    Parameters
+    ----------
+    F :
+        Factor matrix of shape ``(K, s)`` such that
+        ``Gram_k = F[:k] @ F[:k].T``. For TPPIS/PPIS this is ``V_S(M)``;
+        for FPSIS it is ``V_S(M) * mu_S``; for SIS it is ``X(M).T``.
+    rhs :
+        ``X_hat(M_K).T @ y_hat`` restricted to the ranked columns, length ``K``.
+        Equals ``omega[order[:K]]``.
+    X_sel, y :
+        Ranked original (standardized) columns and the centered response, used
+        only for the residual of equation (10).
+    p :
+        Original number of predictors, for the BIC penalty.
+    kind :
+        Criterion flavour, see :func:`bic_value`.
+
+    Returns
+    -------
+    CriterionSweep
+        Full path and the minimizing ``k``.
+    """
+    K = int(F.shape[0])
+    if K < 1:
+        raise TPPISError("Cannot sweep an empty candidate set.")
+    n = int(y.shape[0])
+    L = np.zeros((K, K), dtype=np.float64)
+    bic = np.empty(K, dtype=np.float64)
+    rss = np.empty(K, dtype=np.float64)
+    best_k = 1
+    best_bic = np.inf
+    best_beta = np.empty(0, dtype=np.float64)
+    use_direct = False
+
+    for k in range(1, K + 1):
+        gk = F[:k] @ F[k - 1]
+        if not use_direct:
+            if k == 1:
+                pivot = float(gk[0])
+                if pivot <= _CHOL_EPS * max(1.0, float(np.abs(F[0] @ F[0]))):
+                    use_direct = True
+                else:
+                    L[0, 0] = np.sqrt(pivot)
+            else:
+                try:
+                    ell = solve_triangular(
+                        L[: k - 1, : k - 1], gk[: k - 1], lower=True, check_finite=False
+                    )
+                    rem = float(gk[k - 1] - ell @ ell)
+                    if rem <= _CHOL_EPS * max(1.0, float(np.abs(gk[k - 1]))):
+                        use_direct = True
+                    else:
+                        L[k - 1, : k - 1] = ell
+                        L[k - 1, k - 1] = np.sqrt(rem)
+                except np.linalg.LinAlgError:
+                    use_direct = True
+        gram = F[:k] @ F[:k].T
+        if use_direct:
+            beta = _min_norm(gram, rhs[:k])
+        else:
+            beta = _chol_solve(L[:k, :k], rhs[:k])
+        resid = y - X_sel[:, :k] @ beta
+        rss_k = float(resid @ resid)
+        bic_k = bic_value(rss_k, k, n, p, kind=kind)
+        rss[k - 1] = rss_k
+        bic[k - 1] = bic_k
+        if bic_k < best_bic:
+            best_bic = bic_k
+            best_k = k
+            best_beta = beta
+
+    on_boundary = best_k in {1, K}
+    if on_boundary:
+        warnings.warn(
+            f"BIC minimum landed on the k-boundary at k={best_k} (k_max={K}).",
+            BoundarySelectionWarning,
+            stacklevel=2,
+        )
+    return CriterionSweep(
+        k=np.arange(1, K + 1, dtype=np.intp),
+        bic=bic,
+        rss=rss,
+        best_k=best_k,
+        best_bic=best_bic,
+        best_beta=best_beta,
+        on_boundary=on_boundary,
+    )

tppis/datasets/__init__.py

@@ -0,0 +1,26 @@
+"""Simulation generators and real-data fetchers."""
+
+from tppis.datasets.real import fetch_hydraulic, fetch_sp500_info
+from tppis.datasets.simulate import (
+    SimulatedData,
+    example1_sigma,
+    example2_sigma,
+    make_example1,
+    make_example2,
+    make_example3,
+    make_example4,
+    min_eigenvalue,
+)
+
+__all__ = [
+    "SimulatedData",
+    "example1_sigma",
+    "example2_sigma",
+    "fetch_hydraulic",
+    "fetch_sp500_info",
+    "make_example1",
+    "make_example2",
+    "make_example3",
+    "make_example4",
+    "min_eigenvalue",
+]

tppis/datasets/real.py

@@ -0,0 +1,97 @@
+"""Fetch scripts for the two real datasets in Section 5 of the paper.
+
+Neither dataset is redistributed. These helpers download from the public
+sources the paper cites, cache locally, and raise a clear error if a source
+is unavailable.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import urllib.request
+from collections.abc import Callable
+from pathlib import Path
+
+DEFAULT_CACHE = Path(os.environ.get("TPPIS_DATA", Path.home() / ".tppis_data"))
+
+# Sources as cited in Tanaka and Matsui (2023), Section 5.
+HYDRAULIC_INFO = {
+    "name": "condition monitoring of hydraulic systems",
+    "paper_ref": "[17] Helwig, Pignanelli, Schutze, I2MTC 2015",
+    "url": "https://archive.ics.uci.edu/static/public/447/condition+monitoring+of+hydraulic+systems.zip",
+    "notes": (
+        "The paper uses n=1449 rows taken under stable system settings, "
+        "response = accumulator pressure (130/115/100/90), p=43680 from 17 sensors."
+    ),
+}
+
+SP500_INFO = {
+    "name": "S&P 500, year 2020",
+    "paper_ref": "[18] FRED SP500; [19] Kaggle S&P 500 stocks",
+    "fred_url": "https://fred.stlouisfed.org/series/SP500",
+    "kaggle_url": "https://www.kaggle.com/hanseopark/sp-500-stocks-value-with-financial-statement",
+    "notes": (
+        "The paper uses 253 trading days in 2020. Response = S&P 500 index; "
+        "predictors = constituent stock prices (some companies contribute more "
+        "than one series)."
+    ),
+}
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _download(url: str, dest: Path, *, timeout: int = 60) -> Path:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    if dest.exists():
+        return dest
+    tmp = dest.with_suffix(dest.suffix + ".part")
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as src, tmp.open("wb") as out:
+            out.write(src.read())
+        tmp.replace(dest)
+    except Exception as exc:  # noqa: BLE001
+        if tmp.exists():
+            tmp.unlink()
+        raise RuntimeError(
+            f"Could not download {url}. Fetch the file by hand and place it at {dest}."
+        ) from exc
+    return dest
+
+
+def fetch_hydraulic(
+    cache_dir: Path | None = None,
+    *,
+    downloader: Callable[[str, Path], Path] | None = None,
+) -> Path:
+    """Download the UCI hydraulic-systems archive into ``cache_dir``.
+
+    Returns the path to the zip. Parsing into the paper's ``(X, y)`` layout is
+    left to ``reproduction/run_real_data.py``, because the 17-sensor expansion
+    is study-specific.
+    """
+    root = cache_dir or DEFAULT_CACHE
+    dest = root / "hydraulic" / "hydraulic_systems.zip"
+    fetch = downloader or (lambda url, path: _download(url, path))
+    return fetch(HYDRAULIC_INFO["url"], dest)
+
+
+def fetch_sp500_info() -> dict[str, str]:
+    """Return the documented S&P 500 source URLs. No automatic download.
+
+    The Kaggle source requires an account, so this function only documents
+    the locations. ``reproduction/run_real_data.py`` reads a user-supplied
+    directory of CSVs.
+    """
+    return dict(SP500_INFO)
+
+
+def verify_checksum(path: Path, expected: str) -> bool:
+    """Compare a file to a published SHA-256 hex digest."""
+    return _sha256(path) == expected.lower()