views-frames 1.3.0__tar.gz → 1.6.0__tar.gz

{views_frames-1.3.0 → views_frames-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: views-frames
-Version: 1.3.0
+Version: 1.6.0
 Summary: The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG).
 Project-URL: Homepage, https://github.com/views-platform/views-frames
 Project-URL: Repository, https://github.com/views-platform/views-frames
@@ -30,10 +30,14 @@ Description-Content-Type: text/markdown
 > containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
 > that every other repo depends on and that depends on nothing internal.
 >
-> **Status:** **v1.3.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
+> **Status:** **v1.6.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
 > purely additive — the coherent posterior summary, ADR-019; v1.2.0 rebuilt the tower
 > `outside-in`, C-44; v1.3.0 makes the tower summary distribution-agnostic — no magnitude
-> zeroing by default, register C-45). This README is the design
+> zeroing by default, register C-45; v1.4.0 adds generic provenance to `FrameMetadata`
+> (`run_id`/`data_version`) and publishes the shared `assert_frame_envelope` checker,
+> ADR-020; v1.5.0 adds the threshold **exceedance** estimator `P(Y > c)`, ADR-021; v1.6.0
+> adds the worst-case **expected_shortfall** estimator, ADR-022). This
+> README is the design
 > bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
 > io, conformance suite) plus the `src/views_frames_summarize/` sibling package
 > (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators

{views_frames-1.3.0 → views_frames-1.6.0}/README.md

@@ -4,10 +4,14 @@
 > containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
 > that every other repo depends on and that depends on nothing internal.
 >
-> **Status:** **v1.3.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
+> **Status:** **v1.6.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
 > purely additive — the coherent posterior summary, ADR-019; v1.2.0 rebuilt the tower
 > `outside-in`, C-44; v1.3.0 makes the tower summary distribution-agnostic — no magnitude
-> zeroing by default, register C-45). This README is the design
+> zeroing by default, register C-45; v1.4.0 adds generic provenance to `FrameMetadata`
+> (`run_id`/`data_version`) and publishes the shared `assert_frame_envelope` checker,
+> ADR-020; v1.5.0 adds the threshold **exceedance** estimator `P(Y > c)`, ADR-021; v1.6.0
+> adds the worst-case **expected_shortfall** estimator, ADR-022). This
+> README is the design
 > bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
 > io, conformance suite) plus the `src/views_frames_summarize/` sibling package
 > (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators

{views_frames-1.3.0 → views_frames-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "views-frames"
-version = "1.3.0"
+version = "1.6.0"
 description = "The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG)."
 authors = [
     { name = "Simon Polichinel von der Maase", email = "simmaa@prio.org" },

{views_frames-1.3.0 → views_frames-1.6.0}/src/views_frames/conformance/__init__.py

@@ -10,6 +10,14 @@ Usage in a consumer's test::
     from views_frames.conformance import assert_frame_contract
     assert_frame_contract(my_adapter_output())
 
+A consumer whose type is **not** spatiotemporal (a string-keyed evaluation output, e.g.
+views-evaluation's ``MetricFrame``) validates against the shared envelope instead, so
+the envelope has one written authority rather than re-asserted copies that drift
+(ADR-020, register C-46)::
+
+    from views_frames.conformance import assert_frame_envelope
+    assert_frame_envelope(my_metric_frame())
+
 The floor is governed in ``GOVERNANCE.md``; ``CONFORMANCE_FLOOR`` records the version
 this suite belongs to.
 """
@@ -27,20 +35,26 @@ __all__ = [
     "CONFORMANCE_FLOOR",
     "assert_cross_level_alignment_law",
     "assert_frame_contract",
+    "assert_frame_envelope",
     "assert_index_alignment_laws",
 ]
 
 
-def assert_frame_contract(frame: Any) -> None:
-    """Assert ``frame`` satisfies the views-frames data contract.
+def assert_frame_envelope(frame: Any) -> None:
+    """Assert ``frame`` satisfies the shared **frame envelope**.
+
+    The envelope is the subset of the contract that applies to *any* frame-like value
+    object, spatiotemporal or not: float32 values with no object dtype, an explicit
+    trailing axis, rows equal to ``n_rows``, and a save/load round-trip that preserves
+    the values and every identifier array. It deliberately says nothing about *which*
+    identifiers exist or their dtype — that is the spatiotemporal layer's concern.
 
-    Checks the structural invariants (float32 values, no object dtype, an explicit
-    trailing axis, complete integer identifiers of length ``n_rows``) and the
-    save/load round-trip. (Sample-axis reduction is the ``views_frames_summarize``
-    package's concern, not the contract's — ADR-017.)
+    This is the single written authority for the envelope so a sibling that re-asserts
+    it (views-evaluation's ``MetricFrame``, keyed by string axes — not ``(time, unit)``)
+    validates against this checker rather than drifting from it (ADR-020, C-46).
 
     Raises:
-        AssertionError: any part of the contract is violated.
+        AssertionError: any part of the envelope is violated.
     """
     values = frame.values
     assert isinstance(values, np.ndarray), "values must be a numpy array"
@@ -49,6 +63,22 @@ def assert_frame_contract(frame: Any) -> None:
     assert values.ndim >= 2, "values must have an explicit trailing sample axis"
     assert values.shape[0] == frame.n_rows, "values rows must equal n_rows"
 
+    _assert_roundtrip(frame)
+
+
+def assert_frame_contract(frame: Any) -> None:
+    """Assert ``frame`` satisfies the full views-frames **spatiotemporal** contract.
+
+    The shared envelope (:func:`assert_frame_envelope`) plus the spatiotemporal
+    requirement: complete integer ``time``/``unit`` identifiers of length ``n_rows``.
+    (Sample-axis reduction is the ``views_frames_summarize`` package's concern, not the
+    contract's — ADR-017.)
+
+    Raises:
+        AssertionError: the envelope or the spatiotemporal identifier rule is violated.
+    """
+    assert_frame_envelope(frame)
+
     ids = frame.identifiers
     for key in ("time", "unit"):
         assert key in ids, f"missing required identifier '{key}'"
@@ -56,14 +86,14 @@ def assert_frame_contract(frame: Any) -> None:
         assert np.issubdtype(arr.dtype, np.integer), f"'{key}' must be integer"
         assert arr.shape == (frame.n_rows,), f"'{key}' must be length n_rows"
 
-    _assert_roundtrip(frame)
-
 
 def _assert_roundtrip(frame: Any) -> None:
     with tempfile.TemporaryDirectory() as directory:
         frame.save(directory)
         loaded = type(frame).load(directory)
-        assert np.array_equal(loaded.values, frame.values), "save/load changed values"
+        assert np.array_equal(loaded.values, frame.values, equal_nan=True), (
+            "save/load changed values"
+        )
         for key, arr in frame.identifiers.items():
             assert np.array_equal(loaded.identifiers[key], arr), (
                 f"save/load changed identifier '{key}'"

{views_frames-1.3.0 → views_frames-1.6.0}/src/views_frames/metadata.py

@@ -3,6 +3,11 @@
 Not a free-form dict: a frozen dataclass with all-optional, validated fields, so
 adding a field is a MINOR change and consumers cannot diverge on key names (the
 store-side cause of reporting's C-48). It is the typed home for run/eval identity.
+
+Provenance fields are **generic only** (``run_id``, ``data_version``): meaningful for
+any frame. Evaluation-specific provenance (``scoring_code_version``, full-precision
+``evaluation_timestamp``) stays in views-evaluation's ``MetricFrame`` and must not leak
+into this generic header (ADR-020, register C-47).
 """
 
 from __future__ import annotations
@@ -20,6 +25,8 @@ class FrameMetadata:
     run_type: str | None = None
     timestamp: int | None = None
     seed: int | None = None
+    run_id: str | None = None
+    data_version: str | None = None
 
     def to_dict(self) -> dict[str, Any]:
         """Serialize to a plain dict, omitting unset (``None``) fields."""

{views_frames-1.3.0 → views_frames-1.6.0}/src/views_frames_summarize/__init__.py

@@ -17,6 +17,8 @@ from views_frames_summarize.aggregate import (
 )
 from views_frames_summarize.bimodality import bimodality
 from views_frames_summarize.collapse import collapse
+from views_frames_summarize.exceedance import exceedance, exceedance_reducer
+from views_frames_summarize.expected_shortfall import expected_shortfall
 from views_frames_summarize.interval import hdi, quantiles
 from views_frames_summarize.point import map_estimate
 from views_frames_summarize.summarize_tower import TowerSummary, summarize_tower
@@ -29,6 +31,9 @@ __all__ = [
     "aggregate_distributions_arrays",
     "bimodality",
     "collapse",
+    "exceedance",
+    "exceedance_reducer",
+    "expected_shortfall",
     "hdi",
     "hdi_tower",
     "map_estimate",

{views_frames-1.3.0 → views_frames-1.6.0}/src/views_frames_summarize/conformance.py

@@ -13,6 +13,8 @@ from views_frames_summarize import config
 from views_frames_summarize._common import AnyFrame
 from views_frames_summarize.bimodality import bimodality
 from views_frames_summarize.collapse import collapse
+from views_frames_summarize.exceedance import exceedance
+from views_frames_summarize.expected_shortfall import expected_shortfall
 from views_frames_summarize.interval import hdi, quantiles
 from views_frames_summarize.point import map_estimate
 from views_frames_summarize.summarize_tower import summarize_tower
@@ -44,6 +46,36 @@ def assert_summarizer_contract(frame: AnyFrame) -> None:
     assert qs.shape[0] == n, "quantiles must be aligned to the frame's rows"
     assert qs.shape[-1] == 3, "quantiles must produce one column per quantile"
 
+    # exceedance laws (ADR-021): a survival function — in [0, 1], non-increasing in the
+    # threshold, with P(> -inf) = 1 and P(> +inf) = 0.
+    thresholds = [-np.inf, 0.5, 1.5, np.inf]
+    exc = exceedance(frame, thresholds)
+    assert exc.shape[0] == n, "exceedance must be aligned to the frame's rows"
+    assert exc.shape[-1] == len(thresholds), "exceedance → one column per threshold"
+    assert ((exc >= 0.0) & (exc <= 1.0)).all(), (
+        "exceedance must be a probability in [0, 1]"
+    )
+    assert (np.diff(exc, axis=-1) <= 1e-6).all(), (
+        "exceedance must be non-increasing in the threshold"
+    )
+    assert (exc[..., 0] == 1.0).all(), "P(> -inf) must be 1"
+    assert (exc[..., -1] == 0.0).all(), "P(> +inf) must be 0"
+
+    # expected-shortfall laws (ADR-022): a tail mean — in [min, max], non-decreasing as
+    # the tail deepens (t → 0), and dominating its VaR (ES(t) ≥ the (1 − t) quantile).
+    tails = [1.0, 0.5, 0.1]  # widening tails (deepening worst-case)
+    es = expected_shortfall(frame, tails)
+    lo = frame.values.min(axis=-1)[..., np.newaxis]
+    hi = frame.values.max(axis=-1)[..., np.newaxis]
+    assert es.shape[0] == n, "expected_shortfall must be aligned to the frame's rows"
+    assert es.shape[-1] == len(tails), "expected_shortfall → one column per tail"
+    assert ((es >= lo - 1e-6) & (es <= hi + 1e-6)).all(), "ES must lie in [min, max]"
+    assert (np.diff(es, axis=-1) >= -1e-6).all(), (
+        "ES must be non-decreasing as the tail deepens"
+    )
+    var = quantiles(frame, [0.0, 0.5, 0.9])  # the (1 − t) quantiles for t = 1, 0.5, 0.1
+    assert (es >= var - 1e-6).all(), "ES(t) must dominate the (1 − t) quantile"
+
     _assert_tower_contract(frame, n)
 
 

views_frames-1.6.0/src/views_frames_summarize/exceedance.py

@@ -0,0 +1,96 @@
+"""Threshold exceedance probabilities over the sample axis (ADR-021).
+
+`exceedance(frame, thresholds)` returns the per-row empirical **survival fraction**
+`P(Y > c)` for each threshold — an index-aligned array `(N, …, K)`, the same shape/role
+family as `quantiles`. `exceedance_reducer(c)` is a `collapse`-compatible factory for
+the single-threshold `(N, …, 1)` **frame** path.
+
+Distribution-agnostic (a counting reducer — no histogram, no config, no unimodality
+assumption), so it is robust where the tower is weakest (register C-34); the flagship
+is `P(Y > 0)` = probability of any activity (onset). Thresholds are **required**
+per-call, in the frame's own units (an *input*, like `quantiles`' ``qs`` — never a
+tunable or a default). **Strict `>`** (register D-08). Fails loud on any non-finite
+draw — NaN or ±inf (register C-50) — and on empty thresholds; the joint-sample
+requirement for aggregate exceedance is the consumer's (C-49).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
+from views_frames_summarize.collapse import Reducer
+
+
+def _exceed(
+    values: NDArray[np.float32], thresholds: NDArray[np.float64]
+) -> NDArray[np.float32]:
+    """Fraction of the trailing (sample) axis **strictly** greater than each threshold.
+
+    ``values`` is ``(…, S)`` and ``thresholds`` is ``(K,)``; returns ``(…, K)``.
+
+    Fails loud on any **non-finite** draw (NaN or ±inf): ``NaN > c`` is silently
+    ``False``, so a naive count would deflate ``P(Y > c)`` — worst on the ``P(Y > 0)``
+    onset flagship; and an ±inf draw (always an upstream bug) silently blesses ``P``
+    as a valid probability, masking the bug. A draw must be finite (C-50).
+    """
+    if not bool(np.isfinite(values).all()):
+        raise ValueError(
+            "exceedance is undefined on non-finite draws (NaN or ±inf); strip or "
+            "impute upstream (NaN > c is silently False and would deflate P(Y > c); "
+            "an inf draw masks an upstream bug behind a valid-looking P; register C-50)"
+        )
+    # (…, S) vs (K,) → (…, K, S) by broadcasting; strict ">"; mean over the sample axis.
+    above = values[..., np.newaxis, :] > thresholds[:, np.newaxis]
+    return np.asarray(above.mean(axis=-1), dtype=np.float32)
+
+
+def exceedance(
+    frame: AnyFrame,
+    thresholds: Sequence[float],
+    *,
+    block_rows: int = ROW_BLOCK,
+) -> NDArray[np.float32]:
+    """Per-row exceedance probability `P(Y > c)` over the sample axis → `(N, …, K)`.
+
+    For each of the ``K`` caller-supplied ``thresholds`` (required, in the frame's own
+    units), the empirical fraction of draws **strictly** greater than it. An
+    index-aligned numpy array (the caller holds the index), vectorized in row-blocks of
+    ``block_rows`` so peak memory is bounded by one block's working set (register C-25).
+
+    Raises:
+        ValueError: ``thresholds`` is empty (no default), or any draw is non-finite —
+            NaN or ±inf (C-50).
+    """
+    thr = np.asarray(thresholds, dtype=np.float64)
+    if thr.size == 0:
+        raise ValueError(
+            "exceedance requires at least one threshold; there is no default "
+            "(thresholds are an input in the frame's units, not a tunable — ADR-021)."
+        )
+
+    def _block(vals: NDArray[np.float32]) -> NDArray[np.float32]:
+        return _exceed(vals, thr)
+
+    out = block_apply(frame.values, block_rows, _block)
+    return np.asarray(out, dtype=np.float32)
+
+
+def exceedance_reducer(threshold: float) -> Reducer:
+    """A `collapse`-compatible reducer for single-threshold exceedance.
+
+    ``collapse(frame, exceedance_reducer(c))`` returns `P(Y > c)` as a `(N, …, 1)`
+    frame, sharing `exceedance`'s strict-`>` and fail-loud-non-finite policy (so a
+    consumer never re-implements ``np.mean(values > c)`` and drifts on the convention).
+    """
+    thr = np.asarray([threshold], dtype=np.float64)
+
+    def _reducer(values: NDArray[np.float32], axis: int = -1) -> NDArray[np.float32]:
+        # `collapse` calls reducer(values, axis=-1) and appends the trailing axis
+        # itself, so drop the singleton K axis (collapse re-adds the sample-axis slot).
+        return _exceed(values, thr)[..., 0]
+
+    return _reducer

views_frames-1.6.0/src/views_frames_summarize/expected_shortfall.py

@@ -0,0 +1,88 @@
+"""Worst-case scenario summary over the sample axis (ADR-022).
+
+`expected_shortfall(frame, tails)` returns the per-row **mean of the worst `⌈t·S⌉`
+draws** for each upper-tail fraction `t` — the tail mean / CVaR, a coherent worst-case
+risk measure and the companion to `exceedance`. An index-aligned array `(N, …, K)`, the
+same shape/role family as `quantiles`.
+
+`max` is deliberately **not** offered — it is the highest-variance, non-reproducible
+summary this replaces. Tails are **required** per-call, in `(0, 1]` (e.g. `0.01` = "the
+worst 1%") — the consumer's policy, never a default or config. Fails loud on any
+non-finite draw — NaN or ±inf (numpy sorts them *last*, so a naive top-`k` mean silently
+selects them — register C-56), on empty `tails`, and on any `t` outside `(0, 1]`.
+
+Written explicitly in its own module — it does **not** share a "tail reducer"
+abstraction with `quantiles`/`exceedance` (the duplication is shallow and the concerns
+change independently — WET before DRY). It reuses only the stable `_common` primitives.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+from numpy.typing import NDArray
+
+from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
+
+
+def _expected_shortfall(
+    values: NDArray[np.float32], tails: NDArray[np.float64]
+) -> NDArray[np.float32]:
+    """Mean of the worst ``⌈t·S⌉`` draws of the trailing (sample) axis, per tail.
+
+    ``values`` is ``(…, S)``, ``tails`` is ``(K,)`` in ``(0, 1]``; returns ``(…, K)``.
+
+    Fails loud on any **non-finite** draw (NaN or ±inf): numpy sorts NaN *last*, so a
+    top-`k` mean would silently select the NaNs — a NaN/garbage worst-case; and an ±inf
+    draw (always an upstream bug) sorts last and contaminates the tail mean to ±inf — a
+    degenerate "worst case". A draw must be a usable finite number (register C-56).
+    """
+    if not bool(np.isfinite(values).all()):
+        raise ValueError(
+            "expected_shortfall is undefined on non-finite draws (NaN or ±inf); strip "
+            "or impute upstream (numpy sorts NaN/+inf last, so the worst-k mean would "
+            "silently select them — a NaN/inf worst-case; register C-56)"
+        )
+    s = values.shape[-1]
+    srt = np.sort(values, axis=-1)  # ascending: the worst (largest) draws are last
+    out = np.empty((*values.shape[:-1], tails.shape[0]), dtype=np.float32)
+    for i in range(tails.shape[0]):
+        k = int(np.ceil(float(tails[i]) * s))  # number of worst draws to average (≥ 1)
+        out[..., i] = srt[..., s - k :].mean(axis=-1)
+    return out
+
+
+def expected_shortfall(
+    frame: AnyFrame,
+    tails: Sequence[float],
+    *,
+    block_rows: int = ROW_BLOCK,
+) -> NDArray[np.float32]:
+    """Per-row worst-case **expected shortfall** over the sample axis → `(N, …, K)`.
+
+    For each upper-tail fraction ``t`` in ``tails`` (required, in ``(0, 1]``), the mean
+    of the worst ``⌈t·S⌉`` draws — the average of the worst-case scenarios. An
+    index-aligned numpy array (the caller holds the index), block-applied in row-blocks.
+
+    Raises:
+        ValueError: ``tails`` is empty (no default), any ``t ∉ (0, 1]`` (a tail with no
+            samples), or any draw is non-finite — NaN or ±inf (register C-56).
+    """
+    thr = np.asarray(tails, dtype=np.float64)
+    if thr.size == 0:
+        raise ValueError(
+            "expected_shortfall requires at least one tail level; there is no default "
+            "(the tail fraction is an input in (0, 1], not a tunable — ADR-022)."
+        )
+    if bool(((thr <= 0.0) | (thr > 1.0)).any()):
+        raise ValueError(
+            f"expected_shortfall tails must be in (0, 1]; got {thr.tolist()} "
+            "(a tail <= 0 contains no samples; > 1 is not a fraction)."
+        )
+
+    def _block(vals: NDArray[np.float32]) -> NDArray[np.float32]:
+        return _expected_shortfall(vals, thr)
+
+    out = block_apply(frame.values, block_rows, _block)
+    return np.asarray(out, dtype=np.float32)