views-frames 1.3.0__tar.gz → 1.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {views_frames-1.3.0 → views_frames-1.7.0}/PKG-INFO +51 -4
  2. {views_frames-1.3.0 → views_frames-1.7.0}/README.md +50 -3
  3. {views_frames-1.3.0 → views_frames-1.7.0}/pyproject.toml +7 -2
  4. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/conformance/__init__.py +40 -10
  5. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/metadata.py +7 -0
  6. views_frames-1.7.0/src/views_frames_reconcile/__init__.py +11 -0
  7. views_frames-1.7.0/src/views_frames_reconcile/conformance.py +74 -0
  8. views_frames-1.7.0/src/views_frames_reconcile/frames.py +69 -0
  9. views_frames-1.7.0/src/views_frames_reconcile/grouping.py +92 -0
  10. views_frames-1.7.0/src/views_frames_reconcile/module.py +68 -0
  11. views_frames-1.7.0/src/views_frames_reconcile/proportional.py +74 -0
  12. views_frames-1.7.0/src/views_frames_reconcile/validation.py +79 -0
  13. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/__init__.py +5 -0
  14. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/conformance.py +32 -0
  15. views_frames-1.7.0/src/views_frames_summarize/exceedance.py +96 -0
  16. views_frames-1.7.0/src/views_frames_summarize/expected_shortfall.py +88 -0
  17. views_frames-1.7.0/src/views_frames_summarize/py.typed +0 -0
  18. {views_frames-1.3.0 → views_frames-1.7.0}/.gitignore +0 -0
  19. {views_frames-1.3.0 → views_frames-1.7.0}/LICENSE +0 -0
  20. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/__init__.py +0 -0
  21. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/_typing.py +0 -0
  22. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/_validation.py +0 -0
  23. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/feature_frame.py +0 -0
  24. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/index.py +0 -0
  25. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/io/__init__.py +0 -0
  26. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/io/arrow.py +0 -0
  27. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/io/npz.py +0 -0
  28. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/prediction_frame.py +0 -0
  29. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/protocols.py +0 -0
  30. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/py.typed +0 -0
  31. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/spatial_level.py +0 -0
  32. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames/target_frame.py +0 -0
  33. {views_frames-1.3.0/src/views_frames_summarize → views_frames-1.7.0/src/views_frames_reconcile}/py.typed +0 -0
  34. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/_common.py +0 -0
  35. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/aggregate.py +0 -0
  36. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/bimodality.py +0 -0
  37. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/collapse.py +0 -0
  38. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/config.py +0 -0
  39. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/interval.py +0 -0
  40. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/point.py +0 -0
  41. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/summarize_tower.py +0 -0
  42. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/tower.py +0 -0
  43. {views_frames-1.3.0 → views_frames-1.7.0}/src/views_frames_summarize/tower_point.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: views-frames
3
- Version: 1.3.0
3
+ Version: 1.7.0
4
4
  Summary: The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG).
5
5
  Project-URL: Homepage, https://github.com/views-platform/views-frames
6
6
  Project-URL: Repository, https://github.com/views-platform/views-frames
@@ -30,15 +30,22 @@ Description-Content-Type: text/markdown
30
30
  > containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
31
31
  > that every other repo depends on and that depends on nothing internal.
32
32
  >
33
- > **Status:** **v1.3.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
33
+ > **Status:** **v1.7.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
34
34
  > purely additive — the coherent posterior summary, ADR-019; v1.2.0 rebuilt the tower
35
35
  > `outside-in`, C-44; v1.3.0 makes the tower summary distribution-agnostic — no magnitude
36
- > zeroing by default, register C-45). This README is the design
36
+ > zeroing by default, register C-45; v1.4.0 adds generic provenance to `FrameMetadata`
37
+ > (`run_id`/`data_version`) and publishes the shared `assert_frame_envelope` checker,
38
+ > ADR-020; v1.5.0 adds the threshold **exceedance** estimator `P(Y > c)`, ADR-021; v1.6.0
39
+ > adds the worst-case **expected_shortfall** estimator, ADR-022; v1.7.0 adds a third
40
+ > sibling package **`views_frames_reconcile`** — forecast reconciliation, ADR-023). This
41
+ > README is the design
37
42
  > bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
38
43
  > io, conformance suite) plus the `src/views_frames_summarize/` sibling package
39
44
  > (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators
40
45
  > `hdi_tower`/`tower_point`/`bimodality`/`summarize_tower`, + cross-level
41
- > aggregation; ADR-017, ADR-019). The blocking design decisions are resolved (§13a) and
46
+ > aggregation; ADR-017, ADR-019) and the `src/views_frames_reconcile/` sibling package
47
+ > (forecast reconciliation — make grid forecasts sum to country totals per draw; ADR-023).
48
+ > The blocking design decisions are resolved (§13a) and
42
49
  > ratified as ADRs 011–018; two rounds of consumer review validated
43
50
  > the design. Consumer adoption (re-export shims, pandas migration) is the owner's
44
51
  > migration, not this repo's.
@@ -417,6 +424,14 @@ views-frames/
417
424
  │ ├── point.py # map_estimate (histogram MAP)
418
425
  │ ├── interval.py # hdi, quantiles → arrays aligned to the frame index
419
426
  │ └── aggregate.py # conservation-correct cross-level aggregation
427
+ ├── src/views_frames_reconcile/ # forecast reconciliation OVER frames (ADR-023)
428
+ │ ├── __init__.py # depends on views_frames + numpy only; never the reverse
429
+ │ ├── proportional.py # reconcile_proportional — per-draw top-down scaling
430
+ │ ├── grouping.py # reconcile_pgm_to_cm — group by (time, country)
431
+ │ ├── frames.py # prediction_frame_from_arrays adapter
432
+ │ ├── validation.py # fail-loud input guards
433
+ │ ├── module.py # ReconciliationModule (holds the injected mapping)
434
+ │ └── conformance.py # assert_reconcile_contract
420
435
  └── tests/
421
436
  ├── conformance/ # the published contract suite consumers re-run (see §9)
422
437
  └── unit/
@@ -439,6 +454,38 @@ Layout rules (these *are* the screaming-architecture requirements):
439
454
  - A new developer should infer every responsibility from the file tree without
440
455
  reading bodies.
441
456
 
457
+ ### 6a. The `views_frames_reconcile` sibling (ADR-023)
458
+
459
+ Forecast **reconciliation** — making grid (`pgm`) predictions sum to their country
460
+ (`cm`) totals — is a numpy-only frame→frame operation, structurally the same kind of
461
+ thing as `views_frames_summarize` (ADR-017). It is its own sibling package, **not** in
462
+ the leaf and **not** in views-postprocessing (its old, mis-homed host).
463
+
464
+ **Charter.** **May:** forecast-reconciliation algorithms on frames (the per-draw
465
+ top-down proportional method; future methods as sibling modules — the principled
466
+ probabilistic upgrade, C-37); per-sample reconciliation; fail-loud validation; a
467
+ conformance suite. **Must not:** IO; **fetch the `(time, unit) → country` mapping** (it
468
+ is *injected* by the caller as numpy arrays, exactly like `cross_level_align` — ADR-014);
469
+ actuals/scoring (views-evaluation); plotting; or any `views_*` import except
470
+ `views_frames`. Import-DAG: `views_frames_reconcile → {views_frames}`.
471
+
472
+ ```python
473
+ import numpy as np
474
+ from views_frames_reconcile import ReconciliationModule
475
+
476
+ # mapping is INJECTED by the caller (sourced from the producer; never fetched here):
477
+ # map_keys: (M, 2) int (time, priogrid_gid) map_vals: (M,) int country_id
478
+ reconciler = ReconciliationModule(map_keys, map_vals)
479
+ pgm_reconciled = reconciler.reconcile(cm_frame, pgm_frame) # new pgm PredictionFrame
480
+ # each (time, country) group's cells now sum, per draw, to the country total
481
+ # (all-zero grid draws stay zero); zeros preserved; non-negative.
482
+ ```
483
+
484
+ > **Future DRY pass (deferred).** `grouping.py` labels grid rows by country with the
485
+ > leaf's `cross_level_align_arrays` and overlaps `cross_level_align` (index.py). Folding
486
+ > the two is a separate later story — the v1.7.0 relocation is a faithful **WET** move,
487
+ > proven bit-identical to the original before any refactor.
488
+
442
489
  ---
443
490
 
444
491
  ## 7. On-disk / serialization contract (where "doesn't scale" is actually decided)
@@ -4,15 +4,22 @@
4
4
  > containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
5
5
  > that every other repo depends on and that depends on nothing internal.
6
6
  >
7
- > **Status:** **v1.3.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
7
+ > **Status:** **v1.7.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1 surface is
8
8
  > purely additive — the coherent posterior summary, ADR-019; v1.2.0 rebuilt the tower
9
9
  > `outside-in`, C-44; v1.3.0 makes the tower summary distribution-agnostic — no magnitude
10
- > zeroing by default, register C-45). This README is the design
10
+ > zeroing by default, register C-45; v1.4.0 adds generic provenance to `FrameMetadata`
11
+ > (`run_id`/`data_version`) and publishes the shared `assert_frame_envelope` checker,
12
+ > ADR-020; v1.5.0 adds the threshold **exceedance** estimator `P(Y > c)`, ADR-021; v1.6.0
13
+ > adds the worst-case **expected_shortfall** estimator, ADR-022; v1.7.0 adds a third
14
+ > sibling package **`views_frames_reconcile`** — forecast reconciliation, ADR-023). This
15
+ > README is the design
11
16
  > bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
12
17
  > io, conformance suite) plus the `src/views_frames_summarize/` sibling package
13
18
  > (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators
14
19
  > `hdi_tower`/`tower_point`/`bimodality`/`summarize_tower`, + cross-level
15
- > aggregation; ADR-017, ADR-019). The blocking design decisions are resolved (§13a) and
20
+ > aggregation; ADR-017, ADR-019) and the `src/views_frames_reconcile/` sibling package
21
+ > (forecast reconciliation — make grid forecasts sum to country totals per draw; ADR-023).
22
+ > The blocking design decisions are resolved (§13a) and
16
23
  > ratified as ADRs 011–018; two rounds of consumer review validated
17
24
  > the design. Consumer adoption (re-export shims, pandas migration) is the owner's
18
25
  > migration, not this repo's.
@@ -391,6 +398,14 @@ views-frames/
391
398
  │ ├── point.py # map_estimate (histogram MAP)
392
399
  │ ├── interval.py # hdi, quantiles → arrays aligned to the frame index
393
400
  │ └── aggregate.py # conservation-correct cross-level aggregation
401
+ ├── src/views_frames_reconcile/ # forecast reconciliation OVER frames (ADR-023)
402
+ │ ├── __init__.py # depends on views_frames + numpy only; never the reverse
403
+ │ ├── proportional.py # reconcile_proportional — per-draw top-down scaling
404
+ │ ├── grouping.py # reconcile_pgm_to_cm — group by (time, country)
405
+ │ ├── frames.py # prediction_frame_from_arrays adapter
406
+ │ ├── validation.py # fail-loud input guards
407
+ │ ├── module.py # ReconciliationModule (holds the injected mapping)
408
+ │ └── conformance.py # assert_reconcile_contract
394
409
  └── tests/
395
410
  ├── conformance/ # the published contract suite consumers re-run (see §9)
396
411
  └── unit/
@@ -413,6 +428,38 @@ Layout rules (these *are* the screaming-architecture requirements):
413
428
  - A new developer should infer every responsibility from the file tree without
414
429
  reading bodies.
415
430
 
431
+ ### 6a. The `views_frames_reconcile` sibling (ADR-023)
432
+
433
+ Forecast **reconciliation** — making grid (`pgm`) predictions sum to their country
434
+ (`cm`) totals — is a numpy-only frame→frame operation, structurally the same kind of
435
+ thing as `views_frames_summarize` (ADR-017). It is its own sibling package, **not** in
436
+ the leaf and **not** in views-postprocessing (its old, mis-homed host).
437
+
438
+ **Charter.** **May:** forecast-reconciliation algorithms on frames (the per-draw
439
+ top-down proportional method; future methods as sibling modules — the principled
440
+ probabilistic upgrade, C-37); per-sample reconciliation; fail-loud validation; a
441
+ conformance suite. **Must not:** IO; **fetch the `(time, unit) → country` mapping** (it
442
+ is *injected* by the caller as numpy arrays, exactly like `cross_level_align` — ADR-014);
443
+ actuals/scoring (views-evaluation); plotting; or any `views_*` import except
444
+ `views_frames`. Import-DAG: `views_frames_reconcile → {views_frames}`.
445
+
446
+ ```python
447
+ import numpy as np
448
+ from views_frames_reconcile import ReconciliationModule
449
+
450
+ # mapping is INJECTED by the caller (sourced from the producer; never fetched here):
451
+ # map_keys: (M, 2) int (time, priogrid_gid) map_vals: (M,) int country_id
452
+ reconciler = ReconciliationModule(map_keys, map_vals)
453
+ pgm_reconciled = reconciler.reconcile(cm_frame, pgm_frame) # new pgm PredictionFrame
454
+ # each (time, country) group's cells now sum, per draw, to the country total
455
+ # (all-zero grid draws stay zero); zeros preserved; non-negative.
456
+ ```
457
+
458
+ > **Future DRY pass (deferred).** `grouping.py` labels grid rows by country with the
459
+ > leaf's `cross_level_align_arrays` and overlaps `cross_level_align` (index.py). Folding
460
+ > the two is a separate later story — the v1.7.0 relocation is a faithful **WET** move,
461
+ > proven bit-identical to the original before any refactor.
462
+
416
463
  ---
417
464
 
418
465
  ## 7. On-disk / serialization contract (where "doesn't scale" is actually decided)
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "views-frames"
7
- version = "1.3.0"
7
+ version = "1.7.0"
8
8
  description = "The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG)."
9
9
  authors = [
10
10
  { name = "Simon Polichinel von der Maase", email = "simmaa@prio.org" },
@@ -52,7 +52,7 @@ dev = [
52
52
  ]
53
53
 
54
54
  [tool.hatch.build.targets.wheel]
55
- packages = ["src/views_frames", "src/views_frames_summarize"]
55
+ packages = ["src/views_frames", "src/views_frames_summarize", "src/views_frames_reconcile"]
56
56
 
57
57
  [tool.hatch.build.targets.sdist]
58
58
  include = ["/src"]
@@ -87,6 +87,11 @@ select = ["E", "F", "I", "N", "UP", "B", "A", "C4", "SIM"]
87
87
  # The falsification stubs assert against the README via long, literal regexes
88
88
  # whose lines cannot wrap without changing their semantics.
89
89
  "tests/test_falsification_*.py" = ["E501"]
90
+ # The reconciliation parity tests + their oracle-regeneration scripts are
91
+ # faithfully relocated from views-postprocessing (Epic 11 / ADR-023); their dense
92
+ # numpy/dict-comprehension assertions are kept verbatim rather than re-wrapped.
93
+ "tests/test_reconciliation_*.py" = ["E501"]
94
+ "scripts/gen_reconciliation_*.py" = ["E501"]
90
95
 
91
96
  [tool.mypy]
92
97
  python_version = "3.10"
@@ -10,6 +10,14 @@ Usage in a consumer's test::
10
10
  from views_frames.conformance import assert_frame_contract
11
11
  assert_frame_contract(my_adapter_output())
12
12
 
13
+ A consumer whose type is **not** spatiotemporal (a string-keyed evaluation output, e.g.
14
+ views-evaluation's ``MetricFrame``) validates against the shared envelope instead, so
15
+ the envelope has one written authority rather than re-asserted copies that drift
16
+ (ADR-020, register C-46)::
17
+
18
+ from views_frames.conformance import assert_frame_envelope
19
+ assert_frame_envelope(my_metric_frame())
20
+
13
21
  The floor is governed in ``GOVERNANCE.md``; ``CONFORMANCE_FLOOR`` records the version
14
22
  this suite belongs to.
15
23
  """
@@ -27,20 +35,26 @@ __all__ = [
27
35
  "CONFORMANCE_FLOOR",
28
36
  "assert_cross_level_alignment_law",
29
37
  "assert_frame_contract",
38
+ "assert_frame_envelope",
30
39
  "assert_index_alignment_laws",
31
40
  ]
32
41
 
33
42
 
34
- def assert_frame_contract(frame: Any) -> None:
35
- """Assert ``frame`` satisfies the views-frames data contract.
43
+ def assert_frame_envelope(frame: Any) -> None:
44
+ """Assert ``frame`` satisfies the shared **frame envelope**.
45
+
46
+ The envelope is the subset of the contract that applies to *any* frame-like value
47
+ object, spatiotemporal or not: float32 values with no object dtype, an explicit
48
+ trailing axis, rows equal to ``n_rows``, and a save/load round-trip that preserves
49
+ the values and every identifier array. It deliberately says nothing about *which*
50
+ identifiers exist or their dtype — that is the spatiotemporal layer's concern.
36
51
 
37
- Checks the structural invariants (float32 values, no object dtype, an explicit
38
- trailing axis, complete integer identifiers of length ``n_rows``) and the
39
- save/load round-trip. (Sample-axis reduction is the ``views_frames_summarize``
40
- package's concern, not the contract's — ADR-017.)
52
+ This is the single written authority for the envelope so a sibling that re-asserts
53
+ it (views-evaluation's ``MetricFrame``, keyed by string axes — not ``(time, unit)``)
54
+ validates against this checker rather than drifting from it (ADR-020, C-46).
41
55
 
42
56
  Raises:
43
- AssertionError: any part of the contract is violated.
57
+ AssertionError: any part of the envelope is violated.
44
58
  """
45
59
  values = frame.values
46
60
  assert isinstance(values, np.ndarray), "values must be a numpy array"
@@ -49,6 +63,22 @@ def assert_frame_contract(frame: Any) -> None:
49
63
  assert values.ndim >= 2, "values must have an explicit trailing sample axis"
50
64
  assert values.shape[0] == frame.n_rows, "values rows must equal n_rows"
51
65
 
66
+ _assert_roundtrip(frame)
67
+
68
+
69
+ def assert_frame_contract(frame: Any) -> None:
70
+ """Assert ``frame`` satisfies the full views-frames **spatiotemporal** contract.
71
+
72
+ The shared envelope (:func:`assert_frame_envelope`) plus the spatiotemporal
73
+ requirement: complete integer ``time``/``unit`` identifiers of length ``n_rows``.
74
+ (Sample-axis reduction is the ``views_frames_summarize`` package's concern, not the
75
+ contract's — ADR-017.)
76
+
77
+ Raises:
78
+ AssertionError: the envelope or the spatiotemporal identifier rule is violated.
79
+ """
80
+ assert_frame_envelope(frame)
81
+
52
82
  ids = frame.identifiers
53
83
  for key in ("time", "unit"):
54
84
  assert key in ids, f"missing required identifier '{key}'"
@@ -56,14 +86,14 @@ def assert_frame_contract(frame: Any) -> None:
56
86
  assert np.issubdtype(arr.dtype, np.integer), f"'{key}' must be integer"
57
87
  assert arr.shape == (frame.n_rows,), f"'{key}' must be length n_rows"
58
88
 
59
- _assert_roundtrip(frame)
60
-
61
89
 
62
90
  def _assert_roundtrip(frame: Any) -> None:
63
91
  with tempfile.TemporaryDirectory() as directory:
64
92
  frame.save(directory)
65
93
  loaded = type(frame).load(directory)
66
- assert np.array_equal(loaded.values, frame.values), "save/load changed values"
94
+ assert np.array_equal(loaded.values, frame.values, equal_nan=True), (
95
+ "save/load changed values"
96
+ )
67
97
  for key, arr in frame.identifiers.items():
68
98
  assert np.array_equal(loaded.identifiers[key], arr), (
69
99
  f"save/load changed identifier '{key}'"
@@ -3,6 +3,11 @@
3
3
  Not a free-form dict: a frozen dataclass with all-optional, validated fields, so
4
4
  adding a field is a MINOR change and consumers cannot diverge on key names (the
5
5
  store-side cause of reporting's C-48). It is the typed home for run/eval identity.
6
+
7
+ Provenance fields are **generic only** (``run_id``, ``data_version``): meaningful for
8
+ any frame. Evaluation-specific provenance (``scoring_code_version``, full-precision
9
+ ``evaluation_timestamp``) stays in views-evaluation's ``MetricFrame`` and must not leak
10
+ into this generic header (ADR-020, register C-47).
6
11
  """
7
12
 
8
13
  from __future__ import annotations
@@ -20,6 +25,8 @@ class FrameMetadata:
20
25
  run_type: str | None = None
21
26
  timestamp: int | None = None
22
27
  seed: int | None = None
28
+ run_id: str | None = None
29
+ data_version: str | None = None
23
30
 
24
31
  def to_dict(self) -> dict[str, Any]:
25
32
  """Serialize to a plain dict, omitting unset (``None``) fields."""
@@ -0,0 +1,11 @@
1
+ """Forecast reconciliation (pgm forecasts reconciled to cm totals).
2
+
3
+ Slice 1 ports the top-down proportional method from views-reporting as a pure
4
+ numpy function. New methods (e.g. principled probabilistic reconciliation, C-37)
5
+ should be added as sibling modules, not by modifying ``proportional``.
6
+ """
7
+
8
+ from views_frames_reconcile.module import ReconciliationModule
9
+ from views_frames_reconcile.proportional import reconcile_proportional
10
+
11
+ __all__ = ["ReconciliationModule", "reconcile_proportional"]
@@ -0,0 +1,74 @@
1
+ """Conformance checks for the reconcile package (ADR-023).
2
+
3
+ A consumer can re-run these against its own frame factories to confirm the reconciler
4
+ behaves: grid (``pgm``) predictions sum, per draw, to their country (``cm``) totals
5
+ (except all-zero grid draws, which stay zero — the documented edge); zeros preserved;
6
+ values stay non-negative; the output is a same-shape ``pgm`` ``PredictionFrame`` at PGM
7
+ level; and the cm/pgm mapping is **injected, never fetched** (ADR-014/ADR-023).
8
+
9
+ Mirrors ``views_frames_summarize/conformance.py:assert_summarizer_contract``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from typing import Any
15
+
16
+ import numpy as np
17
+ from numpy.typing import NDArray
18
+
19
+ from views_frames import PredictionFrame, SpatialLevel
20
+ from views_frames_reconcile.module import ReconciliationModule
21
+
22
+
23
+ def assert_reconcile_contract(
24
+ cm_frame: PredictionFrame,
25
+ pgm_frame: PredictionFrame,
26
+ map_keys: NDArray[np.integer[Any]] | object,
27
+ map_vals: NDArray[np.integer[Any]] | object,
28
+ ) -> None:
29
+ """Assert the reconciler obeys its contract on ``(cm_frame, pgm_frame, mapping)``.
30
+
31
+ The mapping is **injected** (``map_keys``/``map_vals`` arrays) — never fetched
32
+ (enforced by the signature + the import-DAG; honoured here by the sum-to-country
33
+ law, which only holds if each cell is grouped under its injected country).
34
+
35
+ Raises:
36
+ AssertionError: a contract law is violated.
37
+ """
38
+ mk = np.asarray(map_keys)
39
+ mv = np.asarray(map_vals)
40
+ out = ReconciliationModule(mk, mv).reconcile(cm_frame, pgm_frame)
41
+
42
+ # 1. Output type / shape / level / index: a same-shape pgm PredictionFrame at PGM.
43
+ assert type(out) is type(pgm_frame), "reconcile must return the pgm frame type"
44
+ assert out.values.shape == pgm_frame.values.shape, "reconcile must preserve (N, S)"
45
+ assert out.index.level is SpatialLevel.PGM, "reconciled frame stays at PGM level"
46
+ assert np.array_equal(out.index.time, pgm_frame.index.time), "time index preserved"
47
+ assert np.array_equal(out.index.unit, pgm_frame.index.unit), "unit index preserved"
48
+
49
+ # 2. Non-negativity.
50
+ assert bool((out.values >= 0).all()), "reconciled forecasts must be non-negative"
51
+
52
+ # 3. Zero-preservation: an input zero cell stays zero.
53
+ assert bool((out.values[pgm_frame.values == 0] == 0).all()), "zeros preserved"
54
+
55
+ # 4. Sum-to-country per draw: each (time, country) group's cells sum, per draw, to
56
+ # its country total — except all-zero input draws, which stay zero (the edge).
57
+ cm_units = pgm_frame.index.cross_level_align_arrays(mk, mv, SpatialLevel.CM).unit
58
+ pg_time = pgm_frame.index.time
59
+ cm_pos = {
60
+ (int(t), int(u)): j
61
+ for j, (t, u) in enumerate(
62
+ zip(cm_frame.index.time, cm_frame.index.unit, strict=True)
63
+ )
64
+ }
65
+ for t, c in np.unique(np.stack([pg_time, cm_units], axis=1), axis=0):
66
+ rows = (pg_time == t) & (cm_units == c)
67
+ in_sum = pgm_frame.values[rows].sum(axis=0) # (S,)
68
+ out_sum = out.values[rows].sum(axis=0) # (S,)
69
+ total = cm_frame.values[cm_pos[(int(t), int(c))]] # (S,)
70
+ active = in_sum != 0
71
+ np.testing.assert_allclose(
72
+ out_sum[active], total[active], rtol=1e-4, atol=1e-3
73
+ )
74
+ assert bool((out_sum[~active] == 0).all()), "all-zero draws stay zero"
@@ -0,0 +1,69 @@
1
+ """Array → `PredictionFrame` adapters for reconciliation (epic #31, story #33).
2
+
3
+ Reconciliation works on views-frames `PredictionFrame`s at two spatial levels:
4
+ country (`cm`) and PRIO-GRID (`pgm`). Both are built the same way from
5
+ `(time, unit, values)` arrays — only the `SpatialLevel` and the unit identifier
6
+ (`country_id` vs `priogrid_gid`) differ. Predictions here carry a real posterior
7
+ sample axis, so values are `(N, S)` with `S >= 1`.
8
+
9
+ This is the reconciliation package's **own** I/O: it differs from the unfao
10
+ delivery adapters (`unfao/frames.py`), which wrap pandas *scalar* columns as
11
+ `(N, 1)` point frames — so per CRP they are not forced together. numpy +
12
+ views-frames only; no torch, no pandas.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any
18
+
19
+ import numpy as np
20
+ from numpy.typing import NDArray
21
+
22
+ from views_frames import (
23
+ FrameMetadata,
24
+ PredictionFrame,
25
+ SpatialLevel,
26
+ SpatioTemporalIndex,
27
+ )
28
+
29
+
30
+ def prediction_frame_from_arrays(
31
+ time: NDArray[np.integer[Any]] | object,
32
+ unit: NDArray[np.integer[Any]] | object,
33
+ values: NDArray[np.floating[Any]] | object,
34
+ *,
35
+ level: SpatialLevel,
36
+ metadata: FrameMetadata | None = None,
37
+ ) -> PredictionFrame:
38
+ """Build a `PredictionFrame` from `(time, unit, values)` at ``level``.
39
+
40
+ Args:
41
+ time: 1-D integer array, length ``N`` (``month_id``).
42
+ unit: 1-D integer array, length ``N`` — ``country_id`` for
43
+ ``SpatialLevel.CM``, ``priogrid_gid`` for ``SpatialLevel.PGM``.
44
+ values: ``(N, S)`` float32-coercible array of posterior samples.
45
+ level: the frame's spatial level.
46
+
47
+ Returns:
48
+ A `PredictionFrame` of shape ``(N, S)`` at ``level``. The values buffer
49
+ is reused without copy when already float32 (views-frames C-07); the
50
+ input arrays are never mutated.
51
+
52
+ Raises:
53
+ ValueError: ``values`` is not 2-D, or ``time``/``unit`` are not 1-D of
54
+ length ``N``.
55
+ """
56
+ time_arr = np.asarray(time, dtype=np.int64)
57
+ unit_arr = np.asarray(unit, dtype=np.int64)
58
+ vals = np.asarray(values, dtype=np.float32)
59
+
60
+ if vals.ndim != 2:
61
+ raise ValueError(f"values must be 2-D (N, S); got ndim={vals.ndim}")
62
+ if time_arr.shape != (vals.shape[0],) or unit_arr.shape != (vals.shape[0],):
63
+ raise ValueError(
64
+ f"time {time_arr.shape} and unit {unit_arr.shape} must both be 1-D "
65
+ f"of length N={vals.shape[0]}"
66
+ )
67
+
68
+ index = SpatioTemporalIndex(time=time_arr, unit=unit_arr, level=level)
69
+ return PredictionFrame(vals, index, metadata)
@@ -0,0 +1,92 @@
1
+ """Reconcile a pgm `PredictionFrame` to cm country totals (epic #31, story #34).
2
+
3
+ The heart of the migration: for each `(time, country)`, scale that country's grid
4
+ cells so their per-draw sum matches the country forecast, using the parity-proven
5
+ leaf `reconcile_proportional` (PR #30). Grid rows are labelled by country with
6
+ views-frames `cross_level_align` — the sanctioned cm↔pgm primitive, which fails
7
+ loud if any grid row lacks a country (mirrors the original's "valid countries"
8
+ guard). De-mutated: returns a **new** pgm frame (C-184); the input is untouched.
9
+
10
+ numpy + views-frames only. The loop is over `(time, country)` groups (a small
11
+ number), not over rows; each group call is fully vectorised over cells × samples.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from typing import Any
17
+
18
+ import numpy as np
19
+ from numpy.typing import NDArray
20
+
21
+ from views_frames import PredictionFrame, SpatialLevel
22
+ from views_frames_reconcile.proportional import reconcile_proportional
23
+
24
+
25
+ def reconcile_pgm_to_cm(
26
+ pgm_frame: PredictionFrame,
27
+ cm_frame: PredictionFrame,
28
+ map_keys: NDArray[np.integer[Any]] | object,
29
+ map_vals: NDArray[np.integer[Any]] | object,
30
+ ) -> PredictionFrame:
31
+ """Return a new pgm `PredictionFrame` reconciled to ``cm_frame``'s totals.
32
+
33
+ Args:
34
+ pgm_frame: grid forecasts at PGM level, values ``(N_pg, S)``.
35
+ cm_frame: country forecasts at CM level, values ``(N_cm, S)``.
36
+ map_keys: ``(M, 2)`` int ``(time, priogrid_gid)`` covering every pgm row.
37
+ map_vals: ``(M,)`` int ``country_id`` for each key (injected; geography is
38
+ never embedded here — views-frames ADR-014).
39
+
40
+ Returns:
41
+ A new pgm `PredictionFrame` (same index/metadata as ``pgm_frame``) whose
42
+ cells sum, per draw, to the country forecast — except all-zero country
43
+ draws, which stay zero (the leaf's documented edge case).
44
+
45
+ Raises:
46
+ ValueError: a grid row has no country mapping (raised by
47
+ ``cross_level_align``), or a ``(time, country)`` group has no matching
48
+ country forecast in ``cm_frame``.
49
+ """
50
+ # 1. Label every grid row with its country (cm-level units); fails loud if a
51
+ # row's (time, priogrid) is absent from the injected mapping.
52
+ cm_units = pgm_frame.index.cross_level_align_arrays(
53
+ np.asarray(map_keys), np.asarray(map_vals), SpatialLevel.CM
54
+ ).unit # (N_pg,)
55
+ pg_time = pgm_frame.index.time
56
+
57
+ # 2. (time, country) -> row position in the country frame.
58
+ cm_time = cm_frame.index.time
59
+ cm_unit = cm_frame.index.unit
60
+ cm_vals = cm_frame.values
61
+ cm_pos = {(int(cm_time[j]), int(cm_unit[j])): j for j in range(cm_frame.n_rows)}
62
+
63
+ # 3. Group grid rows by (time, country) and reconcile each group with the leaf.
64
+ # Group-by-sort: a single argsort lays the rows out contiguously per group
65
+ # (O(N log N), one index array) — no per-group full-array scan — so it
66
+ # scales to the full grid (register C-38). Group iteration order is
67
+ # irrelevant: each group writes only its own rows.
68
+ pg_vals = pgm_frame.values
69
+ out = np.empty_like(pg_vals)
70
+ group_key = np.stack([pg_time, cm_units], axis=1) # (N_pg, 2)
71
+ unique_groups, inverse, counts = np.unique(
72
+ group_key, axis=0, return_inverse=True, return_counts=True
73
+ )
74
+ inverse = np.asarray(inverse).reshape(-1)
75
+ order = np.argsort(inverse, kind="stable") # rows grouped contiguously by group
76
+ # group gi -> order[bounds[gi]:bounds[gi + 1]]
77
+ bounds = np.concatenate(([0], np.cumsum(counts)))
78
+
79
+ for gi in range(unique_groups.shape[0]):
80
+ t, c = int(unique_groups[gi, 0]), int(unique_groups[gi, 1])
81
+ if (t, c) not in cm_pos:
82
+ raise ValueError(
83
+ f"grid group (time={t}, country={c}) has no country forecast "
84
+ "in cm_frame"
85
+ )
86
+ rows = order[bounds[gi]:bounds[gi + 1]]
87
+ country_total = cm_vals[cm_pos[(t, c)]] # (S,)
88
+ # leaf convention: grid is (samples, cells); our frame slice is (cells, samples)
89
+ scaled = reconcile_proportional(pg_vals[rows].T, country_total) # (S, n_cells)
90
+ out[rows] = scaled.T # back to (n_cells, S)
91
+
92
+ return PredictionFrame(out, pgm_frame.index, pgm_frame.metadata)
@@ -0,0 +1,68 @@
1
+ """Frames-native reconciliation orchestration (epic #31, story #36).
2
+
3
+ `ReconciliationModule` holds the injected `(time, priogrid_gid) -> country_id`
4
+ mapping (geography is injected, never embedded — views-frames ADR-014) and
5
+ applies it: `reconcile(cm_frame, pgm_frame)` validates the inputs and scales the
6
+ grid forecasts to the country totals, returning a **new** pgm frame (de-mutated,
7
+ C-184).
8
+
9
+ **SRP:** orchestration only — the scaling math is the leaf (`proportional`), the
10
+ grouping is `grouping`, the guards are `validation`, the I/O is `frames`. No
11
+ torch, no pandas, no viewser, no wandb: the original's `ProcessPoolExecutor` and
12
+ WandB alerting are dropped (numpy is fast; there is no GPU). If scale ever needs
13
+ parallelism, add it behind this same interface (OCP). Multi-target inputs are
14
+ reconciled by calling `reconcile` once per target.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from typing import Any
20
+
21
+ import numpy as np
22
+ from numpy.typing import NDArray
23
+
24
+ from views_frames import PredictionFrame
25
+ from views_frames_reconcile.grouping import reconcile_pgm_to_cm
26
+ from views_frames_reconcile.validation import validate_reconciliation_inputs
27
+
28
+
29
+ class ReconciliationModule:
30
+ """Reconcile pgm forecasts to cm country totals (one target per call)."""
31
+
32
+ def __init__(
33
+ self,
34
+ map_keys: NDArray[np.integer[Any]] | object,
35
+ map_vals: NDArray[np.integer[Any]] | object,
36
+ ) -> None:
37
+ """Inject the `(time, priogrid_gid) -> country_id` mapping.
38
+
39
+ Args:
40
+ map_keys: ``(M, 2)`` int array of ``(time, priogrid_gid)`` pairs.
41
+ map_vals: ``(M,)`` int ``country_id`` for each key.
42
+
43
+ Raises:
44
+ ValueError: ``map_keys`` is not ``(M, 2)`` or ``map_vals`` is not
45
+ length ``M``.
46
+ """
47
+ keys = np.asarray(map_keys)
48
+ vals = np.asarray(map_vals)
49
+ if keys.ndim != 2 or keys.shape[1] != 2:
50
+ raise ValueError("map_keys must be an (M, 2) array of (time, priogrid_gid)")
51
+ if vals.shape != (keys.shape[0],):
52
+ raise ValueError("map_vals must be a length-M array aligned to map_keys")
53
+ self._map_keys = keys
54
+ self._map_vals = vals
55
+
56
+ def reconcile(
57
+ self, cm_frame: PredictionFrame, pgm_frame: PredictionFrame
58
+ ) -> PredictionFrame:
59
+ """Validate the inputs, then return a new pgm frame reconciled to cm totals.
60
+
61
+ Raises:
62
+ ValueError: the inputs fail validation (level / sample-count / time
63
+ coverage / missing country forecast).
64
+ """
65
+ validate_reconciliation_inputs(
66
+ cm_frame, pgm_frame, self._map_keys, self._map_vals
67
+ )
68
+ return reconcile_pgm_to_cm(pgm_frame, cm_frame, self._map_keys, self._map_vals)
@@ -0,0 +1,74 @@
1
+ """Top-down proportional reconciliation (numpy port — phase 2, slice 1).
2
+
3
+ Makes PRIO-GRID-month (pgm) forecasts sum to their country-month (cm) total by
4
+ **top-down disaggregation using forecast proportions** (FPP3 terminology),
5
+ applied **per posterior draw**: within a draw, each grid cell keeps its relative
6
+ share and the cells are rescaled so they sum to that draw's country total. Zeros
7
+ stay zero; country totals are authoritative; the result is non-negative.
8
+
9
+ This is a *faithful, numpy-only* port of views-reporting's
10
+ ``ForecastReconciler.reconcile_forecast`` (torch), migrated here because the
11
+ algorithm belongs in post-processing, not reporting (views-reporting issue #72).
12
+ It is intentionally the **same** method — a pragmatic per-draw approximation, not
13
+ principled joint probabilistic reconciliation. The upgrade to the latter is
14
+ tracked as **C-37** and is deliberately deferred until this port's parity with
15
+ the original is proven and the move is wired.
16
+
17
+ No torch, no pandas — numpy only.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from typing import Any
23
+
24
+ import numpy as np
25
+ from numpy.typing import NDArray
26
+
27
+ _EPS = np.float32(1e-8)
28
+
29
+
30
+ def reconcile_proportional(
31
+ grid: NDArray[np.floating[Any]] | object,
32
+ country: NDArray[np.floating[Any]] | float | object,
33
+ ) -> NDArray[np.float32]:
34
+ """Rescale grid forecasts so each draw sums to its country total.
35
+
36
+ Args:
37
+ grid: Grid-level forecasts, float32-coercible. Either
38
+ ``(num_samples, num_grid_cells)`` (probabilistic) or
39
+ ``(num_grid_cells,)`` (point).
40
+ country: Country-level total. Either ``(num_samples,)`` (probabilistic)
41
+ or a scalar (point). Must align with ``grid``'s sample axis.
42
+
43
+ Returns:
44
+ Adjusted grid forecasts, float32, same shape as ``grid``. ``sum`` over
45
+ grid cells equals ``country`` per sample; zero cells stay zero; values
46
+ are clamped to be non-negative.
47
+
48
+ Raises:
49
+ ValueError: the grid and country sample counts disagree.
50
+ """
51
+ grid_arr = np.asarray(grid, dtype=np.float32)
52
+ is_point = grid_arr.ndim == 1
53
+
54
+ if is_point:
55
+ grid_arr = grid_arr[np.newaxis, :] # (1, N)
56
+ country_arr = np.asarray([country], dtype=np.float32)
57
+ else:
58
+ country_arr = np.asarray(country, dtype=np.float32).reshape(-1)
59
+
60
+ if grid_arr.shape[0] != country_arr.shape[0]:
61
+ raise ValueError(
62
+ f"Mismatch in sample count: grid has {grid_arr.shape[0]}, "
63
+ f"country has {country_arr.shape[0]}"
64
+ )
65
+
66
+ # Preserve zeros: only strictly-positive cells carry probability mass.
67
+ nonzero = np.where(grid_arr > 0, grid_arr, np.float32(0.0))
68
+
69
+ # Per-draw proportional scaling to the (authoritative) country total.
70
+ sum_nonzero = nonzero.sum(axis=1, keepdims=True) # (S, 1)
71
+ scaling = country_arr.reshape(-1, 1) / (sum_nonzero + _EPS) # (S, 1)
72
+ adjusted = np.clip(nonzero * scaling, 0.0, None).astype(np.float32)
73
+
74
+ return np.asarray(adjusted[0] if is_point else adjusted, dtype=np.float32)
@@ -0,0 +1,79 @@
1
+ """Fail-loud validation for reconciliation inputs (epic #31, story #35).
2
+
3
+ Ports views-reporting `ReconciliationModule.__init__`'s guards to frames-native
4
+ checks, run before any work. SRP: small, independently testable helpers — not
5
+ buried in the orchestrator. The original's intents map as:
6
+
7
+ - dataset type checks -> spatial-level guard (cm@CM, pgm@PGM)
8
+ - same time steps + exact overlap -> identical set of time values
9
+ - (per-draw scaling needs it) -> equal `sample_count`
10
+ - valid countries -> every (time, country) the grid maps to has
11
+ a country forecast in the cm frame
12
+
13
+ The original's "different time units" (e.g. month_id vs year_id) check is
14
+ subsumed by the level guard; per-target intersection is handled by the
15
+ orchestrator, since frames here are single-target.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from typing import Any
21
+
22
+ import numpy as np
23
+ from numpy.typing import NDArray
24
+
25
+ from views_frames import PredictionFrame, SpatialLevel
26
+
27
+
28
+ def validate_reconciliation_inputs(
29
+ cm_frame: PredictionFrame,
30
+ pgm_frame: PredictionFrame,
31
+ map_keys: NDArray[np.integer[Any]] | object,
32
+ map_vals: NDArray[np.integer[Any]] | object,
33
+ ) -> None:
34
+ """Raise ``ValueError`` if the reconciliation inputs are inconsistent.
35
+
36
+ Checks spatial levels, sample-count alignment, identical time coverage, and
37
+ that every country the grid maps to has a forecast in ``cm_frame``.
38
+ """
39
+ if cm_frame.index.level is not SpatialLevel.CM:
40
+ raise ValueError(
41
+ f"country frame must be at SpatialLevel.CM, got {cm_frame.index.level}"
42
+ )
43
+ if pgm_frame.index.level is not SpatialLevel.PGM:
44
+ raise ValueError(
45
+ f"grid frame must be at SpatialLevel.PGM, got {pgm_frame.index.level}"
46
+ )
47
+ if cm_frame.sample_count != pgm_frame.sample_count:
48
+ raise ValueError(
49
+ f"sample-count mismatch: cm has {cm_frame.sample_count}, "
50
+ f"pgm has {pgm_frame.sample_count}"
51
+ )
52
+
53
+ cm_times = {int(t) for t in np.unique(cm_frame.index.time)}
54
+ pg_times = {int(t) for t in np.unique(pgm_frame.index.time)}
55
+ if cm_times != pg_times:
56
+ raise ValueError(
57
+ "cm and pgm cover different time steps: "
58
+ f"cm-only={sorted(cm_times - pg_times)}, "
59
+ f"pgm-only={sorted(pg_times - cm_times)}"
60
+ )
61
+
62
+ # Valid countries: every (time, country) the grid maps to must have a forecast.
63
+ cm_units = pgm_frame.index.cross_level_align_arrays(
64
+ np.asarray(map_keys), np.asarray(map_vals), SpatialLevel.CM
65
+ ).unit
66
+ needed = {
67
+ (int(t), int(c))
68
+ for t, c in zip(pgm_frame.index.time, cm_units, strict=True)
69
+ }
70
+ have = {
71
+ (int(t), int(c))
72
+ for t, c in zip(cm_frame.index.time, cm_frame.index.unit, strict=True)
73
+ }
74
+ missing = needed - have
75
+ if missing:
76
+ raise ValueError(
77
+ f"{len(missing)} grid group(s) have no country forecast in cm_frame, "
78
+ f"e.g. {sorted(missing)[:5]}"
79
+ )
@@ -17,6 +17,8 @@ from views_frames_summarize.aggregate import (
17
17
  )
18
18
  from views_frames_summarize.bimodality import bimodality
19
19
  from views_frames_summarize.collapse import collapse
20
+ from views_frames_summarize.exceedance import exceedance, exceedance_reducer
21
+ from views_frames_summarize.expected_shortfall import expected_shortfall
20
22
  from views_frames_summarize.interval import hdi, quantiles
21
23
  from views_frames_summarize.point import map_estimate
22
24
  from views_frames_summarize.summarize_tower import TowerSummary, summarize_tower
@@ -29,6 +31,9 @@ __all__ = [
29
31
  "aggregate_distributions_arrays",
30
32
  "bimodality",
31
33
  "collapse",
34
+ "exceedance",
35
+ "exceedance_reducer",
36
+ "expected_shortfall",
32
37
  "hdi",
33
38
  "hdi_tower",
34
39
  "map_estimate",
@@ -13,6 +13,8 @@ from views_frames_summarize import config
13
13
  from views_frames_summarize._common import AnyFrame
14
14
  from views_frames_summarize.bimodality import bimodality
15
15
  from views_frames_summarize.collapse import collapse
16
+ from views_frames_summarize.exceedance import exceedance
17
+ from views_frames_summarize.expected_shortfall import expected_shortfall
16
18
  from views_frames_summarize.interval import hdi, quantiles
17
19
  from views_frames_summarize.point import map_estimate
18
20
  from views_frames_summarize.summarize_tower import summarize_tower
@@ -44,6 +46,36 @@ def assert_summarizer_contract(frame: AnyFrame) -> None:
44
46
  assert qs.shape[0] == n, "quantiles must be aligned to the frame's rows"
45
47
  assert qs.shape[-1] == 3, "quantiles must produce one column per quantile"
46
48
 
49
+ # exceedance laws (ADR-021): a survival function — in [0, 1], non-increasing in the
50
+ # threshold, with P(> -inf) = 1 and P(> +inf) = 0.
51
+ thresholds = [-np.inf, 0.5, 1.5, np.inf]
52
+ exc = exceedance(frame, thresholds)
53
+ assert exc.shape[0] == n, "exceedance must be aligned to the frame's rows"
54
+ assert exc.shape[-1] == len(thresholds), "exceedance → one column per threshold"
55
+ assert ((exc >= 0.0) & (exc <= 1.0)).all(), (
56
+ "exceedance must be a probability in [0, 1]"
57
+ )
58
+ assert (np.diff(exc, axis=-1) <= 1e-6).all(), (
59
+ "exceedance must be non-increasing in the threshold"
60
+ )
61
+ assert (exc[..., 0] == 1.0).all(), "P(> -inf) must be 1"
62
+ assert (exc[..., -1] == 0.0).all(), "P(> +inf) must be 0"
63
+
64
+ # expected-shortfall laws (ADR-022): a tail mean — in [min, max], non-decreasing as
65
+ # the tail deepens (t → 0), and dominating its VaR (ES(t) ≥ the (1 − t) quantile).
66
+ tails = [1.0, 0.5, 0.1] # widening tails (deepening worst-case)
67
+ es = expected_shortfall(frame, tails)
68
+ lo = frame.values.min(axis=-1)[..., np.newaxis]
69
+ hi = frame.values.max(axis=-1)[..., np.newaxis]
70
+ assert es.shape[0] == n, "expected_shortfall must be aligned to the frame's rows"
71
+ assert es.shape[-1] == len(tails), "expected_shortfall → one column per tail"
72
+ assert ((es >= lo - 1e-6) & (es <= hi + 1e-6)).all(), "ES must lie in [min, max]"
73
+ assert (np.diff(es, axis=-1) >= -1e-6).all(), (
74
+ "ES must be non-decreasing as the tail deepens"
75
+ )
76
+ var = quantiles(frame, [0.0, 0.5, 0.9]) # the (1 − t) quantiles for t = 1, 0.5, 0.1
77
+ assert (es >= var - 1e-6).all(), "ES(t) must dominate the (1 − t) quantile"
78
+
47
79
  _assert_tower_contract(frame, n)
48
80
 
49
81
 
@@ -0,0 +1,96 @@
1
+ """Threshold exceedance probabilities over the sample axis (ADR-021).
2
+
3
+ `exceedance(frame, thresholds)` returns the per-row empirical **survival fraction**
4
+ `P(Y > c)` for each threshold — an index-aligned array `(N, …, K)`, the same shape/role
5
+ family as `quantiles`. `exceedance_reducer(c)` is a `collapse`-compatible factory for
6
+ the single-threshold `(N, …, 1)` **frame** path.
7
+
8
+ Distribution-agnostic (a counting reducer — no histogram, no config, no unimodality
9
+ assumption), so it is robust where the tower is weakest (register C-34); the flagship
10
+ is `P(Y > 0)` = probability of any activity (onset). Thresholds are **required**
11
+ per-call, in the frame's own units (an *input*, like `quantiles`' ``qs`` — never a
12
+ tunable or a default). **Strict `>`** (register D-08). Fails loud on any non-finite
13
+ draw — NaN or ±inf (register C-50) — and on empty thresholds; the joint-sample
14
+ requirement for aggregate exceedance is the consumer's (C-49).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Sequence
20
+
21
+ import numpy as np
22
+ from numpy.typing import NDArray
23
+
24
+ from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
25
+ from views_frames_summarize.collapse import Reducer
26
+
27
+
28
+ def _exceed(
29
+ values: NDArray[np.float32], thresholds: NDArray[np.float64]
30
+ ) -> NDArray[np.float32]:
31
+ """Fraction of the trailing (sample) axis **strictly** greater than each threshold.
32
+
33
+ ``values`` is ``(…, S)`` and ``thresholds`` is ``(K,)``; returns ``(…, K)``.
34
+
35
+ Fails loud on any **non-finite** draw (NaN or ±inf): ``NaN > c`` is silently
36
+ ``False``, so a naive count would deflate ``P(Y > c)`` — worst on the ``P(Y > 0)``
37
+ onset flagship; and an ±inf draw (always an upstream bug) silently blesses ``P``
38
+ as a valid probability, masking the bug. A draw must be finite (C-50).
39
+ """
40
+ if not bool(np.isfinite(values).all()):
41
+ raise ValueError(
42
+ "exceedance is undefined on non-finite draws (NaN or ±inf); strip or "
43
+ "impute upstream (NaN > c is silently False and would deflate P(Y > c); "
44
+ "an inf draw masks an upstream bug behind a valid-looking P; register C-50)"
45
+ )
46
+ # (…, S) vs (K,) → (…, K, S) by broadcasting; strict ">"; mean over the sample axis.
47
+ above = values[..., np.newaxis, :] > thresholds[:, np.newaxis]
48
+ return np.asarray(above.mean(axis=-1), dtype=np.float32)
49
+
50
+
51
+ def exceedance(
52
+ frame: AnyFrame,
53
+ thresholds: Sequence[float],
54
+ *,
55
+ block_rows: int = ROW_BLOCK,
56
+ ) -> NDArray[np.float32]:
57
+ """Per-row exceedance probability `P(Y > c)` over the sample axis → `(N, …, K)`.
58
+
59
+ For each of the ``K`` caller-supplied ``thresholds`` (required, in the frame's own
60
+ units), the empirical fraction of draws **strictly** greater than it. An
61
+ index-aligned numpy array (the caller holds the index), vectorized in row-blocks of
62
+ ``block_rows`` so peak memory is bounded by one block's working set (register C-25).
63
+
64
+ Raises:
65
+ ValueError: ``thresholds`` is empty (no default), or any draw is non-finite —
66
+ NaN or ±inf (C-50).
67
+ """
68
+ thr = np.asarray(thresholds, dtype=np.float64)
69
+ if thr.size == 0:
70
+ raise ValueError(
71
+ "exceedance requires at least one threshold; there is no default "
72
+ "(thresholds are an input in the frame's units, not a tunable — ADR-021)."
73
+ )
74
+
75
+ def _block(vals: NDArray[np.float32]) -> NDArray[np.float32]:
76
+ return _exceed(vals, thr)
77
+
78
+ out = block_apply(frame.values, block_rows, _block)
79
+ return np.asarray(out, dtype=np.float32)
80
+
81
+
82
+ def exceedance_reducer(threshold: float) -> Reducer:
83
+ """A `collapse`-compatible reducer for single-threshold exceedance.
84
+
85
+ ``collapse(frame, exceedance_reducer(c))`` returns `P(Y > c)` as a `(N, …, 1)`
86
+ frame, sharing `exceedance`'s strict-`>` and fail-loud-non-finite policy (so a
87
+ consumer never re-implements ``np.mean(values > c)`` and drifts on the convention).
88
+ """
89
+ thr = np.asarray([threshold], dtype=np.float64)
90
+
91
+ def _reducer(values: NDArray[np.float32], axis: int = -1) -> NDArray[np.float32]:
92
+ # `collapse` calls reducer(values, axis=-1) and appends the trailing axis
93
+ # itself, so drop the singleton K axis (collapse re-adds the sample-axis slot).
94
+ return _exceed(values, thr)[..., 0]
95
+
96
+ return _reducer
@@ -0,0 +1,88 @@
1
+ """Worst-case scenario summary over the sample axis (ADR-022).
2
+
3
+ `expected_shortfall(frame, tails)` returns the per-row **mean of the worst `⌈t·S⌉`
4
+ draws** for each upper-tail fraction `t` — the tail mean / CVaR, a coherent worst-case
5
+ risk measure and the companion to `exceedance`. An index-aligned array `(N, …, K)`, the
6
+ same shape/role family as `quantiles`.
7
+
8
+ `max` is deliberately **not** offered — it is the highest-variance, non-reproducible
9
+ summary this replaces. Tails are **required** per-call, in `(0, 1]` (e.g. `0.01` = "the
10
+ worst 1%") — the consumer's policy, never a default or config. Fails loud on any
11
+ non-finite draw — NaN or ±inf (numpy sorts them *last*, so a naive top-`k` mean silently
12
+ selects them — register C-56), on empty `tails`, and on any `t` outside `(0, 1]`.
13
+
14
+ Written explicitly in its own module — it does **not** share a "tail reducer"
15
+ abstraction with `quantiles`/`exceedance` (the duplication is shallow and the concerns
16
+ change independently — WET before DRY). It reuses only the stable `_common` primitives.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Sequence
22
+
23
+ import numpy as np
24
+ from numpy.typing import NDArray
25
+
26
+ from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
27
+
28
+
29
+ def _expected_shortfall(
30
+ values: NDArray[np.float32], tails: NDArray[np.float64]
31
+ ) -> NDArray[np.float32]:
32
+ """Mean of the worst ``⌈t·S⌉`` draws of the trailing (sample) axis, per tail.
33
+
34
+ ``values`` is ``(…, S)``, ``tails`` is ``(K,)`` in ``(0, 1]``; returns ``(…, K)``.
35
+
36
+ Fails loud on any **non-finite** draw (NaN or ±inf): numpy sorts NaN *last*, so a
37
+ top-`k` mean would silently select the NaNs — a NaN/garbage worst-case; and an ±inf
38
+ draw (always an upstream bug) sorts last and contaminates the tail mean to ±inf — a
39
+ degenerate "worst case". A draw must be a usable finite number (register C-56).
40
+ """
41
+ if not bool(np.isfinite(values).all()):
42
+ raise ValueError(
43
+ "expected_shortfall is undefined on non-finite draws (NaN or ±inf); strip "
44
+ "or impute upstream (numpy sorts NaN/+inf last, so the worst-k mean would "
45
+ "silently select them — a NaN/inf worst-case; register C-56)"
46
+ )
47
+ s = values.shape[-1]
48
+ srt = np.sort(values, axis=-1) # ascending: the worst (largest) draws are last
49
+ out = np.empty((*values.shape[:-1], tails.shape[0]), dtype=np.float32)
50
+ for i in range(tails.shape[0]):
51
+ k = int(np.ceil(float(tails[i]) * s)) # number of worst draws to average (≥ 1)
52
+ out[..., i] = srt[..., s - k :].mean(axis=-1)
53
+ return out
54
+
55
+
56
+ def expected_shortfall(
57
+ frame: AnyFrame,
58
+ tails: Sequence[float],
59
+ *,
60
+ block_rows: int = ROW_BLOCK,
61
+ ) -> NDArray[np.float32]:
62
+ """Per-row worst-case **expected shortfall** over the sample axis → `(N, …, K)`.
63
+
64
+ For each upper-tail fraction ``t`` in ``tails`` (required, in ``(0, 1]``), the mean
65
+ of the worst ``⌈t·S⌉`` draws — the average of the worst-case scenarios. An
66
+ index-aligned numpy array (the caller holds the index), block-applied in row-blocks.
67
+
68
+ Raises:
69
+ ValueError: ``tails`` is empty (no default), any ``t ∉ (0, 1]`` (a tail with no
70
+ samples), or any draw is non-finite — NaN or ±inf (register C-56).
71
+ """
72
+ thr = np.asarray(tails, dtype=np.float64)
73
+ if thr.size == 0:
74
+ raise ValueError(
75
+ "expected_shortfall requires at least one tail level; there is no default "
76
+ "(the tail fraction is an input in (0, 1], not a tunable — ADR-022)."
77
+ )
78
+ if bool(((thr <= 0.0) | (thr > 1.0)).any()):
79
+ raise ValueError(
80
+ f"expected_shortfall tails must be in (0, 1]; got {thr.tolist()} "
81
+ "(a tail <= 0 contains no samples; > 1 is not a fraction)."
82
+ )
83
+
84
+ def _block(vals: NDArray[np.float32]) -> NDArray[np.float32]:
85
+ return _expected_shortfall(vals, thr)
86
+
87
+ out = block_apply(frame.values, block_rows, _block)
88
+ return np.asarray(out, dtype=np.float32)
File without changes
File without changes
File without changes