views-frames 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- views_frames-1.1.0/.gitignore +46 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/PKG-INFO +28 -19
- {views_frames-1.0.0 → views_frames-1.1.0}/README.md +17 -18
- {views_frames-1.0.0 → views_frames-1.1.0}/pyproject.toml +31 -1
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/__init__.py +9 -0
- views_frames-1.1.0/src/views_frames_summarize/bimodality.py +113 -0
- views_frames-1.1.0/src/views_frames_summarize/conformance.py +87 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/point.py +10 -0
- views_frames-1.1.0/src/views_frames_summarize/summarize_tower.py +89 -0
- views_frames-1.1.0/src/views_frames_summarize/tower.py +191 -0
- views_frames-1.1.0/src/views_frames_summarize/tower_point.py +46 -0
- views_frames-1.0.0/.gitignore +0 -30
- views_frames-1.0.0/src/views_frames_summarize/conformance.py +0 -40
- {views_frames-1.0.0 → views_frames-1.1.0}/LICENSE +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/__init__.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/_typing.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/_validation.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/conformance/__init__.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/feature_frame.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/index.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/io/__init__.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/io/arrow.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/io/npz.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/metadata.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/prediction_frame.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/protocols.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/py.typed +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/spatial_level.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames/target_frame.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/_common.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/aggregate.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/collapse.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/interval.py +0 -0
- {views_frames-1.0.0 → views_frames-1.1.0}/src/views_frames_summarize/py.typed +0 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
.eggs/
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
|
|
13
|
+
# Tool caches
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.ruff_cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
|
|
18
|
+
# Coverage artifacts
|
|
19
|
+
.coverage
|
|
20
|
+
.coverage.*
|
|
21
|
+
htmlcov/
|
|
22
|
+
coverage.xml
|
|
23
|
+
|
|
24
|
+
# Environment and secrets
|
|
25
|
+
.env
|
|
26
|
+
.env.*
|
|
27
|
+
|
|
28
|
+
# OS
|
|
29
|
+
.DS_Store
|
|
30
|
+
Thumbs.db
|
|
31
|
+
|
|
32
|
+
# IDE
|
|
33
|
+
.idea/
|
|
34
|
+
.vscode/
|
|
35
|
+
*.swp
|
|
36
|
+
*.swo
|
|
37
|
+
|
|
38
|
+
# Internal-only working docs — kept on disk, never published.
|
|
39
|
+
critiqus/
|
|
40
|
+
perspectives/
|
|
41
|
+
|
|
42
|
+
# Research lab — code is tracked; generated artifacts are not.
|
|
43
|
+
research/**/.cache/
|
|
44
|
+
research/**/results.tsv
|
|
45
|
+
research/**/run.log
|
|
46
|
+
research/**/audit_plots/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: views-frames
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG).
|
|
5
5
|
Project-URL: Homepage, https://github.com/views-platform/views-frames
|
|
6
6
|
Project-URL: Repository, https://github.com/views-platform/views-frames
|
|
@@ -8,6 +8,16 @@ Project-URL: Changelog, https://github.com/views-platform/views-frames/blob/main
|
|
|
8
8
|
Author-email: Simon Polichinel von der Maase <simmaa@prio.org>
|
|
9
9
|
License-Expression: MIT
|
|
10
10
|
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering
|
|
20
|
+
Classifier: Typing :: Typed
|
|
11
21
|
Requires-Python: >=3.10
|
|
12
22
|
Requires-Dist: numpy<3,>=1.26
|
|
13
23
|
Provides-Extra: arrow
|
|
@@ -20,12 +30,14 @@ Description-Content-Type: text/markdown
|
|
|
20
30
|
> containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
|
|
21
31
|
> that every other repo depends on and that depends on nothing internal.
|
|
22
32
|
>
|
|
23
|
-
> **Status:** **
|
|
33
|
+
> **Status:** **v1.1.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1.0 surface
|
|
34
|
+
> is purely additive — the coherent posterior summary, ADR-019). This README is the design
|
|
24
35
|
> bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
|
|
25
36
|
> io, conformance suite) plus the `src/views_frames_summarize/` sibling package
|
|
26
|
-
> (sample-axis summarization — `collapse`/MAP/HDI/quantiles
|
|
27
|
-
>
|
|
28
|
-
>
|
|
37
|
+
> (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators
|
|
38
|
+
> `hdi_tower`/`tower_point`/`bimodality`/`summarize_tower`, + cross-level
|
|
39
|
+
> aggregation; ADR-017, ADR-019). The blocking design decisions are resolved (§13a) and
|
|
40
|
+
> ratified as ADRs 011–018; two rounds of consumer review validated
|
|
29
41
|
> the design. Consumer adoption (re-export shims, pandas migration) is the owner's
|
|
30
42
|
> migration, not this repo's.
|
|
31
43
|
> If the code and this README disagree, that is a bug — reconcile before merging.
|
|
@@ -123,8 +135,7 @@ register):
|
|
|
123
135
|
first materializes the full-sample tensor). This is C-40/C-66 firing for real —
|
|
124
136
|
pipeline-core **C-186**, the **first observed-in-production member** of the
|
|
125
137
|
Data-Contract Gap cluster, and the live use-case that motivates this package.
|
|
126
|
-
A dense, collapsed array frame is the fix.
|
|
127
|
-
`perspectives/from_views-pipeline-core_perspective.md`.
|
|
138
|
+
A dense, collapsed array frame is the fix.
|
|
128
139
|
- **God-class data handler with leaked internals.** `_ViewsDataset`
|
|
129
140
|
(`data/handlers.py`, ~950 LOC, C-36) is consumed across three repos by reaching
|
|
130
141
|
into its **private** members (`_time_id`, `_entity_id`, `_get_entity_index`,
|
|
@@ -180,14 +191,13 @@ imports **no** `views_*` package, ever. If it ever needs to, the boundary is
|
|
|
180
191
|
wrong. This is what makes it impossible to participate in a cycle (ADP) and what
|
|
181
192
|
makes it safe to depend on from everywhere (SDP).
|
|
182
193
|
|
|
183
|
-
> **Consumer
|
|
184
|
-
> these frames
|
|
185
|
-
> `perspectives/from_views-reporting_perspective.md` — the presentation layer that
|
|
194
|
+
> **Consumer views.** Each downstream repo has a detailed view of how it uses
|
|
195
|
+
> these frames. The first is **views-reporting** — the presentation layer that
|
|
186
196
|
> *consumes* `PredictionFrame`, `TargetFrame`, and `MetricFrame` and routes its
|
|
187
197
|
> data contract through this leaf (which is what breaks the
|
|
188
198
|
> views-pipeline-core ↔ views-reporting cycle, reporting issue **#113**).
|
|
189
199
|
>
|
|
190
|
-
>
|
|
200
|
+
> **views-pipeline-core** is the **origin/orchestration**
|
|
191
201
|
> repo's view — not a pure downstream consumer but the repo that *owns these types
|
|
192
202
|
> today* (`PredictionFrame`, `_ViewsDataset`, the converter) and hands the contract
|
|
193
203
|
> off to this leaf. It carries the worked failure mode (#181 report-stage OOM,
|
|
@@ -267,7 +277,7 @@ uses `pd.isna` for the NaN-check (§10.2).
|
|
|
267
277
|
| **`TargetFrame`** (a.k.a. `ActualsFrame`) | `y_true: (N, 1)` | The **evaluation boundary** still takes pandas actuals (`adapter.py`). A target frame makes eval array-native and kills that pandas dependency. Structurally `PredictionFrame` with `S=1`. | **next** |
|
|
268
278
|
| **`WeightFrame`** | `w: (N,)` or `(N, S)` | Weighted losses / weighted metrics. Same identifiers, different `values` meaning. | when weighting lands |
|
|
269
279
|
| **`MaskFrame`** | `mask: (N,)` bool | Partial-data / sparse-actuals evaluation (C-26 silent truncation). Marks which (time, unit) cells are present. | when partial eval lands |
|
|
270
|
-
| **`MetricFrame`** (a.k.a. `ScoreFrame`) | `(K, …)` keyed by `(target, step, unit)` | Evaluation **outputs** are currently scattered into wandb summaries + parquet. First-class array form. **views-reporting's eval report is the consumer of record** — today it scrapes wandb and renders the wrong run (its
|
|
280
|
+
| **`MetricFrame`** (a.k.a. `ScoreFrame`) | `(K, …)` keyed by `(target, step, unit)` | Evaluation **outputs** are currently scattered into wandb summaries + parquet. First-class array form. **views-reporting's eval report is the consumer of record** — today it scrapes wandb and renders the wrong run (its run-selection bug, C-48). | exploratory |
|
|
271
281
|
|
|
272
282
|
**Already exists externally — do NOT rebuild:** `EvaluationFrame` lives in
|
|
273
283
|
`views-evaluation` (aligned pred×actual×(origin, step)). `views-frames` should
|
|
@@ -299,8 +309,8 @@ genuinely reused core. Build this once:
|
|
|
299
309
|
does not depend on), never fetched or versioned here — embedding versioned domain
|
|
300
310
|
data would make the leaf change for data reasons and break §8 maximal stability.
|
|
301
311
|
This resolves the falsified "domain-free cross-level" claim
|
|
302
|
-
(
|
|
303
|
-
existence proof
|
|
312
|
+
(a falsification audit, 5 hard falsifications); faoapi's producer-materialised
|
|
313
|
+
metadata is the existence proof.
|
|
304
314
|
- `SpatialLevel` (currently `views-pipeline-core/domain/spatial.py`) should move
|
|
305
315
|
here — it is a tiny, stable value object that *is* part of the identifier
|
|
306
316
|
vocabulary (it defines `index_names` and `entity_column`: cm→`country_id`,
|
|
@@ -413,7 +423,7 @@ Layout rules (these *are* the screaming-architecture requirements):
|
|
|
413
423
|
The scaling failure in the platform today is the **list-in-cell `object`-dtype
|
|
414
424
|
DataFrame** (a cell holds a Python list of S samples) — measured ~33× blow-up
|
|
415
425
|
(C-40/C-66), and ~50–160× per-row over dense float32 in the #181 report-stage
|
|
416
|
-
investigation (C-186
|
|
426
|
+
investigation (C-186).
|
|
417
427
|
`views-frames` standardizes two scalable formats and **bans list-in-cell**:
|
|
418
428
|
|
|
419
429
|
- **Native (`io/npz.py`):** `values.npy` (contiguous float32) + `identifiers.npz`.
|
|
@@ -527,8 +537,7 @@ typed contract → frame I/O contract), **C-184** (cross-repo mutation of
|
|
|
527
537
|
(circular dependency) and informs **D-28** (relocate reconciliation) and **D-33**
|
|
528
538
|
(collapse the `CMDataset/PGMDataset` hierarchy into a `SpatialLevel` value).
|
|
529
539
|
|
|
530
|
-
From the **views-reporting** consumer (its own register
|
|
531
|
-
`perspectives/from_views-reporting_perspective.md`) this package *forbids* its
|
|
540
|
+
From the **views-reporting** consumer (its own register) this package *forbids* its
|
|
532
541
|
**C-184** (the `reconciled_dataframe` mutation) and the reporting side of
|
|
533
542
|
**C-135** (private `_entity_id`/`_time_id` reads → published index protocol), and
|
|
534
543
|
*enables* fixing **C-48** (wandb eval scrape → a typed `MetricFrame`) and **C-44**
|
|
@@ -565,8 +574,8 @@ pipeline-core **C-48** listed above — two registers, same number.)
|
|
|
565
574
|
mapping.** Same-level alignment lives in the leaf; the cross-level country↔grid
|
|
566
575
|
join needs a viewser-sourced, time-varying `priogrid→country` **mapping** that
|
|
567
576
|
is **injected by the consumer and never embedded in the leaf**. The leaf owns
|
|
568
|
-
only `cross_level_align(index, mapping)`. See §4.3; resolves
|
|
569
|
-
|
|
577
|
+
only `cross_level_align(index, mapping)`. See §4.3; resolves the
|
|
578
|
+
falsified "domain-free cross-level" claim (a falsification audit, 5 hard falsifications).
|
|
570
579
|
5. **`SpatialLevel` lives here, as identifier vocabulary only** — relocated with
|
|
571
580
|
the C-65 reversed index-tuple and the `priogrid_gid`/`priogrid_id`
|
|
572
581
|
inconsistency **fixed, not ported** (§4.3). It carries the level labels, never
|
|
@@ -4,12 +4,14 @@
|
|
|
4
4
|
> containers (`FeatureFrame`, `PredictionFrame`, and their anticipated siblings)
|
|
5
5
|
> that every other repo depends on and that depends on nothing internal.
|
|
6
6
|
>
|
|
7
|
-
> **Status:** **
|
|
7
|
+
> **Status:** **v1.1.0 — frozen API** (frozen since v1.0.0, ADR-018; the v1.1.0 surface
|
|
8
|
+
> is purely additive — the coherent posterior summary, ADR-019). This README is the design
|
|
8
9
|
> bible; the contract it specifies is realised in `src/views_frames/` (index, frames,
|
|
9
10
|
> io, conformance suite) plus the `src/views_frames_summarize/` sibling package
|
|
10
|
-
> (sample-axis summarization — `collapse`/MAP/HDI/quantiles
|
|
11
|
-
>
|
|
12
|
-
>
|
|
11
|
+
> (sample-axis summarization — `collapse`/MAP/HDI/quantiles, the coherent-tower estimators
|
|
12
|
+
> `hdi_tower`/`tower_point`/`bimodality`/`summarize_tower`, + cross-level
|
|
13
|
+
> aggregation; ADR-017, ADR-019). The blocking design decisions are resolved (§13a) and
|
|
14
|
+
> ratified as ADRs 011–018; two rounds of consumer review validated
|
|
13
15
|
> the design. Consumer adoption (re-export shims, pandas migration) is the owner's
|
|
14
16
|
> migration, not this repo's.
|
|
15
17
|
> If the code and this README disagree, that is a bug — reconcile before merging.
|
|
@@ -107,8 +109,7 @@ register):
|
|
|
107
109
|
first materializes the full-sample tensor). This is C-40/C-66 firing for real —
|
|
108
110
|
pipeline-core **C-186**, the **first observed-in-production member** of the
|
|
109
111
|
Data-Contract Gap cluster, and the live use-case that motivates this package.
|
|
110
|
-
A dense, collapsed array frame is the fix.
|
|
111
|
-
`perspectives/from_views-pipeline-core_perspective.md`.
|
|
112
|
+
A dense, collapsed array frame is the fix.
|
|
112
113
|
- **God-class data handler with leaked internals.** `_ViewsDataset`
|
|
113
114
|
(`data/handlers.py`, ~950 LOC, C-36) is consumed across three repos by reaching
|
|
114
115
|
into its **private** members (`_time_id`, `_entity_id`, `_get_entity_index`,
|
|
@@ -164,14 +165,13 @@ imports **no** `views_*` package, ever. If it ever needs to, the boundary is
|
|
|
164
165
|
wrong. This is what makes it impossible to participate in a cycle (ADP) and what
|
|
165
166
|
makes it safe to depend on from everywhere (SDP).
|
|
166
167
|
|
|
167
|
-
> **Consumer
|
|
168
|
-
> these frames
|
|
169
|
-
> `perspectives/from_views-reporting_perspective.md` — the presentation layer that
|
|
168
|
+
> **Consumer views.** Each downstream repo has a detailed view of how it uses
|
|
169
|
+
> these frames. The first is **views-reporting** — the presentation layer that
|
|
170
170
|
> *consumes* `PredictionFrame`, `TargetFrame`, and `MetricFrame` and routes its
|
|
171
171
|
> data contract through this leaf (which is what breaks the
|
|
172
172
|
> views-pipeline-core ↔ views-reporting cycle, reporting issue **#113**).
|
|
173
173
|
>
|
|
174
|
-
>
|
|
174
|
+
> **views-pipeline-core** is the **origin/orchestration**
|
|
175
175
|
> repo's view — not a pure downstream consumer but the repo that *owns these types
|
|
176
176
|
> today* (`PredictionFrame`, `_ViewsDataset`, the converter) and hands the contract
|
|
177
177
|
> off to this leaf. It carries the worked failure mode (#181 report-stage OOM,
|
|
@@ -251,7 +251,7 @@ uses `pd.isna` for the NaN-check (§10.2).
|
|
|
251
251
|
| **`TargetFrame`** (a.k.a. `ActualsFrame`) | `y_true: (N, 1)` | The **evaluation boundary** still takes pandas actuals (`adapter.py`). A target frame makes eval array-native and kills that pandas dependency. Structurally `PredictionFrame` with `S=1`. | **next** |
|
|
252
252
|
| **`WeightFrame`** | `w: (N,)` or `(N, S)` | Weighted losses / weighted metrics. Same identifiers, different `values` meaning. | when weighting lands |
|
|
253
253
|
| **`MaskFrame`** | `mask: (N,)` bool | Partial-data / sparse-actuals evaluation (C-26 silent truncation). Marks which (time, unit) cells are present. | when partial eval lands |
|
|
254
|
-
| **`MetricFrame`** (a.k.a. `ScoreFrame`) | `(K, …)` keyed by `(target, step, unit)` | Evaluation **outputs** are currently scattered into wandb summaries + parquet. First-class array form. **views-reporting's eval report is the consumer of record** — today it scrapes wandb and renders the wrong run (its
|
|
254
|
+
| **`MetricFrame`** (a.k.a. `ScoreFrame`) | `(K, …)` keyed by `(target, step, unit)` | Evaluation **outputs** are currently scattered into wandb summaries + parquet. First-class array form. **views-reporting's eval report is the consumer of record** — today it scrapes wandb and renders the wrong run (its run-selection bug, C-48). | exploratory |
|
|
255
255
|
|
|
256
256
|
**Already exists externally — do NOT rebuild:** `EvaluationFrame` lives in
|
|
257
257
|
`views-evaluation` (aligned pred×actual×(origin, step)). `views-frames` should
|
|
@@ -283,8 +283,8 @@ genuinely reused core. Build this once:
|
|
|
283
283
|
does not depend on), never fetched or versioned here — embedding versioned domain
|
|
284
284
|
data would make the leaf change for data reasons and break §8 maximal stability.
|
|
285
285
|
This resolves the falsified "domain-free cross-level" claim
|
|
286
|
-
(
|
|
287
|
-
existence proof
|
|
286
|
+
(a falsification audit, 5 hard falsifications); faoapi's producer-materialised
|
|
287
|
+
metadata is the existence proof.
|
|
288
288
|
- `SpatialLevel` (currently `views-pipeline-core/domain/spatial.py`) should move
|
|
289
289
|
here — it is a tiny, stable value object that *is* part of the identifier
|
|
290
290
|
vocabulary (it defines `index_names` and `entity_column`: cm→`country_id`,
|
|
@@ -397,7 +397,7 @@ Layout rules (these *are* the screaming-architecture requirements):
|
|
|
397
397
|
The scaling failure in the platform today is the **list-in-cell `object`-dtype
|
|
398
398
|
DataFrame** (a cell holds a Python list of S samples) — measured ~33× blow-up
|
|
399
399
|
(C-40/C-66), and ~50–160× per-row over dense float32 in the #181 report-stage
|
|
400
|
-
investigation (C-186
|
|
400
|
+
investigation (C-186).
|
|
401
401
|
`views-frames` standardizes two scalable formats and **bans list-in-cell**:
|
|
402
402
|
|
|
403
403
|
- **Native (`io/npz.py`):** `values.npy` (contiguous float32) + `identifiers.npz`.
|
|
@@ -511,8 +511,7 @@ typed contract → frame I/O contract), **C-184** (cross-repo mutation of
|
|
|
511
511
|
(circular dependency) and informs **D-28** (relocate reconciliation) and **D-33**
|
|
512
512
|
(collapse the `CMDataset/PGMDataset` hierarchy into a `SpatialLevel` value).
|
|
513
513
|
|
|
514
|
-
From the **views-reporting** consumer (its own register
|
|
515
|
-
`perspectives/from_views-reporting_perspective.md`) this package *forbids* its
|
|
514
|
+
From the **views-reporting** consumer (its own register) this package *forbids* its
|
|
516
515
|
**C-184** (the `reconciled_dataframe` mutation) and the reporting side of
|
|
517
516
|
**C-135** (private `_entity_id`/`_time_id` reads → published index protocol), and
|
|
518
517
|
*enables* fixing **C-48** (wandb eval scrape → a typed `MetricFrame`) and **C-44**
|
|
@@ -549,8 +548,8 @@ pipeline-core **C-48** listed above — two registers, same number.)
|
|
|
549
548
|
mapping.** Same-level alignment lives in the leaf; the cross-level country↔grid
|
|
550
549
|
join needs a viewser-sourced, time-varying `priogrid→country` **mapping** that
|
|
551
550
|
is **injected by the consumer and never embedded in the leaf**. The leaf owns
|
|
552
|
-
only `cross_level_align(index, mapping)`. See §4.3; resolves
|
|
553
|
-
|
|
551
|
+
only `cross_level_align(index, mapping)`. See §4.3; resolves the
|
|
552
|
+
falsified "domain-free cross-level" claim (a falsification audit, 5 hard falsifications).
|
|
554
553
|
5. **`SpatialLevel` lives here, as identifier vocabulary only** — relocated with
|
|
555
554
|
the C-65 reversed index-tuple and the `priogrid_gid`/`priogrid_id`
|
|
556
555
|
inconsistency **fixed, not ported** (§4.3). It carries the level labels, never
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "views-frames"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "The VIEWS platform data-contract layer: immutable array+identifier frames (numpy only, root of the dependency DAG)."
|
|
9
9
|
authors = [
|
|
10
10
|
{ name = "Simon Polichinel von der Maase", email = "simmaa@prio.org" },
|
|
@@ -13,6 +13,20 @@ readme = "README.md"
|
|
|
13
13
|
requires-python = ">=3.10"
|
|
14
14
|
license = "MIT"
|
|
15
15
|
license-files = ["LICENSE"]
|
|
16
|
+
# License is declared via the SPDX `license` expression above (PEP 639); the deprecated
|
|
17
|
+
# `License ::` classifier is intentionally omitted to avoid the expression/classifier clash.
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 5 - Production/Stable",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: Scientific/Engineering",
|
|
28
|
+
"Typing :: Typed",
|
|
29
|
+
]
|
|
16
30
|
dependencies = [
|
|
17
31
|
"numpy>=1.26,<3",
|
|
18
32
|
]
|
|
@@ -29,6 +43,7 @@ arrow = ["pyarrow>=14,<20"]
|
|
|
29
43
|
[dependency-groups]
|
|
30
44
|
dev = [
|
|
31
45
|
"pytest>=8.0",
|
|
46
|
+
"pytest-cov>=5",
|
|
32
47
|
"ruff>=0.8",
|
|
33
48
|
"mypy>=1.13",
|
|
34
49
|
# pyarrow is the consumer-facing [arrow] extra; installed in dev so CI
|
|
@@ -46,9 +61,24 @@ include = ["/src"]
|
|
|
46
61
|
testpaths = ["tests"]
|
|
47
62
|
addopts = "-ra -q"
|
|
48
63
|
|
|
64
|
+
# Coverage: the leaf is critical infrastructure (ADR-005) — CI enforces 100%
|
|
65
|
+
# line coverage (`uv run pytest --cov --cov-fail-under=100`). Genuinely
|
|
66
|
+
# unreachable defensive lines must carry an explicit `# pragma: no cover`.
|
|
67
|
+
[tool.coverage.run]
|
|
68
|
+
source = ["src"]
|
|
69
|
+
# Branch coverage on: the `--cov-fail-under=100` gate enforces line AND branch
|
|
70
|
+
# coverage, so a conditional with an untested outcome fails CI (register C-36).
|
|
71
|
+
branch = true
|
|
72
|
+
|
|
73
|
+
[tool.coverage.report]
|
|
74
|
+
show_missing = true
|
|
75
|
+
|
|
49
76
|
[tool.ruff]
|
|
50
77
|
line-length = 88
|
|
51
78
|
target-version = "py310"
|
|
79
|
+
# research/ is the volatile estimator lab (outside the frozen, gated src/); it is
|
|
80
|
+
# not held to the leaf's lint/format bar.
|
|
81
|
+
extend-exclude = ["research"]
|
|
52
82
|
|
|
53
83
|
[tool.ruff.lint]
|
|
54
84
|
select = ["E", "F", "I", "N", "UP", "B", "A", "C4", "SIM"]
|
|
@@ -15,15 +15,24 @@ from views_frames_summarize.aggregate import (
|
|
|
15
15
|
aggregate_distributions,
|
|
16
16
|
aggregate_distributions_arrays,
|
|
17
17
|
)
|
|
18
|
+
from views_frames_summarize.bimodality import bimodality
|
|
18
19
|
from views_frames_summarize.collapse import collapse
|
|
19
20
|
from views_frames_summarize.interval import hdi, quantiles
|
|
20
21
|
from views_frames_summarize.point import map_estimate
|
|
22
|
+
from views_frames_summarize.summarize_tower import TowerSummary, summarize_tower
|
|
23
|
+
from views_frames_summarize.tower import hdi_tower
|
|
24
|
+
from views_frames_summarize.tower_point import tower_point
|
|
21
25
|
|
|
22
26
|
__all__ = [
|
|
27
|
+
"TowerSummary",
|
|
23
28
|
"aggregate_distributions",
|
|
24
29
|
"aggregate_distributions_arrays",
|
|
30
|
+
"bimodality",
|
|
25
31
|
"collapse",
|
|
26
32
|
"hdi",
|
|
33
|
+
"hdi_tower",
|
|
27
34
|
"map_estimate",
|
|
28
35
|
"quantiles",
|
|
36
|
+
"summarize_tower",
|
|
37
|
+
"tower_point",
|
|
29
38
|
]
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Per-row bimodality flag (ADR-019) — a `(N, …, 1)` array aligned to the index.
|
|
2
|
+
|
|
3
|
+
A genuinely two-peaked posterior — a zero atom plus a *distinct* positive bump, or two
|
|
4
|
+
well-separated positive bumps — has no well-defined "most likely single value" and no
|
|
5
|
+
well-defined shortest interval (the shortest 50% interval flips between the peaks under
|
|
6
|
+
tiny perturbations, at *any* grid density). `bimodality` flags those rows so a consumer
|
|
7
|
+
is never handed a single point / interval that silently hides a second mode.
|
|
8
|
+
|
|
9
|
+
The detector is a **deliberately conservative heuristic**, not a formal multimodality
|
|
10
|
+
test. Per row: a coarse, lightly-smoothed histogram; then a count of *separated*
|
|
11
|
+
density regions — runs of bins at least ``prominence`` of the row's peak, with
|
|
12
|
+
sub-``prominence`` valleys between them — keeping only regions that carry at least
|
|
13
|
+
``min_mass`` of the samples. A row is flagged iff ≥ 2 such regions survive. Quiet rows
|
|
14
|
+
(the zero short-circuit) are never flagged.
|
|
15
|
+
|
|
16
|
+
It is tuned (against the research battery) for **zero false positives** on the normal
|
|
17
|
+
regime — right-skewed, zero-inflated and active unimodal posteriors all read as
|
|
18
|
+
unimodal — at the cost of recall on *ambiguous*, overlapping mixtures. That trade is
|
|
19
|
+
intentional: today's models are effectively unimodal, so the flag's job is to catch a
|
|
20
|
+
future regime change that produces *clearly* separated modes, not to adjudicate every
|
|
21
|
+
heavy tail. A missed subtle bump is cheaper than crying wolf on every skewed cell.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
from numpy.typing import NDArray
|
|
28
|
+
|
|
29
|
+
from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
|
|
30
|
+
from views_frames_summarize.tower import _zero_mask
|
|
31
|
+
|
|
32
|
+
_SMOOTH = 3 # moving-average window over bins; tames sparse-histogram flicker
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _coarse_counts(flat: NDArray[np.float32], bins: int) -> NDArray[np.intp]:
|
|
36
|
+
"""Per-row histogram counts ``(rows, bins)`` over each row's ``[min, max]``.
|
|
37
|
+
|
|
38
|
+
A simple clipped linear bucket (not ``numpy.histogram``'s edge-exact path) —
|
|
39
|
+
enough to locate density regions. All-equal rows fall into a single bin.
|
|
40
|
+
"""
|
|
41
|
+
rows = flat.shape[0]
|
|
42
|
+
first = flat.min(axis=1)
|
|
43
|
+
last = flat.max(axis=1)
|
|
44
|
+
span = np.where(first == last, np.float32(1.0), last - first)
|
|
45
|
+
idx = (((flat - first[:, None]) / span[:, None]) * bins).astype(np.intp)
|
|
46
|
+
np.clip(idx, 0, bins - 1, out=idx)
|
|
47
|
+
offsets = idx + (np.arange(rows)[:, None] * bins)
|
|
48
|
+
return np.bincount(offsets.ravel(), minlength=rows * bins).reshape(rows, bins)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _bimodal_block(
|
|
52
|
+
block: NDArray[np.float32], bins: int, prominence: float, min_mass: float
|
|
53
|
+
) -> NDArray[np.float32]:
|
|
54
|
+
"""Flag (0/1) per row of a block: ≥ 2 separated regions each holding enough mass."""
|
|
55
|
+
rows = block.shape[0]
|
|
56
|
+
counts = _coarse_counts(block, bins).astype(np.float64)
|
|
57
|
+
|
|
58
|
+
# Light moving-average smoothing (window _SMOOTH) over the bins. Normalise each
|
|
59
|
+
# position by the number of *real* bins in its window — the two edge bins have one
|
|
60
|
+
# fewer neighbour, so dividing them by the full window would deflate an edge peak
|
|
61
|
+
# (e.g. the zero atom, which always lands in bin 0) and hide an atom+bump bimodal.
|
|
62
|
+
pad = np.zeros((rows, 1))
|
|
63
|
+
padded = np.concatenate([pad, counts, pad], axis=1)
|
|
64
|
+
window_sum = padded[:, :-2] + padded[:, 1:-1] + padded[:, 2:]
|
|
65
|
+
divisor = np.full(bins, _SMOOTH, dtype=np.float64)
|
|
66
|
+
divisor[0] = divisor[-1] = _SMOOTH - 1 # edges contribute one fewer real bin
|
|
67
|
+
smooth = window_sum / divisor
|
|
68
|
+
|
|
69
|
+
significant = smooth >= prominence * smooth.max(axis=1, keepdims=True)
|
|
70
|
+
prev = np.concatenate(
|
|
71
|
+
[np.zeros((rows, 1), dtype=bool), significant[:, :-1]], axis=1
|
|
72
|
+
)
|
|
73
|
+
starts = significant & ~prev
|
|
74
|
+
# Label each maximal run of significant bins with a per-row region index (1-based).
|
|
75
|
+
region = np.where(significant, np.cumsum(starts, axis=1), 0)
|
|
76
|
+
|
|
77
|
+
total = counts.sum(axis=1)
|
|
78
|
+
kept = np.zeros(rows, dtype=np.intp)
|
|
79
|
+
for r in range(
|
|
80
|
+
1, int(region.max()) + 1
|
|
81
|
+
): # <= bins iterations, vectorized over rows
|
|
82
|
+
mass = np.where(region == r, counts, 0.0).sum(axis=1)
|
|
83
|
+
kept += (mass >= min_mass * total).astype(np.intp)
|
|
84
|
+
|
|
85
|
+
flag = (kept >= 2).astype(np.float32)
|
|
86
|
+
flag[_zero_mask(block)] = 0.0
|
|
87
|
+
return flag
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def bimodality(
|
|
91
|
+
frame: AnyFrame,
|
|
92
|
+
*,
|
|
93
|
+
bins: int = 16,
|
|
94
|
+
prominence: float = 0.40,
|
|
95
|
+
min_mass: float = 0.15,
|
|
96
|
+
block_rows: int = ROW_BLOCK,
|
|
97
|
+
) -> NDArray[np.float32]:
|
|
98
|
+
"""Per-row bimodality flag over the sample axis → ``(N, …, 1)`` of ``0.0``/``1.0``.
|
|
99
|
+
|
|
100
|
+
Conservative heuristic (see module docstring): ≥ 2 separated density regions, each
|
|
101
|
+
holding ≥ ``min_mass`` of the samples, after coarse binning + light smoothing.
|
|
102
|
+
Aligned to ``frame.index``.
|
|
103
|
+
"""
|
|
104
|
+
values = frame.values
|
|
105
|
+
lead = values.shape[:-1]
|
|
106
|
+
s = values.shape[-1]
|
|
107
|
+
flat = np.ascontiguousarray(values).reshape(-1, s)
|
|
108
|
+
|
|
109
|
+
def _block(block: NDArray[np.float32]) -> NDArray[np.float32]:
|
|
110
|
+
return _bimodal_block(block, bins, prominence, min_mass)
|
|
111
|
+
|
|
112
|
+
out = block_apply(flat, block_rows, _block)
|
|
113
|
+
return np.asarray(out, dtype=np.float32).reshape(lead)[..., np.newaxis]
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Conformance checks for the summarize package (ADR-016/017).
|
|
2
|
+
|
|
3
|
+
A consumer can re-run these against its own frame factories to confirm the
|
|
4
|
+
summarizers behave: point estimates return same-type `(N, …, 1)` frames; interval
|
|
5
|
+
estimates return arrays aligned to the input frame's rows.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from views_frames_summarize._common import AnyFrame
|
|
13
|
+
from views_frames_summarize.bimodality import bimodality
|
|
14
|
+
from views_frames_summarize.collapse import collapse
|
|
15
|
+
from views_frames_summarize.interval import hdi, quantiles
|
|
16
|
+
from views_frames_summarize.point import map_estimate
|
|
17
|
+
from views_frames_summarize.summarize_tower import summarize_tower
|
|
18
|
+
from views_frames_summarize.tower import hdi_tower
|
|
19
|
+
from views_frames_summarize.tower_point import tower_point
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def assert_summarizer_contract(frame: AnyFrame) -> None:
|
|
23
|
+
"""Assert the summarizers behave on ``frame``.
|
|
24
|
+
|
|
25
|
+
Raises:
|
|
26
|
+
AssertionError: a summarizer violates its output contract.
|
|
27
|
+
"""
|
|
28
|
+
n = frame.n_rows
|
|
29
|
+
|
|
30
|
+
point = collapse(frame, np.mean)
|
|
31
|
+
assert type(point) is type(frame), "collapse must return the same frame type"
|
|
32
|
+
assert point.values.shape[-1] == 1, "collapse must reduce the sample axis to 1"
|
|
33
|
+
assert point.n_rows == n, "collapse must preserve rows"
|
|
34
|
+
|
|
35
|
+
mode = map_estimate(frame)
|
|
36
|
+
assert mode.values.shape[-1] == 1 and mode.n_rows == n, "map_estimate → (N,…,1)"
|
|
37
|
+
|
|
38
|
+
lo_hi = hdi(frame, mass=0.9)
|
|
39
|
+
assert lo_hi.shape[0] == n, "hdi must be aligned to the frame's rows"
|
|
40
|
+
assert lo_hi.shape[-1] == 2, "hdi must produce (lower, upper)"
|
|
41
|
+
|
|
42
|
+
qs = quantiles(frame, [0.1, 0.5, 0.9])
|
|
43
|
+
assert qs.shape[0] == n, "quantiles must be aligned to the frame's rows"
|
|
44
|
+
assert qs.shape[-1] == 3, "quantiles must produce one column per quantile"
|
|
45
|
+
|
|
46
|
+
_assert_tower_contract(frame, n)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _assert_tower_contract(frame: AnyFrame, n: int) -> None:
|
|
50
|
+
"""Assert the constrained-nested tower's contract + its laws (ADR-019)."""
|
|
51
|
+
tip = tower_point(frame)
|
|
52
|
+
assert type(tip) is type(frame), "tower_point must return the same frame type"
|
|
53
|
+
assert tip.values.shape[-1] == 1 and tip.n_rows == n, "tower_point → (N,…,1)"
|
|
54
|
+
|
|
55
|
+
flag = bimodality(frame)
|
|
56
|
+
assert flag.shape[0] == n and flag.shape[-1] == 1, "bimodality → (N,…,1)"
|
|
57
|
+
assert np.isin(flag, (0.0, 1.0)).all(), "bimodality must be a 0/1 flag"
|
|
58
|
+
|
|
59
|
+
tower = hdi_tower(frame, masses=(0.5, 0.9, 0.99))
|
|
60
|
+
assert tower.shape[0] == n, "hdi_tower must be aligned to the frame's rows"
|
|
61
|
+
assert tower.shape[-2:] == (3, 2), "hdi_tower → (…, M, 2)"
|
|
62
|
+
|
|
63
|
+
# Nesting law: every wider HDI contains the next-narrower one.
|
|
64
|
+
lower, upper = tower[..., 0], tower[..., 1]
|
|
65
|
+
assert (np.diff(lower, axis=-1) <= 1e-6).all(), (
|
|
66
|
+
"tower lowers must be non-increasing"
|
|
67
|
+
)
|
|
68
|
+
assert (np.diff(upper, axis=-1) >= -1e-6).all(), (
|
|
69
|
+
"tower uppers must be non-decreasing"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Tip-in-narrowest law: the point lies inside the narrowest requested floor.
|
|
73
|
+
lo0, hi0 = tower[..., 0, 0], tower[..., 0, 1]
|
|
74
|
+
assert (tip.values[..., 0] >= lo0 - 1e-6).all(), "tip below the narrowest floor"
|
|
75
|
+
assert (tip.values[..., 0] <= hi0 + 1e-6).all(), "tip above the narrowest floor"
|
|
76
|
+
|
|
77
|
+
# Reproducibility law: the 50% HDI is independent of the other requested masses.
|
|
78
|
+
just_50 = hdi_tower(frame, masses=(0.5,))
|
|
79
|
+
assert np.array_equal(just_50[..., 0, :], tower[..., 0, :]), (
|
|
80
|
+
"the 50% HDI must be identical whether or not other masses are requested"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# The bundle is exactly the three composable functions.
|
|
84
|
+
bundle = summarize_tower(frame, masses=(0.5, 0.9, 0.99))
|
|
85
|
+
assert np.array_equal(bundle.point.values, tip.values), "bundle point ≠ tower_point"
|
|
86
|
+
assert np.array_equal(bundle.intervals, tower), "bundle intervals ≠ hdi_tower"
|
|
87
|
+
assert np.array_equal(bundle.bimodal, flag), "bundle bimodal ≠ bimodality"
|
|
@@ -35,6 +35,13 @@ def map_estimate(
|
|
|
35
35
|
For each row: if a fraction ``>= zero_mass_threshold`` of the samples is ~0 the
|
|
36
36
|
MAP is ``0.0``; otherwise it is the centre of the densest histogram bin. The
|
|
37
37
|
work runs in row-blocks of ``block_rows`` to bound peak memory (register C-22).
|
|
38
|
+
|
|
39
|
+
Caveat (register C-32): the densest-bin tie-break is the **lowest bin index**
|
|
40
|
+
(deterministic; register C-24). At low sample counts the histogram peak is
|
|
41
|
+
usually a multi-way tie, and lowest-index = leftmost = smallest value — so for
|
|
42
|
+
**right-skewed, zero-inflated** posteriors the mode is biased toward the left
|
|
43
|
+
tail (zero). This is therefore **not a drop-in** for a production histogram-MAP
|
|
44
|
+
on such distributions; a robust mode estimator is tracked separately (#89).
|
|
38
45
|
"""
|
|
39
46
|
values = frame.values
|
|
40
47
|
lead = values.shape[:-1]
|
|
@@ -97,6 +104,9 @@ def _batched_map(flat: NDArray[np.float32], bins: int) -> NDArray[np.float32]:
|
|
|
97
104
|
# the float64 bin widths differ by ~1 ulp across numpy versions and flip the
|
|
98
105
|
# argmax (register C-24). Integer ``argmax`` is deterministic and identical on
|
|
99
106
|
# every numpy build, so ``map_estimate`` is portable and reproducible.
|
|
107
|
+
# NOTE (register C-32): lowest-index == leftmost == smallest value, so on the
|
|
108
|
+
# frequent low-sample ties this biases the mode toward zero for right-skewed,
|
|
109
|
+
# zero-inflated posteriors. Portable but not unbiased; redesign tracked in #89.
|
|
100
110
|
densest = np.argmax(counts, axis=1)
|
|
101
111
|
|
|
102
112
|
lo = np.take_along_axis(edges, densest[:, None], axis=1)[:, 0]
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""The single-pass coherent posterior summary (ADR-019).
|
|
2
|
+
|
|
3
|
+
`summarize_tower` is the dashboard hot path: it sorts each row-block **once** and builds
|
|
4
|
+
the canonical tower **once**, deriving the tower-tip point, the pinned nested HDIs, and
|
|
5
|
+
the bimodality flag together. It returns exactly what the three composable functions
|
|
6
|
+
(`tower_point`, `hdi_tower`, `bimodality`) return — the bundle is purely an efficiency
|
|
7
|
+
collapse of the three, not a different computation.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from typing import NamedTuple
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from numpy.typing import NDArray
|
|
17
|
+
|
|
18
|
+
from views_frames_summarize._common import ROW_BLOCK, AnyFrame, rebuild
|
|
19
|
+
from views_frames_summarize.bimodality import _bimodal_block
|
|
20
|
+
from views_frames_summarize.tower import (
|
|
21
|
+
_CANONICAL_FLOORS,
|
|
22
|
+
_dense_tower,
|
|
23
|
+
_ks,
|
|
24
|
+
_pin,
|
|
25
|
+
_tip,
|
|
26
|
+
_zero_mask,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TowerSummary(NamedTuple):
|
|
31
|
+
"""The coherent summary of a frame's posteriors over the sample axis.
|
|
32
|
+
|
|
33
|
+
Attributes:
|
|
34
|
+
point: ``(N, …, 1)`` frame — the tower-tip point estimate.
|
|
35
|
+
intervals: ``(N, …, M, 2)`` array — nested HDIs at the pinned masses.
|
|
36
|
+
bimodal: ``(N, …, 1)`` array — the per-row bimodality flag (0/1).
|
|
37
|
+
masses: ``(M,)`` array — the canonical floors the requested masses pinned to.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
point: AnyFrame
|
|
41
|
+
intervals: NDArray[np.float32]
|
|
42
|
+
bimodal: NDArray[np.float32]
|
|
43
|
+
masses: NDArray[np.float32]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def summarize_tower(
|
|
47
|
+
frame: AnyFrame,
|
|
48
|
+
masses: Sequence[float] = (0.5, 0.9, 0.99),
|
|
49
|
+
*,
|
|
50
|
+
bins: int = 16,
|
|
51
|
+
prominence: float = 0.40,
|
|
52
|
+
min_mass: float = 0.15,
|
|
53
|
+
block_rows: int = ROW_BLOCK,
|
|
54
|
+
) -> TowerSummary:
|
|
55
|
+
"""Point + nested HDIs + bimodality flag in one sort-once pass → `TowerSummary`."""
|
|
56
|
+
values = frame.values
|
|
57
|
+
lead = values.shape[:-1]
|
|
58
|
+
s = values.shape[-1]
|
|
59
|
+
ks = _ks(s)
|
|
60
|
+
pin = _pin(masses)
|
|
61
|
+
k0 = int(ks[0])
|
|
62
|
+
flat = np.ascontiguousarray(values).reshape(-1, s)
|
|
63
|
+
rows = flat.shape[0]
|
|
64
|
+
|
|
65
|
+
point_flat = np.empty(rows, dtype=np.float32)
|
|
66
|
+
intervals_flat = np.empty((rows, pin.shape[0], 2), dtype=np.float32)
|
|
67
|
+
bimodal_flat = np.empty(rows, dtype=np.float32)
|
|
68
|
+
|
|
69
|
+
for start in range(0, rows, block_rows):
|
|
70
|
+
block = flat[start : start + block_rows]
|
|
71
|
+
srt = np.sort(block, axis=-1)
|
|
72
|
+
zero = _zero_mask(block)
|
|
73
|
+
|
|
74
|
+
sel = _dense_tower(srt, ks)[:, pin, :]
|
|
75
|
+
sel[zero] = 0.0
|
|
76
|
+
tip = np.where(zero, np.float32(0.0), _tip(srt, k0))
|
|
77
|
+
|
|
78
|
+
stop = start + block.shape[0]
|
|
79
|
+
point_flat[start:stop] = tip
|
|
80
|
+
intervals_flat[start:stop] = sel
|
|
81
|
+
bimodal_flat[start:stop] = _bimodal_block(block, bins, prominence, min_mass)
|
|
82
|
+
|
|
83
|
+
point = rebuild(frame, point_flat.reshape(lead)[..., np.newaxis])
|
|
84
|
+
intervals = intervals_flat.reshape(*lead, pin.shape[0], 2)
|
|
85
|
+
bimodal = bimodal_flat.reshape(lead)[..., np.newaxis]
|
|
86
|
+
pinned = _CANONICAL_FLOORS[pin].astype(np.float32)
|
|
87
|
+
return TowerSummary(
|
|
88
|
+
point=point, intervals=intervals, bimodal=bimodal, masses=pinned
|
|
89
|
+
)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""The constrained-nested HDI tower (ADR-019) — interval estimates over the sample axis.
|
|
2
|
+
|
|
3
|
+
`hdi_tower` returns, for each requested mass, the highest-density interval read out
|
|
4
|
+
of a **fixed canonical tower**: a dense set of shortest intervals built inside-out so
|
|
5
|
+
each floor is the shortest interval that *contains* the next-narrower one. Nesting is
|
|
6
|
+
therefore guaranteed **by construction** (no post-hoc "move to nest" patch; register
|
|
7
|
+
C-33), and because the tower is always built on the same fixed `_CANONICAL_FLOORS`
|
|
8
|
+
grid — with requested masses *pinned* to the nearest floor, never inserted — "the 50%
|
|
9
|
+
HDI" is identical regardless of which other masses a caller asks for (reproducibility).
|
|
10
|
+
|
|
11
|
+
The construction is vectorized over the sample axis and runs in row-blocks (register
|
|
12
|
+
C-22/C-25): one sort per block, never a whole-grid sorted copy. A raw-count zero
|
|
13
|
+
short-circuit (`max(row) <= 1.0` → the whole summary collapses to 0) kills the quiet
|
|
14
|
+
cells — the overwhelming majority — cheaply.
|
|
15
|
+
|
|
16
|
+
Returns numpy arrays aligned to the frame's index (the interval convention, ADR-017);
|
|
17
|
+
the caller holds the index. The private engine here (`_dense_tower`, `_tip`, `_pin`,
|
|
18
|
+
`_zero_mask`, `_CANONICAL_FLOORS`) is shared by `tower_point`, `bimodality`, and the
|
|
19
|
+
single-pass `summarize_tower` bundle.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from collections.abc import Sequence
|
|
25
|
+
from typing import Final
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
from numpy.typing import NDArray
|
|
29
|
+
|
|
30
|
+
from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply
|
|
31
|
+
|
|
32
|
+
# The fixed canonical mass grid: a 5% body plus a fixed fine high-mass tail so a
|
|
33
|
+
# requested 0.99 pins to 0.99 (not 0.95). Built from rounded literals — NOT
|
|
34
|
+
# ``np.arange`` (whose float accumulation drifts ~1 ulp across numpy versions and
|
|
35
|
+
# would make the grid, and therefore every pinned interval, non-reproducible).
|
|
36
|
+
_CANONICAL_FLOORS: Final[NDArray[np.float64]] = np.array(
|
|
37
|
+
[round(0.05 * i, 2) for i in range(1, 19)] # 0.05 … 0.90
|
|
38
|
+
+ [0.92, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99], # fine high-mass tail (incl. 0.95)
|
|
39
|
+
dtype=np.float64,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# Raw-count zero rule: a row whose every draw is <= this collapses to 0 (the quiet
|
|
43
|
+
# cell — no meaningful positive mass). Deliberately a *count* rule, distinct from
|
|
44
|
+
# ``map_estimate``'s zero-*mass*-fraction rule, so the two estimators stay independent.
|
|
45
|
+
_ZERO_CUTOFF: Final = 1.0
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _ks(sample_count: int) -> NDArray[np.intp]:
|
|
49
|
+
"""The per-floor window size ``k = floor(mass * S)`` for every canonical floor."""
|
|
50
|
+
return np.asarray(np.floor(_CANONICAL_FLOORS * sample_count), dtype=np.intp)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _pin(masses: Sequence[float]) -> NDArray[np.intp]:
|
|
54
|
+
"""Index of the nearest canonical floor for each requested mass.
|
|
55
|
+
|
|
56
|
+
Deterministic (``argmin`` of the distance); the fixed grid produces no exact
|
|
57
|
+
distance ties, so the choice is unambiguous and reproducible. Fails loud
|
|
58
|
+
(ADR-008) on a mass outside ``(0, 1)`` rather than silently pinning a nonsense
|
|
59
|
+
value to the nearest floor and returning a plausible-looking interval.
|
|
60
|
+
"""
|
|
61
|
+
m = np.asarray(masses, dtype=np.float64)
|
|
62
|
+
if m.size == 0 or not np.all((m > 0.0) & (m < 1.0)):
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"masses must each be in the open interval (0, 1); got {masses!r}"
|
|
65
|
+
)
|
|
66
|
+
dist = np.abs(_CANONICAL_FLOORS[None, :] - m[:, None])
|
|
67
|
+
return np.asarray(np.argmin(dist, axis=1), dtype=np.intp)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _zero_mask(block: NDArray[np.float32]) -> NDArray[np.bool_]:
|
|
71
|
+
"""Rows whose maximum draw is at/below the zero cutoff — the quiet cells."""
|
|
72
|
+
return np.asarray(block.max(axis=-1) <= _ZERO_CUTOFF, dtype=np.bool_)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _shortest(
|
|
76
|
+
srt: NDArray[np.float32], k: int
|
|
77
|
+
) -> tuple[NDArray[np.float32], NDArray[np.float32]]:
|
|
78
|
+
"""Shortest interval holding ``k + 1`` samples, per row of a sorted block.
|
|
79
|
+
|
|
80
|
+
For ``k <= 0`` the floor cannot hold two samples (``S`` below the grid's 1/S
|
|
81
|
+
resolution); it degenerates to the per-row median point — the tower-tip seed.
|
|
82
|
+
"""
|
|
83
|
+
n = srt.shape[-1]
|
|
84
|
+
if k <= 0:
|
|
85
|
+
v = srt[:, n // 2]
|
|
86
|
+
return v.copy(), v.copy()
|
|
87
|
+
widths = srt[:, k:] - srt[:, : n - k]
|
|
88
|
+
i = np.argmin(widths, axis=-1)
|
|
89
|
+
lo = np.take_along_axis(srt, i[:, None], axis=-1)[:, 0]
|
|
90
|
+
hi = np.take_along_axis(srt, (i + k)[:, None], axis=-1)[:, 0]
|
|
91
|
+
return lo, hi
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _shortest_containing(
|
|
95
|
+
srt: NDArray[np.float32],
|
|
96
|
+
k: int,
|
|
97
|
+
plo: NDArray[np.float32],
|
|
98
|
+
phi: NDArray[np.float32],
|
|
99
|
+
) -> tuple[NDArray[np.float32], NDArray[np.float32]]:
|
|
100
|
+
"""Shortest ``k+1``-sample interval that contains the inner floor ``[plo, phi]``.
|
|
101
|
+
|
|
102
|
+
This is what makes the tower nested by construction. ``k <= 0`` inherits the inner
|
|
103
|
+
floor; ``k >= n-1`` is the whole row.
|
|
104
|
+
"""
|
|
105
|
+
n = srt.shape[-1]
|
|
106
|
+
if k <= 0:
|
|
107
|
+
return plo.copy(), phi.copy()
|
|
108
|
+
if k >= n - 1:
|
|
109
|
+
return srt[:, 0].copy(), srt[:, -1].copy()
|
|
110
|
+
starts = srt[:, : n - k]
|
|
111
|
+
ends = srt[:, k:]
|
|
112
|
+
ok = (starts <= plo[:, None]) & (ends >= phi[:, None])
|
|
113
|
+
widths = np.where(ok, ends - starts, np.inf)
|
|
114
|
+
i = np.argmin(widths, axis=-1)
|
|
115
|
+
lo = np.take_along_axis(starts, i[:, None], axis=-1)[:, 0]
|
|
116
|
+
hi = np.take_along_axis(ends, i[:, None], axis=-1)[:, 0]
|
|
117
|
+
# Defensive: the construction guarantees an inner floor is itself a sample window,
|
|
118
|
+
# so a wider containing window always exists — but if that invariant is ever
|
|
119
|
+
# broken, expand minimally rather than emit a non-nested floor.
|
|
120
|
+
bad = ~np.isfinite(widths.min(axis=-1))
|
|
121
|
+
if bad.any(): # pragma: no cover - unreachable while inner ⊂ outer holds
|
|
122
|
+
lo = np.where(bad, np.minimum(srt[:, 0], plo), lo)
|
|
123
|
+
hi = np.where(bad, np.maximum(srt[:, -1], phi), hi)
|
|
124
|
+
return lo, hi
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _dense_tower(srt: NDArray[np.float32], ks: NDArray[np.intp]) -> NDArray[np.float32]:
|
|
128
|
+
"""Build the full constrained-nested tower over a sorted block → ``(rows, F, 2)``.
|
|
129
|
+
|
|
130
|
+
F is the number of canonical floors; each is nested in the next-wider one.
|
|
131
|
+
"""
|
|
132
|
+
rows = srt.shape[0]
|
|
133
|
+
out = np.empty((rows, ks.shape[0], 2), dtype=np.float32)
|
|
134
|
+
plo: NDArray[np.float32] | None = None
|
|
135
|
+
phi: NDArray[np.float32] | None = None
|
|
136
|
+
for j, k in enumerate(ks):
|
|
137
|
+
if plo is None or phi is None:
|
|
138
|
+
lo, hi = _shortest(srt, int(k))
|
|
139
|
+
else:
|
|
140
|
+
lo, hi = _shortest_containing(srt, int(k), plo, phi)
|
|
141
|
+
out[:, j, 0] = lo
|
|
142
|
+
out[:, j, 1] = hi
|
|
143
|
+
plo, phi = lo, hi
|
|
144
|
+
return out
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _tip(srt: NDArray[np.float32], k0: int) -> NDArray[np.float32]:
|
|
148
|
+
"""The tower tip: the median of the narrowest floor's ``k0 + 1`` samples, per row.
|
|
149
|
+
|
|
150
|
+
For ``k0 <= 0`` the floor is the median point itself.
|
|
151
|
+
"""
|
|
152
|
+
n = srt.shape[-1]
|
|
153
|
+
if k0 <= 0:
|
|
154
|
+
return np.array(srt[:, n // 2], dtype=np.float32)
|
|
155
|
+
widths = srt[:, k0:] - srt[:, : n - k0]
|
|
156
|
+
i = np.argmin(widths, axis=-1)
|
|
157
|
+
lo_mid = i + (k0 // 2)
|
|
158
|
+
hi_mid = i + ((k0 + 1) // 2)
|
|
159
|
+
a = np.take_along_axis(srt, lo_mid[:, None], axis=-1)[:, 0]
|
|
160
|
+
b = np.take_along_axis(srt, hi_mid[:, None], axis=-1)[:, 0]
|
|
161
|
+
return np.asarray((a + b) * 0.5, dtype=np.float32)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def hdi_tower(
|
|
165
|
+
frame: AnyFrame,
|
|
166
|
+
masses: Sequence[float] = (0.5, 0.9, 0.99),
|
|
167
|
+
*,
|
|
168
|
+
block_rows: int = ROW_BLOCK,
|
|
169
|
+
) -> NDArray[np.float32]:
|
|
170
|
+
"""Per-row constrained-nested HDIs at the requested ``masses`` → ``(N, …, M, 2)``.
|
|
171
|
+
|
|
172
|
+
Each requested mass is pinned to the nearest fixed canonical floor; the interval is
|
|
173
|
+
read out of the full canonical tower (built once per block). Nested by construction
|
|
174
|
+
and reproducible (a mass's interval is independent of the other requested masses).
|
|
175
|
+
Quiet rows (``max <= 1``) collapse to ``(0, 0)``. Aligned to ``frame.index``.
|
|
176
|
+
"""
|
|
177
|
+
values = frame.values
|
|
178
|
+
lead = values.shape[:-1]
|
|
179
|
+
s = values.shape[-1]
|
|
180
|
+
ks = _ks(s)
|
|
181
|
+
pin = _pin(masses)
|
|
182
|
+
flat = np.ascontiguousarray(values).reshape(-1, s)
|
|
183
|
+
|
|
184
|
+
def _block(block: NDArray[np.float32]) -> NDArray[np.float32]:
|
|
185
|
+
srt = np.sort(block, axis=-1)
|
|
186
|
+
sel = _dense_tower(srt, ks)[:, pin, :]
|
|
187
|
+
sel[_zero_mask(block)] = 0.0
|
|
188
|
+
return sel
|
|
189
|
+
|
|
190
|
+
out = block_apply(flat, block_rows, _block)
|
|
191
|
+
return np.asarray(out, dtype=np.float32).reshape(*lead, pin.shape[0], 2)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""The tower-tip point estimate (ADR-019) — a `(N, …, 1)` frame.
|
|
2
|
+
|
|
3
|
+
`tower_point` is the "most likely single value" we report to a consumer: the median of
|
|
4
|
+
the draws inside the **narrowest canonical floor** of the constrained-nested tower (the
|
|
5
|
+
tip), with the same raw-count zero short-circuit as the tower (`max(row) <= 1` → 0).
|
|
6
|
+
|
|
7
|
+
It is a *new* estimator, deliberately distinct from the frozen `map_estimate` (ADR-018):
|
|
8
|
+
`map_estimate` is a binned histogram mode with a zero-*mass*-fraction rule and a
|
|
9
|
+
lowest-index tie-break that is directionally biased on right-skewed, zero-inflated,
|
|
10
|
+
low-sample posteriors (register C-32). The tower tip is unbinned — it reads the median
|
|
11
|
+
of an actual shortest-interval window — so it carries no histogram tie-break. On a
|
|
12
|
+
genuinely multi-peaked row the tip is, like any single point, ambiguous; pair it with
|
|
13
|
+
`bimodality` to detect that case rather than collapse it silently.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
from numpy.typing import NDArray
|
|
20
|
+
|
|
21
|
+
from views_frames_summarize._common import ROW_BLOCK, AnyFrame, block_apply, rebuild
|
|
22
|
+
from views_frames_summarize.tower import _CANONICAL_FLOORS, _tip, _zero_mask
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def tower_point(frame: AnyFrame, *, block_rows: int = ROW_BLOCK) -> AnyFrame:
|
|
26
|
+
"""Per-row tower-tip point estimate over the sample axis → a `(N, …, 1)` frame.
|
|
27
|
+
|
|
28
|
+
The median of the narrowest canonical floor's samples; ``0.0`` for quiet rows
|
|
29
|
+
(every draw ``<= 1``). Returns a frame of the same type as ``frame``.
|
|
30
|
+
"""
|
|
31
|
+
values = frame.values
|
|
32
|
+
lead = values.shape[:-1]
|
|
33
|
+
s = values.shape[-1]
|
|
34
|
+
k0 = int(np.floor(_CANONICAL_FLOORS[0] * s))
|
|
35
|
+
flat = np.ascontiguousarray(values).reshape(-1, s)
|
|
36
|
+
|
|
37
|
+
def _block(block: NDArray[np.float32]) -> NDArray[np.float32]:
|
|
38
|
+
srt = np.sort(block, axis=-1)
|
|
39
|
+
tip = _tip(srt, k0)
|
|
40
|
+
return np.asarray(
|
|
41
|
+
np.where(_zero_mask(block), np.float32(0.0), tip), dtype=np.float32
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
out = block_apply(flat, block_rows, _block)
|
|
45
|
+
reduced = np.asarray(out, dtype=np.float32).reshape(lead)[..., np.newaxis]
|
|
46
|
+
return rebuild(frame, reduced)
|
views_frames-1.0.0/.gitignore
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
# Python
|
|
2
|
-
__pycache__/
|
|
3
|
-
*.py[cod]
|
|
4
|
-
*.egg-info/
|
|
5
|
-
dist/
|
|
6
|
-
build/
|
|
7
|
-
.eggs/
|
|
8
|
-
|
|
9
|
-
# Virtual environments
|
|
10
|
-
.venv/
|
|
11
|
-
venv/
|
|
12
|
-
|
|
13
|
-
# Tool caches
|
|
14
|
-
.pytest_cache/
|
|
15
|
-
.ruff_cache/
|
|
16
|
-
.mypy_cache/
|
|
17
|
-
|
|
18
|
-
# Environment and secrets
|
|
19
|
-
.env
|
|
20
|
-
.env.*
|
|
21
|
-
|
|
22
|
-
# OS
|
|
23
|
-
.DS_Store
|
|
24
|
-
Thumbs.db
|
|
25
|
-
|
|
26
|
-
# IDE
|
|
27
|
-
.idea/
|
|
28
|
-
.vscode/
|
|
29
|
-
*.swp
|
|
30
|
-
*.swo
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
"""Conformance checks for the summarize package (ADR-016/017).
|
|
2
|
-
|
|
3
|
-
A consumer can re-run these against its own frame factories to confirm the
|
|
4
|
-
summarizers behave: point estimates return same-type `(N, …, 1)` frames; interval
|
|
5
|
-
estimates return arrays aligned to the input frame's rows.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import numpy as np
|
|
11
|
-
|
|
12
|
-
from views_frames_summarize._common import AnyFrame
|
|
13
|
-
from views_frames_summarize.collapse import collapse
|
|
14
|
-
from views_frames_summarize.interval import hdi, quantiles
|
|
15
|
-
from views_frames_summarize.point import map_estimate
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def assert_summarizer_contract(frame: AnyFrame) -> None:
|
|
19
|
-
"""Assert the summarizers behave on ``frame``.
|
|
20
|
-
|
|
21
|
-
Raises:
|
|
22
|
-
AssertionError: a summarizer violates its output contract.
|
|
23
|
-
"""
|
|
24
|
-
n = frame.n_rows
|
|
25
|
-
|
|
26
|
-
point = collapse(frame, np.mean)
|
|
27
|
-
assert type(point) is type(frame), "collapse must return the same frame type"
|
|
28
|
-
assert point.values.shape[-1] == 1, "collapse must reduce the sample axis to 1"
|
|
29
|
-
assert point.n_rows == n, "collapse must preserve rows"
|
|
30
|
-
|
|
31
|
-
mode = map_estimate(frame)
|
|
32
|
-
assert mode.values.shape[-1] == 1 and mode.n_rows == n, "map_estimate → (N,…,1)"
|
|
33
|
-
|
|
34
|
-
lo_hi = hdi(frame, mass=0.9)
|
|
35
|
-
assert lo_hi.shape[0] == n, "hdi must be aligned to the frame's rows"
|
|
36
|
-
assert lo_hi.shape[-1] == 2, "hdi must produce (lower, upper)"
|
|
37
|
-
|
|
38
|
-
qs = quantiles(frame, [0.1, 0.5, 0.9])
|
|
39
|
-
assert qs.shape[0] == n, "quantiles must be aligned to the frame's rows"
|
|
40
|
-
assert qs.shape[-1] == 3, "quantiles must produce one column per quantile"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|