graphed-histogram 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. graphed_histogram-0.0.1/.github/workflows/ci.yml +84 -0
  2. graphed_histogram-0.0.1/.github/workflows/wheels.yml +35 -0
  3. graphed_histogram-0.0.1/.gitignore +16 -0
  4. graphed_histogram-0.0.1/.graphed/M23/attempts.md +62 -0
  5. graphed_histogram-0.0.1/.graphed/M29/attempts.md +46 -0
  6. graphed_histogram-0.0.1/.graphed/state.json +76 -0
  7. graphed_histogram-0.0.1/.pre-commit-config.yaml +30 -0
  8. graphed_histogram-0.0.1/CLAUDE.md +42 -0
  9. graphed_histogram-0.0.1/CONTRIBUTING.md +30 -0
  10. graphed_histogram-0.0.1/PKG-INFO +153 -0
  11. graphed_histogram-0.0.1/README.md +121 -0
  12. graphed_histogram-0.0.1/docs/_templates/autosummary/module.rst +45 -0
  13. graphed_histogram-0.0.1/docs/api.rst +12 -0
  14. graphed_histogram-0.0.1/docs/conf.py +23 -0
  15. graphed_histogram-0.0.1/docs/design.rst +126 -0
  16. graphed_histogram-0.0.1/docs/improvements.rst +24 -0
  17. graphed_histogram-0.0.1/docs/index.rst +25 -0
  18. graphed_histogram-0.0.1/pyproject.toml +80 -0
  19. graphed_histogram-0.0.1/src/graphed_histogram/__init__.py +57 -0
  20. graphed_histogram-0.0.1/src/graphed_histogram/_spec.py +135 -0
  21. graphed_histogram-0.0.1/src/graphed_histogram/boost.py +357 -0
  22. graphed_histogram-0.0.1/tests/frozen/m23/README.md +7 -0
  23. graphed_histogram-0.0.1/tests/frozen/m23/behavior_toy.py +34 -0
  24. graphed_histogram-0.0.1/tests/frozen/m23/test_deferred_histograms.py +293 -0
  25. graphed_histogram-0.0.1/tests/frozen/m23/test_group_plan.py +104 -0
  26. graphed_histogram-0.0.1/tests/frozen/m23/test_process_executor_witnesses.py +137 -0
  27. graphed_histogram-0.0.1/tests/frozen/m29/test_multi_weight_fills.py +121 -0
@@ -0,0 +1,84 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ concurrency:
10
+ group: ci-${{ github.ref }}
11
+ cancel-in-progress: true
12
+
13
+ env:
14
+ # the former core/debug/frontend/numpy/awkward siblings are now one consolidated package
15
+ GRAPHED: "graphed[awkward,numpy] @ git+https://github.com/graphed-org/graphed@main"
16
+ # exec-local stays a separate package (used by the M23 process-executor integration tests)
17
+ EXECLOCAL: "graphed-executors @ git+https://github.com/graphed-org/graphed-executors@main"
18
+
19
+ jobs:
20
+ test:
21
+ name: test ${{ matrix.os }} py${{ matrix.python }}
22
+ runs-on: ${{ matrix.os }}
23
+ strategy:
24
+ fail-fast: false
25
+ matrix:
26
+ os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
27
+ python: ["3.11", "3.12", "3.13", "3.14"]
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+ - uses: actions/setup-python@v5
31
+ with:
32
+ python-version: ${{ matrix.python }}
33
+ - uses: dtolnay/rust-toolchain@stable
34
+ - name: Install deps + this package
35
+ shell: bash
36
+ run: |
37
+ python -m pip install --upgrade pip
38
+ python -m pip install "${{ env.GRAPHED }}" "${{ env.EXECLOCAL }}"
39
+ python -m pip install -e ".[dev]"
40
+ - uses: astral-sh/setup-uv@v5
41
+ - name: Lint + types (pre-commit hooks via prek)
42
+ run: uvx prek@0.4.5 run --all-files --show-diff-on-failure
43
+ - name: Pytest + coverage gate (>=90%)
44
+ run: pytest tests/frozen --cov=graphed_histogram --cov-branch --cov-report=term-missing
45
+
46
+ # Free-threaded / bleeding-edge Pythons (plan A.5): genuinely exercises concurrency under no-GIL.
47
+ # Experimental + non-blocking (awkward wheels may lag on 3.14 / 3.14t).
48
+ test-experimental:
49
+ name: test (experimental) py${{ matrix.python }}
50
+ runs-on: ubuntu-latest
51
+ continue-on-error: true
52
+ strategy:
53
+ fail-fast: false
54
+ matrix:
55
+ python: ["3.14t"]
56
+ steps:
57
+ - uses: actions/checkout@v4
58
+ - uses: actions/setup-python@v5
59
+ with:
60
+ python-version: ${{ matrix.python }}
61
+ - uses: dtolnay/rust-toolchain@stable
62
+ - shell: bash
63
+ run: |
64
+ python -m pip install --upgrade pip
65
+ python -m pip install "${{ env.GRAPHED }}" "${{ env.EXECLOCAL }}"
66
+ python -m pip install -e ".[dev]"
67
+ pytest tests/frozen
68
+
69
+ docs:
70
+ name: docs (sphinx -W)
71
+ runs-on: ubuntu-latest
72
+ steps:
73
+ - uses: actions/checkout@v4
74
+ - uses: actions/setup-python@v5
75
+ with:
76
+ python-version: "3.12"
77
+ - uses: dtolnay/rust-toolchain@stable
78
+ - shell: bash
79
+ run: |
80
+ python -m pip install --upgrade pip
81
+ # runtime deps include the consolidated graphed package (git, not on PyPI)
82
+ python -m pip install "${{ env.GRAPHED }}"
83
+ python -m pip install -e ".[docs]"
84
+ sphinx-build -W -b html docs docs/_build/html
@@ -0,0 +1,35 @@
1
+ name: wheels
2
+
3
+ # Build-only release artifacts (plan A.5 / M0): a pure-Python wheel + sdist, validated with
4
+ # `twine check` and uploaded as a CI artifact. There is intentionally NO publish step — nothing
5
+ # is uploaded to PyPI from CI.
6
+
7
+ on:
8
+ push:
9
+ branches: [main]
10
+ pull_request:
11
+ workflow_dispatch:
12
+
13
+ concurrency:
14
+ group: wheels-${{ github.ref }}
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ build:
19
+ name: sdist + wheel (pure Python)
20
+ runs-on: ubuntu-latest
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+ - uses: actions/setup-python@v5
24
+ with:
25
+ python-version: "3.12"
26
+ - name: Build
27
+ run: |
28
+ python -m pip install --upgrade pip build twine
29
+ python -m build
30
+ - name: Validate metadata
31
+ run: python -m twine check dist/*
32
+ - uses: actions/upload-artifact@v4
33
+ with:
34
+ name: dist
35
+ path: dist/*
@@ -0,0 +1,16 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ venv/
5
+ *.egg-info/
6
+ .pytest_cache/
7
+ .mypy_cache/
8
+ .ruff_cache/
9
+ .coverage
10
+ htmlcov/
11
+ coverage.xml
12
+ dist/
13
+ build/
14
+ _build/
15
+ docs/generated/
16
+ .DS_Store
@@ -0,0 +1,62 @@
1
+ # M23 attempts — graphed-histogram (deferred histogram filling; P0.1)
2
+
3
+ ## Iteration 0 — M0 spine + TEST_AUTHORING/TEST_SANITY/IMPLEMENTING — 2026-06-10 (freeze-M23-0)
4
+
5
+ - Repo created per the lazy-repo recipe (spine adapted from graphed-checkpoint: CI matrix,
6
+ tooling, Sphinx + improvements.rst, CONTRIBUTING, distilled CLAUDE.md).
7
+ - frozen suite tests/frozen/m23 (14 tests); NON-VACUOUS (10/10 initial tests fail on the missing
8
+ package; 4 coverage tests added pre-freeze, recorded).
9
+ - Design (user-confirmed plan): fills are External nodes recorded via graphed M23's
10
+ record_external(descriptor=, form=) — backends know nothing about histograms; identity =
11
+ SHA-256 of the canonical versioned axes/storage spec (declarative JSON; growth axes rejected
12
+ as Phase 2); evaluators resolve through evaluate_ir(externals=); compute() = partition-wise
13
+ fill through the compiled IR over the PartitionedSource protocol (whole-dataset loader never
14
+ invoked, counter-witnessed) + native `+` tree-combine; in-memory sources via materialize;
15
+ plan() exports the R15.4 task graph (ProcessExecutor pin). dask-histogram-parity surface:
16
+ boost.Histogram / factory / histogram / histogram2d / histogramdd.
17
+ - gates: frozen 14/14 PASS · coverage 95.07% (>=90, branch) · ruff+format clean · mypy --strict
18
+ clean · sphinx -W clean · IR byte-determinism pinned in-suite.
19
+
20
+ ## Iteration 1 — the hist.graphed integration surface — 2026-06-10
21
+
22
+ - `_wrap_result` hook: compute() converts to the subclass's `_in_memory_type` when declared
23
+ (the hist.dask convention) — `hist.graphed.Hist.compute()` returns a real `hist.Hist`.
24
+ - Axis identity metadata: boost axes carry user attributes (hist's name/label) in `__dict__`,
25
+ not in a `metadata=` kwarg — the canonical spec now captures/restores `__dict__` entries, so
26
+ names and labels survive record -> compute -> wrap. Spec encode/rebuild remains a fixed point
27
+ (pinned).
28
+
29
+ ## Iteration 2 — USER-DIRECTED: no compute() helper (graphed idiom) — 2026-06-11 (freeze-M23-1)
30
+
31
+ - USER: "graphed has its own [idioms] ... we have materialize and the executors for that.
32
+ Remove compute() from graphed-histogram."
33
+ - Histogram.compute() and the _wrap_result/_in_memory_type machinery are REMOVED. Evaluation is
34
+ graphed's: plan() (R15.4) + any R7 executor's run(plan).value IS the aggregated histogram;
35
+ the reference session.materialize(fill_node) evaluates one fill eagerly (multi-fill in-memory:
36
+ zero_of + add_histograms over per-fill materializes — pinned).
37
+ - frozen m23 respun under this authorization (executor/materialize idiom; same pins otherwise);
38
+ freeze tag bumped freeze-M23-0 -> freeze-M23-1.
39
+ - gates: 14/14 · coverage 94.85% · ruff/mypy/sphinx clean.
40
+
41
+ ## Iteration 3 — USER-DIRECTED process-boundary witnesses (freeze-M23-2) — 2026-06-11
42
+
43
+ - USER: weighted fills are HEP's most common case; behavior loss plagued dask-awkward — control
44
+ both early. New frozen file test_process_executor_witnesses.py (+behavior_toy.py, an
45
+ importable toy behavior): a Weight-storage weighted fill through the SPAWNED process pool ==
46
+ sequential EXACTLY (values AND variances) == eager; a ragged awkward fill through the pool ==
47
+ its eager twin; a behavior-carrying session under the DEFAULT plan() FAILS LOUDLY in workers
48
+ (bare backend — behaviors are never silently dropped; pinned); and the supported path:
49
+ plan(backend="module:attr") — workers IMPORT the factory, no behavior dict is ever pickled —
50
+ reproduces the sequential result bit for bit.
51
+ - Implementation: _resolve_backend (str refs resolved IN the worker); plan(backend=) accepts
52
+ factory/class OR "module:attr". Sanity: 3/4 witnesses passed pre-implementation (they pin
53
+ already-true behavior, including the loud failure); the import-ref test failed on the missing
54
+ feature. CI + dev deps gain graphed-awkward so the ragged/behavior witnesses run in CI.
55
+ - gates: frozen 18/18 · coverage >=90 · ruff/mypy/sphinx clean.
56
+
57
+ ## Iteration 4 — plan(partitions=) (ADL P2 entry-target seam) — 2026-06-11
58
+
59
+ - Additive: plan() accepts explicit partitions (a caller-shaped partitioning — absolute
60
+ entry-count chunks for the benchmark sweep) instead of the source's steps_per_file split.
61
+ Frozen test added (explicit eager partitions tile the dataset; counts bit-for-bit; the
62
+ whole-dataset loader still never runs). 19/19 green.
@@ -0,0 +1,46 @@
1
+ # M29 attempts — graphed-histogram (multiple multiplicative weights)
2
+
3
+ ## Iteration 0 — 2026-06-12 (freeze-M29-0)
4
+
5
+ - The M27 replay contract (preserve eval_histogram n_weights) had no producer. Now fill()
6
+ accepts weight= as a SEQUENCE of graphed Arrays: each factor is a real graph input; params
7
+ gain n_weights ONLY when >1 (single-weight node identity byte-for-byte unchanged — pinned);
8
+ FillEvaluator gains n_weights (default 1: old pickles/evaluators valid) and multiplies the
9
+ factors elementwise before filling — the package's OWN plan()/executor path agrees with the
10
+ preserve replay by sharing the evaluator.
11
+ - frozen m29 (5): two-weight materialize == eager (values AND variances, Weight storage);
12
+ the plan()/SequentialRunner path (per-partition fills: deterministic byte-identical across
13
+ runs, allclose(rtol=1e-12) vs single-pass eager — float summation ORDER differs across
14
+ partitions, an honest pin); three weights + the params contract (n_weights=3, 4 graph
15
+ inputs); single-weight unchanged (no n_weights param); jagged weight factors flatten
16
+ consistently. Non-vacuous: weight=[w1,w2] crashed recording pre-impl.
17
+ - Gates: 24 passed · coverage 95.71% · ruff/format/mypy/sphinx clean. Cross-repo: preserve
18
+ frozen m30 pins the bundle replay; the full 11-repo + 3-fork sweep ran green pre-commit.
19
+
20
+ ## Iteration 1 — 2026-06-12
21
+
22
+ - CI exposed a test-dependency gap invisible to the local cross-repo sweep: the frozen m29
23
+ plan-path test builds its partitioned source with ak.to_parquet -> pyarrow, present locally
24
+ but not in this repo's CI install. pyarrow added to the dev extra (the frozen test is
25
+ untouched). Lesson: a green local sweep validates code, not CI environments — new frozen
26
+ tests must declare their dependencies in the repo they land in.
27
+
28
+ ## Iteration 2 — 2026-06-12 (freeze-M29-1)
29
+
30
+ - CI round 2: ak.to_parquet routes through pyarrow's PANDAS SHIM -> ModuleNotFoundError in CI
31
+ (pandas exists locally only; this ecosystem is deliberately pandas-free). FREEZE AMENDMENT
32
+ (sanctioned, dispute-correction path): the plan-path test's fixture now writes parquet via
33
+ pure pyarrow (pq.write_table) — the assertions are byte-identical, only the fixture I/O
34
+ changed. Re-frozen as freeze-M29-1.
35
+ - Iteration 1's pyarrow dev-dep edit ALSO shipped invalid TOML (a regex grabbed the wrong
36
+ bracket; the tomllib check ran but its failure was swallowed by statement chaining). Both
37
+ failure modes are now ENCODED in graphed-orchestrator's new pre-commit gate
38
+ (python -m graphed_orchestrator.precommit, commit 4f0abf5), which gated THIS commit:
39
+ toml-valid ok, integrity-scan REFREEZE:tests/frozen/m29/... (loud, sanctioned), full suite ok.
40
+
41
+ ## Iteration 3 — 2026-06-12
42
+
43
+ - CI round 3: the pandas dependency is in awkward ITSELF (arrow form derivation calls
44
+ pyarrow's to_pandas_dtype on every from_parquet schema read) — the fixture's write side was
45
+ never the whole story. graphed-awkward's dev extra carries pyarrow+pandas for exactly this
46
+ reason (the established precedent); this repo's dev extra now matches. No test changes.
@@ -0,0 +1,76 @@
1
+ {
2
+ "current": "M23",
3
+ "milestones": {
4
+ "M23": {
5
+ "dispute_count": 0,
6
+ "escalated": false,
7
+ "freeze_tag": "freeze-M23-2",
8
+ "gates": {
9
+ "benchmark": null,
10
+ "coverage": true,
11
+ "determinism": true,
12
+ "frozen_tests": true,
13
+ "integrity_scan": true,
14
+ "lint": true,
15
+ "types": true
16
+ },
17
+ "incident": null,
18
+ "l0_count": 0,
19
+ "metrics_history": [
20
+ {
21
+ "benchmark_ok": null,
22
+ "coverage_ok": true,
23
+ "determinism_ok": true,
24
+ "diff_lines": 0,
25
+ "fail_set_hash": "e3b0c44298fc1c14",
26
+ "integrity_ok": true,
27
+ "iteration_index": 0,
28
+ "lint_ok": true,
29
+ "pass_count": 18,
30
+ "tokens_spent": 0,
31
+ "tree_hash": "local",
32
+ "types_ok": true,
33
+ "wall_clock_s": 0.0
34
+ }
35
+ ],
36
+ "phase": "DONE",
37
+ "reject_count": 0
38
+ },
39
+ "M29": {
40
+ "dispute_count": 0,
41
+ "escalated": false,
42
+ "freeze_tag": "freeze-M29-0",
43
+ "gates": {
44
+ "benchmark": null,
45
+ "coverage": true,
46
+ "determinism": true,
47
+ "frozen_tests": true,
48
+ "integrity_scan": true,
49
+ "lint": true,
50
+ "types": true
51
+ },
52
+ "incident": null,
53
+ "l0_count": 0,
54
+ "metrics_history": [
55
+ {
56
+ "benchmark_ok": null,
57
+ "coverage_ok": true,
58
+ "determinism_ok": true,
59
+ "diff_lines": 0,
60
+ "fail_set_hash": "e3b0c44298fc1c14",
61
+ "integrity_ok": true,
62
+ "iteration_index": 0,
63
+ "lint_ok": true,
64
+ "pass_count": 24,
65
+ "tokens_spent": 0,
66
+ "tree_hash": "local",
67
+ "types_ok": true,
68
+ "wall_clock_s": 0.0
69
+ }
70
+ ],
71
+ "phase": "DONE",
72
+ "reject_count": 0
73
+ }
74
+ },
75
+ "updated_at": "2026-06-12T19:48:11Z"
76
+ }
@@ -0,0 +1,30 @@
1
+ # Lint + type-check hooks, run by prek (https://github.com/j178/prek).
2
+ #
3
+ # These commands live here ONCE and are consumed by every gate: CI (`uvx prek run
4
+ # --all-files`), the graphed-orchestrator pre-commit gate, and a developer's local
5
+ # `prek install`. No more hand-rolled `ruff`/`mypy` shell chains duplicated per workflow.
6
+ #
7
+ # ruff and mypy are `system` hooks on purpose: they use the versions already installed in
8
+ # the project's env (the `dev` extra, intentionally unpinned), so there is a single source
9
+ # of truth for tool versions rather than a second set pinned here that could drift.
10
+ repos:
11
+ - repo: local
12
+ hooks:
13
+ - id: ruff-check
14
+ name: ruff check
15
+ entry: ruff check --force-exclude
16
+ language: system
17
+ types_or: [python, pyi]
18
+ require_serial: true
19
+ - id: ruff-format
20
+ name: ruff format
21
+ entry: ruff format --check --force-exclude
22
+ language: system
23
+ types_or: [python, pyi]
24
+ require_serial: true
25
+ - id: mypy
26
+ name: mypy (strict)
27
+ entry: mypy
28
+ language: system
29
+ pass_filenames: false
30
+ always_run: true
@@ -0,0 +1,42 @@
1
+ # CLAUDE.md — graphed-histogram
2
+
3
+ Defers to the root **`graphed-project/CLAUDE.md`**; the **project plan
4
+ (`graphed-project-plan-gated.md`) always wins.** This file distills milestone **M23**
5
+ (P0.1 of the ADL-benchmarks port, user-confirmed 2026-06-10).
6
+
7
+ ## What this repo is
8
+
9
+ **Deferred boost-histogram/hist filling on graphed task graphs** — the `dask-histogram`
10
+ analogue. A `.fill(...)` RECORDS instead of executing: each fill is an **External node** in the
11
+ graphed IR (the M3 correctionlib/ONNX family), carrying a `PayloadDescriptor` whose
12
+ `content_hash` is the SHA-256 of the **canonical axes/storage spec** (declarative params — never
13
+ cloudpickle; UHI in, UHI out, invent no formats). Backends know NOTHING about histograms: fills
14
+ record through `record_external(descriptor=, form=)` (graphed M23) and evaluate through
15
+ `evaluate_ir`'s `externals=` registry.
16
+
17
+ Aggregation rides the M7/M8 seam with graphed's OWN evaluation idiom (no `compute()` helper —
18
+ user-directed, 2026-06-11): `plan()` builds the `Plan(process=fill-partition-through-the-
19
+ compiled-IR, combine=histogram add, empty=zero-hist)`; an R7 executor's `run(plan).value` IS the
20
+ aggregated histogram; the reference `session.materialize(fill_node)` evaluates a fill eagerly.
21
+ Sources implementing `graphed.write.PartitionedSource` are filled partition by partition (their
22
+ whole-dataset loader is NEVER invoked); Int64 counts are exact under any combine tree, float
23
+ storages are deterministic per fixed-tree executor configuration.
24
+
25
+ ## Surface (dask-histogram parity)
26
+
27
+ - `graphed_histogram.boost.Histogram` — deferred `boost_histogram.Histogram`: `.fill()` records
28
+ and returns self (multiple fills accumulate); `.plan()` exports the task graph.
29
+ - `factory(*arrays, histref=, weight=, sample=)`.
30
+ - numpy-like `histogram` / `histogram2d` / `histogramdd`.
31
+ - All standard boost storages (combine is native `+`); axes Regular/Variable/Integer/
32
+ IntCategory/StrCategory/Boolean. **Phase 2 (do NOT build):** growth axes, dask-style
33
+ persist/delayed beyond Plan export.
34
+
35
+ ## Hard rules
36
+
37
+ Frozen tests under `tests/frozen/m23/` — never weakened. One source family per histogram
38
+ (PartitionedSource or in-memory; mixtures rejected). Ragged fill values flatten at fill time.
39
+ The canonical spec encoding is VERSIONED and byte-stable (the content hash is identity).
40
+
41
+ Gates: ruff + ruff format · mypy --strict · pytest (>=90% branch coverage) · sphinx -W.
42
+ Status: see `.graphed/state.json`.
@@ -0,0 +1,30 @@
1
+ # Contributing to graphed-histogram
2
+
3
+ Part of the `graphed` project, governed by the gated three-role pipeline. The root
4
+ [`graphed-project/CLAUDE.md`](https://github.com/graphed-org/graphed-project-mvp) and the project plan
5
+ are authoritative; the plan always wins.
6
+
7
+ ## Guardrails (M8)
8
+
9
+ - **Local filesystem store only** (no distributed store in MVP); **single machine**.
10
+ - M8 is checkpoint/resume — analysis **preservation** is M9, not here.
11
+ - The canonical durable form is the **serializable IR** (`graphed_core.DurablePlan`), never
12
+ cloudpickle except for genuinely opaque callables (flagged `opaque=True`).
13
+ - Resume must be correct under interruption: **no double-count, no lost partition**; a resumed run
14
+ matches an uninterrupted one bit-for-bit. `task_id` must stay content-addressed (cache-poisoning-safe).
15
+
16
+ ## Integrity rules — NON-NEGOTIABLE (plan A.7 / B.6)
17
+
18
+ Never edit/skip/weaken `tests/frozen/**`; never lower a threshold or relax CI; never stub the thing
19
+ under test. Dispute a frozen test via `.graphed/<Mx>/disputes/<test_id>.md`.
20
+
21
+ ## Local gates
22
+
23
+ ```bash
24
+ pip install "graphed[awkward,numpy] @ git+https://github.com/graphed-org/graphed@main" # needs Rust
25
+ pip install "graphed-executors @ git+https://github.com/graphed-org/graphed-executors@main"
26
+ pip install -e ".[dev,docs]"
27
+ ruff check . && ruff format --check . && mypy
28
+ pytest tests/frozen --cov=graphed_histogram --cov-branch
29
+ sphinx-build -W -b html docs docs/_build/html
30
+ ```
@@ -0,0 +1,153 @@
1
+ Metadata-Version: 2.4
2
+ Name: graphed-histogram
3
+ Version: 0.0.1
4
+ Summary: Deferred boost-histogram/hist filling on graphed task graphs (the dask-histogram analogue)
5
+ Project-URL: Homepage, https://github.com/graphed-org/graphed-histogram
6
+ Author: graphed-org
7
+ License-Expression: MIT
8
+ Classifier: Programming Language :: Python :: 3 :: Only
9
+ Classifier: Programming Language :: Python :: 3.11
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: 3.13
12
+ Requires-Python: >=3.11
13
+ Requires-Dist: boost-histogram>=1.4
14
+ Requires-Dist: graphed
15
+ Requires-Dist: numpy>=1.24
16
+ Provides-Extra: dev
17
+ Requires-Dist: awkward>=2.6; extra == 'dev'
18
+ Requires-Dist: graphed-executors; extra == 'dev'
19
+ Requires-Dist: graphed[awkward,numpy]; extra == 'dev'
20
+ Requires-Dist: hist>=2.7; extra == 'dev'
21
+ Requires-Dist: hypothesis; extra == 'dev'
22
+ Requires-Dist: mypy; extra == 'dev'
23
+ Requires-Dist: pandas; extra == 'dev'
24
+ Requires-Dist: pyarrow; extra == 'dev'
25
+ Requires-Dist: pytest; extra == 'dev'
26
+ Requires-Dist: pytest-cov; extra == 'dev'
27
+ Requires-Dist: ruff; extra == 'dev'
28
+ Provides-Extra: docs
29
+ Requires-Dist: furo; extra == 'docs'
30
+ Requires-Dist: sphinx; extra == 'docs'
31
+ Description-Content-Type: text/markdown
32
+
33
+ # graphed-histogram
34
+
35
+ Deferred [boost-histogram](https://github.com/scikit-hep/boost-histogram) /
36
+ [hist](https://github.com/scikit-hep/hist) filling on [graphed](https://github.com/graphed-org)
37
+ task graphs — the [dask-histogram](https://github.com/dask-contrib/dask-histogram) analogue, built
38
+ on graphed's own evaluation idiom (milestone **M23**; P0.1 of the ADL-benchmarks port).
39
+
40
+ A `.fill(...)` **records** instead of executing. Each fill becomes an **External node** in the
41
+ graphed IR (the same M3 family as correctionlib and ONNX nodes): a call into foreign machinery,
42
+ carried in the IR with reproducibility metadata, evaluated later by a registered evaluator.
43
+ Backends know nothing about histograms — fills record through the frontend's
44
+ `record_external(descriptor=, form=)` seam and resolve through `evaluate_ir`'s `externals=`
45
+ registry.
46
+
47
+ ## The deferred histogram in one example
48
+
49
+ ```python
50
+ import boost_histogram as bh
51
+ import graphed_histogram as gh
52
+ from graphed_core.execution import SequentialRunner
53
+
54
+ h = gh.boost.Histogram(bh.axis.Regular(20, 0.0, 10.0), storage=bh.storage.Int64())
55
+ h.fill(x) # x is a graphed Array: RECORDS a fill node, returns h
56
+ h.fill(x * 0.5 + 1.0) # fills accumulate — more nodes, same histogram
57
+
58
+ plan = h.plan(steps_per_file=4) # the deferred task graph
59
+ result = SequentialRunner().run(plan).value # a CONCRETE boost histogram
60
+ # any R7 executor accepts the same plan:
61
+ # ProcessExecutor(max_workers=4, persistent=True).run(plan).value
62
+ ```
63
+
64
+ The eager boost API stays available on `h` (axes, storage, views of the empty state); what
65
+ changes is that filling stages graph nodes and evaluation belongs to executors.
66
+
67
+ ## Why it is built this way
68
+
69
+ - **Fills are External nodes.** The package supplies a `PayloadDescriptor`
70
+ (`kind="histogram"`, `content_hash=sha256(spec)`, `io_schema="uhi"`) and an opaque histogram
71
+ form; the backend is never consulted. Nothing in graphed-core, graphed, or any backend mentions
72
+ histograms.
73
+ - **The canonical spec is the identity.** A histogram's identity is the SHA-256 of its
74
+ **canonical, versioned axes/storage spec** — key-sorted JSON covering every supported axis and
75
+ storage (declarative params, never cloudpickle; UHI in, UHI out, no invented formats). Identical
76
+ fills intern to one graph node; the spec string is the fill's preservation payload; a plan
77
+ re-run on another machine resolves its evaluator by the same hash. `spec_of(h)` reads it;
78
+ `zero_of(spec)` rebuilds the empty histogram anywhere.
79
+ - **Aggregation is plans and executors, not `compute()`.** There is deliberately no `compute()`
80
+ method — evaluation is graphed's machinery. `h.plan(...)` builds a
81
+ `Plan(process=fill-partition-through-the-compiled-IR, combine=native +, empty=zero)`, and any R7
82
+ executor's `run(plan).value` **is** the aggregated histogram. Histograms form a monoid under
83
+ native `+` for every standard storage, so the executor's fixed combine tree applies unchanged:
84
+ Int64 counts are exact under any tree, float storages are deterministic per fixed-tree executor
85
+ configuration. The reference path for in-memory sources is `session.materialize(fill_node)`.
86
+
87
+ ## Public surface
88
+
89
+ | | |
90
+ |---|---|
91
+ | `graphed_histogram.boost.Histogram` | deferred `boost_histogram.Histogram`; `.fill()` records and returns self (fills accumulate), `.plan()` exports the task graph |
92
+ | `factory(*arrays, histref=, weight=, sample=)` | a deferred histogram from a reference histogram's axes/storage plus one staged fill (the dask-histogram `factory` shape) |
93
+ | `histogram` / `histogram2d` / `histogramdd` | numpy-like deferred entry points (explicit bins + range) |
94
+ | `plan(histograms, ...)` | one plan aggregating **several** deferred histograms that share a source in a **single pass** (the `compute(dict_of_hists)` analogue) |
95
+ | `spec_of` / `zero_of` / `content_hash` | the canonical-spec helpers |
96
+ | `evaluators(*histograms)` | merged content-hash -> evaluator registry for `evaluate_ir(externals=...)` |
97
+ | `add_histograms` | native-`+` combine helper for multi-fill sums |
98
+
99
+ All standard boost storages (combine is native `+`); axes
100
+ Regular/Variable/Integer/IntCategory/StrCategory/Boolean. Sources implementing
101
+ `graphed.write.PartitionedSource` are filled partition by partition — their whole-dataset loader is
102
+ never invoked. One source family per histogram (PartitionedSource or in-memory; mixtures rejected).
103
+ Ragged fill values flatten completely at fill time.
104
+
105
+ ### Multiplicative weights (M29)
106
+
107
+ HEP event weights arrive as several factors (generator weight x pileup x trigger SFs ...).
108
+ `fill(weight=...)` accepts `weight=` as a **sequence** of graphed Arrays, a first-class fill
109
+ signature: each weight is recorded as a real graph input and evaluation multiplies them
110
+ elementwise into the single fill weight.
111
+
112
+ ```python
113
+ h.fill(g.pt, weight=[g.genweight, g.pileup_sf, g.trigger_sf])
114
+ ```
115
+
116
+ A single weight records exactly as before — no `n_weights` param — so pre-M29 node identities,
117
+ specs, and preservation bundles are untouched.
118
+
119
+ ### One pass over several histograms
120
+
121
+ `plan(histograms, ...)` compiles all the fills of several histograms into **one** multi-output IR,
122
+ so a sub-graph feeding multiple histograms (e.g. a trijet selection feeding both a pT and a b-tag
123
+ histogram) is read and evaluated **once** — not once per histogram. `run(plan).value` is the
124
+ matching `{label: histogram}` mapping (string keys for a `Mapping`, `"0"`, `"1"`, ... for a plain
125
+ sequence). Column projection covers the union of all the histograms' fills.
126
+
127
+ ### Worker backends
128
+
129
+ `plan(backend=...)` accepts a zero-arg factory/class or an importable `"module:attr"` string
130
+ resolved **in the worker** — the required form for behavior-carrying backends, because behavior
131
+ dicts contain lambdas and do not pickle. A worker built without required behaviors fails loudly; it
132
+ never silently fills the wrong thing.
133
+
134
+ ## The hist integration
135
+
136
+ `hist.graphed` (in the `hist` fork) supplies `Hist`/`NamedHist` as thin MRO sandwiches over this
137
+ package's `Histogram`: the familiar QuickConstruct
138
+ (`Hist.new.Reg(100, 0, 200, name="met").Double()`) and named-axis fills record deferred; executor
139
+ results wrap back into in-memory `hist.Hist` objects with names and labels intact (they ride the
140
+ canonical spec). The eight ADL benchmark queries run on exactly this surface.
141
+
142
+ ## Phase 2 (deliberately not built)
143
+
144
+ Growth axes (combining grown category axes across partitions needs a category-union merge,
145
+ rejected at spec time for now); dask-style collection protocols (`persist`, `to_delayed`) — the
146
+ durable artifact is the compiled IR / `Plan`; behavior-reference forwarding by default.
147
+
148
+ ## Status and gates
149
+
150
+ Frozen tests under `tests/frozen/m23/` and `tests/frozen/m29/` — never weakened. Gates: ruff +
151
+ ruff format · `mypy --strict` · pytest (>= 90% branch coverage) · `sphinx -W`. See `CLAUDE.md`
152
+ for the milestone digest, `docs/design.rst` for the engineering walkthrough, and
153
+ `.graphed/state.json` for current status.