graphed-histogram 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphed_histogram-0.0.1/.github/workflows/ci.yml +84 -0
- graphed_histogram-0.0.1/.github/workflows/wheels.yml +35 -0
- graphed_histogram-0.0.1/.gitignore +16 -0
- graphed_histogram-0.0.1/.graphed/M23/attempts.md +62 -0
- graphed_histogram-0.0.1/.graphed/M29/attempts.md +46 -0
- graphed_histogram-0.0.1/.graphed/state.json +76 -0
- graphed_histogram-0.0.1/.pre-commit-config.yaml +30 -0
- graphed_histogram-0.0.1/CLAUDE.md +42 -0
- graphed_histogram-0.0.1/CONTRIBUTING.md +30 -0
- graphed_histogram-0.0.1/PKG-INFO +153 -0
- graphed_histogram-0.0.1/README.md +121 -0
- graphed_histogram-0.0.1/docs/_templates/autosummary/module.rst +45 -0
- graphed_histogram-0.0.1/docs/api.rst +12 -0
- graphed_histogram-0.0.1/docs/conf.py +23 -0
- graphed_histogram-0.0.1/docs/design.rst +126 -0
- graphed_histogram-0.0.1/docs/improvements.rst +24 -0
- graphed_histogram-0.0.1/docs/index.rst +25 -0
- graphed_histogram-0.0.1/pyproject.toml +80 -0
- graphed_histogram-0.0.1/src/graphed_histogram/__init__.py +57 -0
- graphed_histogram-0.0.1/src/graphed_histogram/_spec.py +135 -0
- graphed_histogram-0.0.1/src/graphed_histogram/boost.py +357 -0
- graphed_histogram-0.0.1/tests/frozen/m23/README.md +7 -0
- graphed_histogram-0.0.1/tests/frozen/m23/behavior_toy.py +34 -0
- graphed_histogram-0.0.1/tests/frozen/m23/test_deferred_histograms.py +293 -0
- graphed_histogram-0.0.1/tests/frozen/m23/test_group_plan.py +104 -0
- graphed_histogram-0.0.1/tests/frozen/m23/test_process_executor_witnesses.py +137 -0
- graphed_histogram-0.0.1/tests/frozen/m29/test_multi_weight_fills.py +121 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: ci-${{ github.ref }}
|
|
11
|
+
cancel-in-progress: true
|
|
12
|
+
|
|
13
|
+
env:
|
|
14
|
+
# the former core/debug/frontend/numpy/awkward siblings are now one consolidated package
|
|
15
|
+
GRAPHED: "graphed[awkward,numpy] @ git+https://github.com/graphed-org/graphed@main"
|
|
16
|
+
# exec-local stays a separate package (used by the M23 process-executor integration tests)
|
|
17
|
+
EXECLOCAL: "graphed-executors @ git+https://github.com/graphed-org/graphed-executors@main"
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
test:
|
|
21
|
+
name: test ${{ matrix.os }} py${{ matrix.python }}
|
|
22
|
+
runs-on: ${{ matrix.os }}
|
|
23
|
+
strategy:
|
|
24
|
+
fail-fast: false
|
|
25
|
+
matrix:
|
|
26
|
+
os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
|
|
27
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
- uses: actions/setup-python@v5
|
|
31
|
+
with:
|
|
32
|
+
python-version: ${{ matrix.python }}
|
|
33
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
34
|
+
- name: Install deps + this package
|
|
35
|
+
shell: bash
|
|
36
|
+
run: |
|
|
37
|
+
python -m pip install --upgrade pip
|
|
38
|
+
python -m pip install "${{ env.GRAPHED }}" "${{ env.EXECLOCAL }}"
|
|
39
|
+
python -m pip install -e ".[dev]"
|
|
40
|
+
- uses: astral-sh/setup-uv@v5
|
|
41
|
+
- name: Lint + types (pre-commit hooks via prek)
|
|
42
|
+
run: uvx prek@0.4.5 run --all-files --show-diff-on-failure
|
|
43
|
+
- name: Pytest + coverage gate (>=90%)
|
|
44
|
+
run: pytest tests/frozen --cov=graphed_histogram --cov-branch --cov-report=term-missing
|
|
45
|
+
|
|
46
|
+
# Free-threaded / bleeding-edge Pythons (plan A.5): genuinely exercises concurrency under no-GIL.
|
|
47
|
+
# Experimental + non-blocking (awkward wheels may lag on 3.14 / 3.14t).
|
|
48
|
+
test-experimental:
|
|
49
|
+
name: test (experimental) py${{ matrix.python }}
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
continue-on-error: true
|
|
52
|
+
strategy:
|
|
53
|
+
fail-fast: false
|
|
54
|
+
matrix:
|
|
55
|
+
python: ["3.14t"]
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/checkout@v4
|
|
58
|
+
- uses: actions/setup-python@v5
|
|
59
|
+
with:
|
|
60
|
+
python-version: ${{ matrix.python }}
|
|
61
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
62
|
+
- shell: bash
|
|
63
|
+
run: |
|
|
64
|
+
python -m pip install --upgrade pip
|
|
65
|
+
python -m pip install "${{ env.GRAPHED }}" "${{ env.EXECLOCAL }}"
|
|
66
|
+
python -m pip install -e ".[dev]"
|
|
67
|
+
pytest tests/frozen
|
|
68
|
+
|
|
69
|
+
docs:
|
|
70
|
+
name: docs (sphinx -W)
|
|
71
|
+
runs-on: ubuntu-latest
|
|
72
|
+
steps:
|
|
73
|
+
- uses: actions/checkout@v4
|
|
74
|
+
- uses: actions/setup-python@v5
|
|
75
|
+
with:
|
|
76
|
+
python-version: "3.12"
|
|
77
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
78
|
+
- shell: bash
|
|
79
|
+
run: |
|
|
80
|
+
python -m pip install --upgrade pip
|
|
81
|
+
# runtime deps include the consolidated graphed package (git, not on PyPI)
|
|
82
|
+
python -m pip install "${{ env.GRAPHED }}"
|
|
83
|
+
python -m pip install -e ".[docs]"
|
|
84
|
+
sphinx-build -W -b html docs docs/_build/html
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
name: wheels
|
|
2
|
+
|
|
3
|
+
# Build-only release artifacts (plan A.5 / M0): a pure-Python wheel + sdist, validated with
|
|
4
|
+
# `twine check` and uploaded as a CI artifact. There is intentionally NO publish step — nothing
|
|
5
|
+
# is uploaded to PyPI from CI.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
push:
|
|
9
|
+
branches: [main]
|
|
10
|
+
pull_request:
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
concurrency:
|
|
14
|
+
group: wheels-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
build:
|
|
19
|
+
name: sdist + wheel (pure Python)
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
- uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
- name: Build
|
|
27
|
+
run: |
|
|
28
|
+
python -m pip install --upgrade pip build twine
|
|
29
|
+
python -m build
|
|
30
|
+
- name: Validate metadata
|
|
31
|
+
run: python -m twine check dist/*
|
|
32
|
+
- uses: actions/upload-artifact@v4
|
|
33
|
+
with:
|
|
34
|
+
name: dist
|
|
35
|
+
path: dist/*
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# M23 attempts — graphed-histogram (deferred histogram filling; P0.1)
|
|
2
|
+
|
|
3
|
+
## Iteration 0 — M0 spine + TEST_AUTHORING/TEST_SANITY/IMPLEMENTING — 2026-06-10 (freeze-M23-0)
|
|
4
|
+
|
|
5
|
+
- Repo created per the lazy-repo recipe (spine adapted from graphed-checkpoint: CI matrix,
|
|
6
|
+
tooling, Sphinx + improvements.rst, CONTRIBUTING, distilled CLAUDE.md).
|
|
7
|
+
- frozen suite tests/frozen/m23 (14 tests); NON-VACUOUS (10/10 initial tests fail on the missing
|
|
8
|
+
package; 4 coverage tests added pre-freeze, recorded).
|
|
9
|
+
- Design (user-confirmed plan): fills are External nodes recorded via graphed M23's
|
|
10
|
+
record_external(descriptor=, form=) — backends know nothing about histograms; identity =
|
|
11
|
+
SHA-256 of the canonical versioned axes/storage spec (declarative JSON; growth axes rejected
|
|
12
|
+
as Phase 2); evaluators resolve through evaluate_ir(externals=); compute() = partition-wise
|
|
13
|
+
fill through the compiled IR over the PartitionedSource protocol (whole-dataset loader never
|
|
14
|
+
invoked, counter-witnessed) + native `+` tree-combine; in-memory sources via materialize;
|
|
15
|
+
plan() exports the R15.4 task graph (ProcessExecutor pin). dask-histogram-parity surface:
|
|
16
|
+
boost.Histogram / factory / histogram / histogram2d / histogramdd.
|
|
17
|
+
- gates: frozen 14/14 PASS · coverage 95.07% (>=90, branch) · ruff+format clean · mypy --strict
|
|
18
|
+
clean · sphinx -W clean · IR byte-determinism pinned in-suite.
|
|
19
|
+
|
|
20
|
+
## Iteration 1 — the hist.graphed integration surface — 2026-06-10
|
|
21
|
+
|
|
22
|
+
- `_wrap_result` hook: compute() converts to the subclass's `_in_memory_type` when declared
|
|
23
|
+
(the hist.dask convention) — `hist.graphed.Hist.compute()` returns a real `hist.Hist`.
|
|
24
|
+
- Axis identity metadata: boost axes carry user attributes (hist's name/label) in `__dict__`,
|
|
25
|
+
not in a `metadata=` kwarg — the canonical spec now captures/restores `__dict__` entries, so
|
|
26
|
+
names and labels survive record -> compute -> wrap. Spec encode/rebuild remains a fixed point
|
|
27
|
+
(pinned).
|
|
28
|
+
|
|
29
|
+
## Iteration 2 — USER-DIRECTED: no compute() helper (graphed idiom) — 2026-06-11 (freeze-M23-1)
|
|
30
|
+
|
|
31
|
+
- USER: "graphed has its own [idioms] ... we have materialize and the executors for that.
|
|
32
|
+
Remove compute() from graphed-histogram."
|
|
33
|
+
- Histogram.compute() and the _wrap_result/_in_memory_type machinery are REMOVED. Evaluation is
|
|
34
|
+
graphed's: plan() (R15.4) + any R7 executor's run(plan).value IS the aggregated histogram;
|
|
35
|
+
the reference session.materialize(fill_node) evaluates one fill eagerly (multi-fill in-memory:
|
|
36
|
+
zero_of + add_histograms over per-fill materializes — pinned).
|
|
37
|
+
- frozen m23 respun under this authorization (executor/materialize idiom; same pins otherwise);
|
|
38
|
+
freeze tag bumped freeze-M23-0 -> freeze-M23-1.
|
|
39
|
+
- gates: 14/14 · coverage 94.85% · ruff/mypy/sphinx clean.
|
|
40
|
+
|
|
41
|
+
## Iteration 3 — USER-DIRECTED process-boundary witnesses (freeze-M23-2) — 2026-06-11
|
|
42
|
+
|
|
43
|
+
- USER: weighted fills are HEP's most common case; behavior loss plagued dask-awkward — control
|
|
44
|
+
both early. New frozen file test_process_executor_witnesses.py (+behavior_toy.py, an
|
|
45
|
+
importable toy behavior): a Weight-storage weighted fill through the SPAWNED process pool ==
|
|
46
|
+
sequential EXACTLY (values AND variances) == eager; a ragged awkward fill through the pool ==
|
|
47
|
+
its eager twin; a behavior-carrying session under the DEFAULT plan() FAILS LOUDLY in workers
|
|
48
|
+
(bare backend — behaviors are never silently dropped; pinned); and the supported path:
|
|
49
|
+
plan(backend="module:attr") — workers IMPORT the factory, no behavior dict is ever pickled —
|
|
50
|
+
reproduces the sequential result bit for bit.
|
|
51
|
+
- Implementation: _resolve_backend (str refs resolved IN the worker); plan(backend=) accepts
|
|
52
|
+
factory/class OR "module:attr". Sanity: 3/4 witnesses passed pre-implementation (they pin
|
|
53
|
+
already-true behavior, including the loud failure); the import-ref test failed on the missing
|
|
54
|
+
feature. CI + dev deps gain graphed-awkward so the ragged/behavior witnesses run in CI.
|
|
55
|
+
- gates: frozen 18/18 · coverage >=90 · ruff/mypy/sphinx clean.
|
|
56
|
+
|
|
57
|
+
## Iteration 4 — plan(partitions=) (ADL P2 entry-target seam) — 2026-06-11
|
|
58
|
+
|
|
59
|
+
- Additive: plan() accepts explicit partitions (a caller-shaped partitioning — absolute
|
|
60
|
+
entry-count chunks for the benchmark sweep) instead of the source's steps_per_file split.
|
|
61
|
+
Frozen test added (explicit eager partitions tile the dataset; counts bit-for-bit; the
|
|
62
|
+
whole-dataset loader still never runs). 19/19 green.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# M29 attempts — graphed-histogram (multiple multiplicative weights)
|
|
2
|
+
|
|
3
|
+
## Iteration 0 — 2026-06-12 (freeze-M29-0)
|
|
4
|
+
|
|
5
|
+
- The M27 replay contract (preserve eval_histogram n_weights) had no producer. Now fill()
|
|
6
|
+
accepts weight= as a SEQUENCE of graphed Arrays: each factor is a real graph input; params
|
|
7
|
+
gain n_weights ONLY when >1 (single-weight node identity byte-for-byte unchanged — pinned);
|
|
8
|
+
FillEvaluator gains n_weights (default 1: old pickles/evaluators valid) and multiplies the
|
|
9
|
+
factors elementwise before filling — the package's OWN plan()/executor path agrees with the
|
|
10
|
+
preserve replay by sharing the evaluator.
|
|
11
|
+
- frozen m29 (5): two-weight materialize == eager (values AND variances, Weight storage);
|
|
12
|
+
the plan()/SequentialRunner path (per-partition fills: deterministic byte-identical across
|
|
13
|
+
runs, allclose(rtol=1e-12) vs single-pass eager — float summation ORDER differs across
|
|
14
|
+
partitions, an honest pin); three weights + the params contract (n_weights=3, 4 graph
|
|
15
|
+
inputs); single-weight unchanged (no n_weights param); jagged weight factors flatten
|
|
16
|
+
consistently. Non-vacuous: weight=[w1,w2] crashed recording pre-impl.
|
|
17
|
+
- Gates: 24 passed · coverage 95.71% · ruff/format/mypy/sphinx clean. Cross-repo: preserve
|
|
18
|
+
frozen m30 pins the bundle replay; the full 11-repo + 3-fork sweep ran green pre-commit.
|
|
19
|
+
|
|
20
|
+
## Iteration 1 — 2026-06-12
|
|
21
|
+
|
|
22
|
+
- CI exposed a test-dependency gap invisible to the local cross-repo sweep: the frozen m29
|
|
23
|
+
plan-path test builds its partitioned source with ak.to_parquet -> pyarrow, present locally
|
|
24
|
+
but not in this repo's CI install. pyarrow added to the dev extra (the frozen test is
|
|
25
|
+
untouched). Lesson: a green local sweep validates code, not CI environments — new frozen
|
|
26
|
+
tests must declare their dependencies in the repo they land in.
|
|
27
|
+
|
|
28
|
+
## Iteration 2 — 2026-06-12 (freeze-M29-1)
|
|
29
|
+
|
|
30
|
+
- CI round 2: ak.to_parquet routes through pyarrow's PANDAS SHIM -> ModuleNotFoundError in CI
|
|
31
|
+
(pandas exists locally only; this ecosystem is deliberately pandas-free). FREEZE AMENDMENT
|
|
32
|
+
(sanctioned, dispute-correction path): the plan-path test's fixture now writes parquet via
|
|
33
|
+
pure pyarrow (pq.write_table) — the assertions are byte-identical, only the fixture I/O
|
|
34
|
+
changed. Re-frozen as freeze-M29-1.
|
|
35
|
+
- Iteration 1's pyarrow dev-dep edit ALSO shipped invalid TOML (a regex grabbed the wrong
|
|
36
|
+
bracket; the tomllib check ran but its failure was swallowed by statement chaining). Both
|
|
37
|
+
failure modes are now ENCODED in graphed-orchestrator's new pre-commit gate
|
|
38
|
+
(python -m graphed_orchestrator.precommit, commit 4f0abf5), which gated THIS commit:
|
|
39
|
+
toml-valid ok, integrity-scan REFREEZE:tests/frozen/m29/... (loud, sanctioned), full suite ok.
|
|
40
|
+
|
|
41
|
+
## Iteration 3 — 2026-06-12
|
|
42
|
+
|
|
43
|
+
- CI round 3: the pandas dependency is in awkward ITSELF (arrow form derivation calls
|
|
44
|
+
pyarrow's to_pandas_dtype on every from_parquet schema read) — the fixture's write side was
|
|
45
|
+
never the whole story. graphed-awkward's dev extra carries pyarrow+pandas for exactly this
|
|
46
|
+
reason (the established precedent); this repo's dev extra now matches. No test changes.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
{
|
|
2
|
+
"current": "M23",
|
|
3
|
+
"milestones": {
|
|
4
|
+
"M23": {
|
|
5
|
+
"dispute_count": 0,
|
|
6
|
+
"escalated": false,
|
|
7
|
+
"freeze_tag": "freeze-M23-2",
|
|
8
|
+
"gates": {
|
|
9
|
+
"benchmark": null,
|
|
10
|
+
"coverage": true,
|
|
11
|
+
"determinism": true,
|
|
12
|
+
"frozen_tests": true,
|
|
13
|
+
"integrity_scan": true,
|
|
14
|
+
"lint": true,
|
|
15
|
+
"types": true
|
|
16
|
+
},
|
|
17
|
+
"incident": null,
|
|
18
|
+
"l0_count": 0,
|
|
19
|
+
"metrics_history": [
|
|
20
|
+
{
|
|
21
|
+
"benchmark_ok": null,
|
|
22
|
+
"coverage_ok": true,
|
|
23
|
+
"determinism_ok": true,
|
|
24
|
+
"diff_lines": 0,
|
|
25
|
+
"fail_set_hash": "e3b0c44298fc1c14",
|
|
26
|
+
"integrity_ok": true,
|
|
27
|
+
"iteration_index": 0,
|
|
28
|
+
"lint_ok": true,
|
|
29
|
+
"pass_count": 18,
|
|
30
|
+
"tokens_spent": 0,
|
|
31
|
+
"tree_hash": "local",
|
|
32
|
+
"types_ok": true,
|
|
33
|
+
"wall_clock_s": 0.0
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"phase": "DONE",
|
|
37
|
+
"reject_count": 0
|
|
38
|
+
},
|
|
39
|
+
"M29": {
|
|
40
|
+
"dispute_count": 0,
|
|
41
|
+
"escalated": false,
|
|
42
|
+
"freeze_tag": "freeze-M29-0",
|
|
43
|
+
"gates": {
|
|
44
|
+
"benchmark": null,
|
|
45
|
+
"coverage": true,
|
|
46
|
+
"determinism": true,
|
|
47
|
+
"frozen_tests": true,
|
|
48
|
+
"integrity_scan": true,
|
|
49
|
+
"lint": true,
|
|
50
|
+
"types": true
|
|
51
|
+
},
|
|
52
|
+
"incident": null,
|
|
53
|
+
"l0_count": 0,
|
|
54
|
+
"metrics_history": [
|
|
55
|
+
{
|
|
56
|
+
"benchmark_ok": null,
|
|
57
|
+
"coverage_ok": true,
|
|
58
|
+
"determinism_ok": true,
|
|
59
|
+
"diff_lines": 0,
|
|
60
|
+
"fail_set_hash": "e3b0c44298fc1c14",
|
|
61
|
+
"integrity_ok": true,
|
|
62
|
+
"iteration_index": 0,
|
|
63
|
+
"lint_ok": true,
|
|
64
|
+
"pass_count": 24,
|
|
65
|
+
"tokens_spent": 0,
|
|
66
|
+
"tree_hash": "local",
|
|
67
|
+
"types_ok": true,
|
|
68
|
+
"wall_clock_s": 0.0
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
"phase": "DONE",
|
|
72
|
+
"reject_count": 0
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
"updated_at": "2026-06-12T19:48:11Z"
|
|
76
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Lint + type-check hooks, run by prek (https://github.com/j178/prek).
|
|
2
|
+
#
|
|
3
|
+
# These commands live here ONCE and are consumed by every gate: CI (`uvx prek run
|
|
4
|
+
# --all-files`), the graphed-orchestrator pre-commit gate, and a developer's local
|
|
5
|
+
# `prek install`. No more hand-rolled `ruff`/`mypy` shell chains duplicated per workflow.
|
|
6
|
+
#
|
|
7
|
+
# ruff and mypy are `system` hooks on purpose: they use the versions already installed in
|
|
8
|
+
# the project's env (the `dev` extra, intentionally unpinned), so there is a single source
|
|
9
|
+
# of truth for tool versions rather than a second set pinned here that could drift.
|
|
10
|
+
repos:
|
|
11
|
+
- repo: local
|
|
12
|
+
hooks:
|
|
13
|
+
- id: ruff-check
|
|
14
|
+
name: ruff check
|
|
15
|
+
entry: ruff check --force-exclude
|
|
16
|
+
language: system
|
|
17
|
+
types_or: [python, pyi]
|
|
18
|
+
require_serial: true
|
|
19
|
+
- id: ruff-format
|
|
20
|
+
name: ruff format
|
|
21
|
+
entry: ruff format --check --force-exclude
|
|
22
|
+
language: system
|
|
23
|
+
types_or: [python, pyi]
|
|
24
|
+
require_serial: true
|
|
25
|
+
- id: mypy
|
|
26
|
+
name: mypy (strict)
|
|
27
|
+
entry: mypy
|
|
28
|
+
language: system
|
|
29
|
+
pass_filenames: false
|
|
30
|
+
always_run: true
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# CLAUDE.md — graphed-histogram
|
|
2
|
+
|
|
3
|
+
Defers to the root **`graphed-project/CLAUDE.md`**; the **project plan
|
|
4
|
+
(`graphed-project-plan-gated.md`) always wins.** This file distills milestone **M23**
|
|
5
|
+
(P0.1 of the ADL-benchmarks port, user-confirmed 2026-06-10).
|
|
6
|
+
|
|
7
|
+
## What this repo is
|
|
8
|
+
|
|
9
|
+
**Deferred boost-histogram/hist filling on graphed task graphs** — the `dask-histogram`
|
|
10
|
+
analogue. A `.fill(...)` RECORDS instead of executing: each fill is an **External node** in the
|
|
11
|
+
graphed IR (the M3 correctionlib/ONNX family), carrying a `PayloadDescriptor` whose
|
|
12
|
+
`content_hash` is the SHA-256 of the **canonical axes/storage spec** (declarative params — never
|
|
13
|
+
cloudpickle; UHI in, UHI out, invent no formats). Backends know NOTHING about histograms: fills
|
|
14
|
+
record through `record_external(descriptor=, form=)` (graphed M23) and evaluate through
|
|
15
|
+
`evaluate_ir`'s `externals=` registry.
|
|
16
|
+
|
|
17
|
+
Aggregation rides the M7/M8 seam with graphed's OWN evaluation idiom (no `compute()` helper —
|
|
18
|
+
user-directed, 2026-06-11): `plan()` builds the `Plan(process=fill-partition-through-the-
|
|
19
|
+
compiled-IR, combine=histogram add, empty=zero-hist)`; an R7 executor's `run(plan).value` IS the
|
|
20
|
+
aggregated histogram; the reference `session.materialize(fill_node)` evaluates a fill eagerly.
|
|
21
|
+
Sources implementing `graphed.write.PartitionedSource` are filled partition by partition (their
|
|
22
|
+
whole-dataset loader is NEVER invoked); Int64 counts are exact under any combine tree, float
|
|
23
|
+
storages are deterministic per fixed-tree executor configuration.
|
|
24
|
+
|
|
25
|
+
## Surface (dask-histogram parity)
|
|
26
|
+
|
|
27
|
+
- `graphed_histogram.boost.Histogram` — deferred `boost_histogram.Histogram`: `.fill()` records
|
|
28
|
+
and returns self (multiple fills accumulate); `.plan()` exports the task graph.
|
|
29
|
+
- `factory(*arrays, histref=, weight=, sample=)`.
|
|
30
|
+
- numpy-like `histogram` / `histogram2d` / `histogramdd`.
|
|
31
|
+
- All standard boost storages (combine is native `+`); axes Regular/Variable/Integer/
|
|
32
|
+
IntCategory/StrCategory/Boolean. **Phase 2 (do NOT build):** growth axes, dask-style
|
|
33
|
+
persist/delayed beyond Plan export.
|
|
34
|
+
|
|
35
|
+
## Hard rules
|
|
36
|
+
|
|
37
|
+
Frozen tests under `tests/frozen/m23/` — never weakened. One source family per histogram
|
|
38
|
+
(PartitionedSource or in-memory; mixtures rejected). Ragged fill values flatten at fill time.
|
|
39
|
+
The canonical spec encoding is VERSIONED and byte-stable (the content hash is identity).
|
|
40
|
+
|
|
41
|
+
Gates: ruff + ruff format · mypy --strict · pytest (>=90% branch coverage) · sphinx -W.
|
|
42
|
+
Status: see `.graphed/state.json`.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Contributing to graphed-histogram
|
|
2
|
+
|
|
3
|
+
Part of the `graphed` project, governed by the gated three-role pipeline. The root
|
|
4
|
+
[`graphed-project/CLAUDE.md`](https://github.com/graphed-org/graphed-project-mvp) and the project plan
|
|
5
|
+
are authoritative; the plan always wins.
|
|
6
|
+
|
|
7
|
+
## Guardrails (M8)
|
|
8
|
+
|
|
9
|
+
- **Local filesystem store only** (no distributed store in MVP); **single machine**.
|
|
10
|
+
- M8 is checkpoint/resume — analysis **preservation** is M9, not here.
|
|
11
|
+
- The canonical durable form is the **serializable IR** (`graphed_core.DurablePlan`), never
|
|
12
|
+
cloudpickle except for genuinely opaque callables (flagged `opaque=True`).
|
|
13
|
+
- Resume must be correct under interruption: **no double-count, no lost partition**; a resumed run
|
|
14
|
+
matches an uninterrupted one bit-for-bit. `task_id` must stay content-addressed (cache-poisoning-safe).
|
|
15
|
+
|
|
16
|
+
## Integrity rules — NON-NEGOTIABLE (plan A.7 / B.6)
|
|
17
|
+
|
|
18
|
+
Never edit/skip/weaken `tests/frozen/**`; never lower a threshold or relax CI; never stub the thing
|
|
19
|
+
under test. Dispute a frozen test via `.graphed/<Mx>/disputes/<test_id>.md`.
|
|
20
|
+
|
|
21
|
+
## Local gates
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install "graphed[awkward,numpy] @ git+https://github.com/graphed-org/graphed@main" # needs Rust
|
|
25
|
+
pip install "graphed-executors @ git+https://github.com/graphed-org/graphed-executors@main"
|
|
26
|
+
pip install -e ".[dev,docs]"
|
|
27
|
+
ruff check . && ruff format --check . && mypy
|
|
28
|
+
pytest tests/frozen --cov=graphed_histogram --cov-branch
|
|
29
|
+
sphinx-build -W -b html docs docs/_build/html
|
|
30
|
+
```
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: graphed-histogram
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Deferred boost-histogram/hist filling on graphed task graphs (the dask-histogram analogue)
|
|
5
|
+
Project-URL: Homepage, https://github.com/graphed-org/graphed-histogram
|
|
6
|
+
Author: graphed-org
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: boost-histogram>=1.4
|
|
14
|
+
Requires-Dist: graphed
|
|
15
|
+
Requires-Dist: numpy>=1.24
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: awkward>=2.6; extra == 'dev'
|
|
18
|
+
Requires-Dist: graphed-executors; extra == 'dev'
|
|
19
|
+
Requires-Dist: graphed[awkward,numpy]; extra == 'dev'
|
|
20
|
+
Requires-Dist: hist>=2.7; extra == 'dev'
|
|
21
|
+
Requires-Dist: hypothesis; extra == 'dev'
|
|
22
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
23
|
+
Requires-Dist: pandas; extra == 'dev'
|
|
24
|
+
Requires-Dist: pyarrow; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
28
|
+
Provides-Extra: docs
|
|
29
|
+
Requires-Dist: furo; extra == 'docs'
|
|
30
|
+
Requires-Dist: sphinx; extra == 'docs'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# graphed-histogram
|
|
34
|
+
|
|
35
|
+
Deferred [boost-histogram](https://github.com/scikit-hep/boost-histogram) /
|
|
36
|
+
[hist](https://github.com/scikit-hep/hist) filling on [graphed](https://github.com/graphed-org)
|
|
37
|
+
task graphs — the [dask-histogram](https://github.com/dask-contrib/dask-histogram) analogue, built
|
|
38
|
+
on graphed's own evaluation idiom (milestone **M23**; P0.1 of the ADL-benchmarks port).
|
|
39
|
+
|
|
40
|
+
A `.fill(...)` **records** instead of executing. Each fill becomes an **External node** in the
|
|
41
|
+
graphed IR (the same M3 family as correctionlib and ONNX nodes): a call into foreign machinery,
|
|
42
|
+
carried in the IR with reproducibility metadata, evaluated later by a registered evaluator.
|
|
43
|
+
Backends know nothing about histograms — fills record through the frontend's
|
|
44
|
+
`record_external(descriptor=, form=)` seam and resolve through `evaluate_ir`'s `externals=`
|
|
45
|
+
registry.
|
|
46
|
+
|
|
47
|
+
## The deferred histogram in one example
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import boost_histogram as bh
|
|
51
|
+
import graphed_histogram as gh
|
|
52
|
+
from graphed_core.execution import SequentialRunner
|
|
53
|
+
|
|
54
|
+
h = gh.boost.Histogram(bh.axis.Regular(20, 0.0, 10.0), storage=bh.storage.Int64())
|
|
55
|
+
h.fill(x) # x is a graphed Array: RECORDS a fill node, returns h
|
|
56
|
+
h.fill(x * 0.5 + 1.0) # fills accumulate — more nodes, same histogram
|
|
57
|
+
|
|
58
|
+
plan = h.plan(steps_per_file=4) # the deferred task graph
|
|
59
|
+
result = SequentialRunner().run(plan).value # a CONCRETE boost histogram
|
|
60
|
+
# any R7 executor accepts the same plan:
|
|
61
|
+
# ProcessExecutor(max_workers=4, persistent=True).run(plan).value
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The eager boost API stays available on `h` (axes, storage, views of the empty state); what
|
|
65
|
+
changes is that filling stages graph nodes and evaluation belongs to executors.
|
|
66
|
+
|
|
67
|
+
## Why it is built this way
|
|
68
|
+
|
|
69
|
+
- **Fills are External nodes.** The package supplies a `PayloadDescriptor`
|
|
70
|
+
(`kind="histogram"`, `content_hash=sha256(spec)`, `io_schema="uhi"`) and an opaque histogram
|
|
71
|
+
form; the backend is never consulted. Nothing in graphed-core, graphed, or any backend mentions
|
|
72
|
+
histograms.
|
|
73
|
+
- **The canonical spec is the identity.** A histogram's identity is the SHA-256 of its
|
|
74
|
+
**canonical, versioned axes/storage spec** — key-sorted JSON covering every supported axis and
|
|
75
|
+
storage (declarative params, never cloudpickle; UHI in, UHI out, no invented formats). Identical
|
|
76
|
+
fills intern to one graph node; the spec string is the fill's preservation payload; a plan
|
|
77
|
+
re-run on another machine resolves its evaluator by the same hash. `spec_of(h)` reads it;
|
|
78
|
+
`zero_of(spec)` rebuilds the empty histogram anywhere.
|
|
79
|
+
- **Aggregation is plans and executors, not `compute()`.** There is deliberately no `compute()`
|
|
80
|
+
method — evaluation is graphed's machinery. `h.plan(...)` builds a
|
|
81
|
+
`Plan(process=fill-partition-through-the-compiled-IR, combine=native +, empty=zero)`, and any R7
|
|
82
|
+
executor's `run(plan).value` **is** the aggregated histogram. Histograms form a monoid under
|
|
83
|
+
native `+` for every standard storage, so the executor's fixed combine tree applies unchanged:
|
|
84
|
+
Int64 counts are exact under any tree, float storages are deterministic per fixed-tree executor
|
|
85
|
+
configuration. The reference path for in-memory sources is `session.materialize(fill_node)`.
|
|
86
|
+
|
|
87
|
+
## Public surface
|
|
88
|
+
|
|
89
|
+
| | |
|
|
90
|
+
|---|---|
|
|
91
|
+
| `graphed_histogram.boost.Histogram` | deferred `boost_histogram.Histogram`; `.fill()` records and returns self (fills accumulate), `.plan()` exports the task graph |
|
|
92
|
+
| `factory(*arrays, histref=, weight=, sample=)` | a deferred histogram from a reference histogram's axes/storage plus one staged fill (the dask-histogram `factory` shape) |
|
|
93
|
+
| `histogram` / `histogram2d` / `histogramdd` | numpy-like deferred entry points (explicit bins + range) |
|
|
94
|
+
| `plan(histograms, ...)` | one plan aggregating **several** deferred histograms that share a source in a **single pass** (the `compute(dict_of_hists)` analogue) |
|
|
95
|
+
| `spec_of` / `zero_of` / `content_hash` | the canonical-spec helpers |
|
|
96
|
+
| `evaluators(*histograms)` | merged content-hash -> evaluator registry for `evaluate_ir(externals=...)` |
|
|
97
|
+
| `add_histograms` | native-`+` combine helper for multi-fill sums |
|
|
98
|
+
|
|
99
|
+
All standard boost storages (combine is native `+`); axes
|
|
100
|
+
Regular/Variable/Integer/IntCategory/StrCategory/Boolean. Sources implementing
|
|
101
|
+
`graphed.write.PartitionedSource` are filled partition by partition — their whole-dataset loader is
|
|
102
|
+
never invoked. One source family per histogram (PartitionedSource or in-memory; mixtures rejected).
|
|
103
|
+
Ragged fill values flatten completely at fill time.
|
|
104
|
+
|
|
105
|
+
### Multiplicative weights (M29)
|
|
106
|
+
|
|
107
|
+
HEP event weights arrive as several factors (generator weight x pileup x trigger SFs ...).
|
|
108
|
+
`fill(weight=...)` accepts `weight=` as a **sequence** of graphed Arrays, a first-class fill
|
|
109
|
+
signature: each weight is recorded as a real graph input and evaluation multiplies them
|
|
110
|
+
elementwise into the single fill weight.
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
h.fill(g.pt, weight=[g.genweight, g.pileup_sf, g.trigger_sf])
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
A single weight records exactly as before — no `n_weights` param — so pre-M29 node identities,
|
|
117
|
+
specs, and preservation bundles are untouched.
|
|
118
|
+
|
|
119
|
+
### One pass over several histograms
|
|
120
|
+
|
|
121
|
+
`plan(histograms, ...)` compiles all the fills of several histograms into **one** multi-output IR,
|
|
122
|
+
so a sub-graph feeding multiple histograms (e.g. a trijet selection feeding both a pT and a b-tag
|
|
123
|
+
histogram) is read and evaluated **once** — not once per histogram. `run(plan).value` is the
|
|
124
|
+
matching `{label: histogram}` mapping (string keys for a `Mapping`, `"0"`, `"1"`, ... for a plain
|
|
125
|
+
sequence). Column projection covers the union of all the histograms' fills.
|
|
126
|
+
|
|
127
|
+
### Worker backends
|
|
128
|
+
|
|
129
|
+
`plan(backend=...)` accepts a zero-arg factory/class or an importable `"module:attr"` string
|
|
130
|
+
resolved **in the worker** — the required form for behavior-carrying backends, because behavior
|
|
131
|
+
dicts contain lambdas and do not pickle. A worker built without required behaviors fails loudly; it
|
|
132
|
+
never silently fills the wrong thing.
|
|
133
|
+
|
|
134
|
+
## The hist integration
|
|
135
|
+
|
|
136
|
+
`hist.graphed` (in the `hist` fork) supplies `Hist`/`NamedHist` as thin MRO sandwiches over this
|
|
137
|
+
package's `Histogram`: the familiar QuickConstruct
|
|
138
|
+
(`Hist.new.Reg(100, 0, 200, name="met").Double()`) and named-axis fills record deferred; executor
|
|
139
|
+
results wrap back into in-memory `hist.Hist` objects with names and labels intact (they ride the
|
|
140
|
+
canonical spec). The eight ADL benchmark queries run on exactly this surface.
|
|
141
|
+
|
|
142
|
+
## Phase 2 (deliberately not built)
|
|
143
|
+
|
|
144
|
+
Growth axes (combining grown category axes across partitions needs a category-union merge,
|
|
145
|
+
rejected at spec time for now); dask-style collection protocols (`persist`, `to_delayed`) — the
|
|
146
|
+
durable artifact is the compiled IR / `Plan`; behavior-reference forwarding by default.
|
|
147
|
+
|
|
148
|
+
## Status and gates
|
|
149
|
+
|
|
150
|
+
Frozen tests under `tests/frozen/m23/` and `tests/frozen/m29/` — never weakened. Gates: ruff +
|
|
151
|
+
ruff format · `mypy --strict` · pytest (>= 90% branch coverage) · `sphinx -W`. See `CLAUDE.md`
|
|
152
|
+
for the milestone digest, `docs/design.rst` for the engineering walkthrough, and
|
|
153
|
+
`.graphed/state.json` for current status.
|