graphed-executors 0.0.1__tar.gz

graphed_executors-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # the former core/debug/frontend/numpy/awkward siblings are now one consolidated package
+  GRAPHED: "graphed[awkward,numpy] @ git+https://github.com/graphed-org/graphed@main"
+  # corpus stays a separate package (test fixtures only, not shipped in the graphed wheel)
+  CORPUS: "graphed-corpus @ git+https://github.com/graphed-org/graphed-corpus-mvp@main"
+
+jobs:
+  test:
+    name: test ${{ matrix.os }} py${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+        python: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install deps + this package
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install "${{ env.GRAPHED }}" "${{ env.CORPUS }}"
+          python -m pip install -e ".[dev]"
+      - uses: astral-sh/setup-uv@v5
+      - name: Lint + types (pre-commit hooks via prek)
+        run: uvx prek@0.4.5 run --all-files --show-diff-on-failure
+      - name: Pytest + coverage gate (>=90%)
+        run: pytest tests/frozen --cov=graphed_executors --cov-branch --cov-report=term-missing
+
+  # Free-threaded / bleeding-edge Pythons (plan A.5): genuinely exercises concurrency under no-GIL.
+  # Experimental + non-blocking (awkward wheels may lag on 3.14 / 3.14t).
+  test-experimental:
+    name: test (experimental) py${{ matrix.python }}
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.14t"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: dtolnay/rust-toolchain@stable
+      - shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install "${{ env.GRAPHED }}" "${{ env.CORPUS }}"
+          python -m pip install -e ".[dev]"
+          pytest tests/frozen
+
+  docs:
+    name: docs (sphinx -W)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install "${{ env.GRAPHED }}"
+          # docs only needs runtime deps + sphinx; the dev-extra backends are not on PyPI.
+          python -m pip install -e ".[docs]"
+          sphinx-build -W -b html docs docs/_build/html

graphed_executors-0.0.1/.github/workflows/wheels.yml

@@ -0,0 +1,35 @@
+name: wheels
+
+# Build-only release artifacts (plan A.5 / M0): a pure-Python wheel + sdist, validated with
+# `twine check` and uploaded as a CI artifact. There is intentionally NO publish step — nothing
+# is uploaded to PyPI from CI.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: wheels-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: sdist + wheel (pure Python)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+      - name: Validate metadata
+        run: python -m twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/*

graphed_executors-0.0.1/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+coverage.xml
+dist/
+build/
+_build/
+docs/generated/
+.DS_Store

graphed_executors-0.0.1/.graphed/M0/attempts.md

@@ -0,0 +1,24 @@
+## Iteration 0 — phase REVIEW — 2026-06-05T19:08:22Z
+
+- summary: local gates green; ci_confirmed=False
+- gates: {'frozen_tests': True, 'coverage': True, 'lint': True, 'types': True, 'determinism': True, 'benchmark': None, 'integrity_scan': True}
+- l0_count=0 escalated=False reject_count=0
+
+## Iteration 0 — phase DONE — 2026-06-05T19:28:21Z
+
+- summary: local gates green; ci_confirmed=True
+- gates: {'frozen_tests': True, 'coverage': True, 'lint': True, 'types': True, 'determinism': True, 'benchmark': None, 'integrity_scan': True}
+- l0_count=0 escalated=False reject_count=0
+
+
+## 2026-06-11 — persistent worker pools (additive; the ADL-notebook/sweep finding)
+
+- Spawning a fresh import-heavy pool per run() dwarfs small-plan work (the ADL notebook's eight
+  queries ran 3x SLOWER parallel than sequential on a 50k skim). persistent=True (opt-in) keeps
+  ONE pool across run() calls — witnessed by worker-global state surviving between runs — with
+  close()/context-manager release and lazy respawn; the DEFAULT (fresh pool per run) is pinned
+  unchanged. New frozen file tests/frozen/m7/test_persistent_pool.py (5 tests; 4/5 failed
+  pre-impl, the default pin passed by design). Measured: the eight ADL queries at 2.0x (50k
+  skim) and 2.8x (400k, 8 files) with a persistent x4 pool vs sequential.
+- Implementation note: a blanket text replace recursed _acquired_pool into itself (caught by
+  the suite, fixed pre-commit).

graphed_executors-0.0.1/.graphed/M31/attempts.md

@@ -0,0 +1,26 @@
+# M31 attempts — graphed-exec-local (ship the process to workers once, not per task)
+
+## Iteration 0 — 2026-06-13 (freeze-M31-0)
+
+- MEASURED finding (notebook speedup audit): the persistent-pool per-task framework round-trip
+  is ~0.1ms (no-op task), NOT the 40-80ms I had wrongly claimed; pickle of the 13.8KB process
+  is ~0.01ms. So at the current scale this is negligible — but concurrent.futures re-pickles
+  and re-ships the `process` callable on EVERY submit (it does not dedupe callables), so a Plan
+  whose process embeds a large compiled IR (or an inlined model) pays that wire cost per
+  partition. The user asked for the architecturally-correct ship-once design regardless.
+- DESIGN: pickle the process ONCE in the driver; broadcast those bytes to every worker; cache
+  worker-side in a module global keyed by sha256(content); submit only (token, partition) per
+  task. Broadcast = a pid-coverage loop (concurrent.futures exposes no worker identity): submit
+  priming tasks that each hold 2ms (so siblings each claim one) and return os.getpid(), until
+  the pid set covers pool._max_workers; idempotent, so extra hits are harmless. Cached per
+  (pool, token): re-running the same plan, or a persistent pool across plans, never re-broadcasts
+  the same process; close()/respawn clears the token set. Threads share memory -> no delivery
+  (direct submit). _entry() replaced by _prepare(pool, process)->submit(partition) on both
+  executors; all three submit paths (fixed, pooled-combine, adaptive) route through it.
+- frozen m31 (4) + probes: CountingProcess records per-WORKER unpickle count via __setstate__;
+  with 40 tasks/4 workers the process is unpickled exactly once per worker (counts == {1}),
+  whereas per-task shipping gave {1..10} (non-vacuity: pinned that exact pre-impl spread); a
+  2MB process does not scale per-task; numeric results unchanged + byte-identical across runs;
+  ThreadExecutor unaffected.
+- Gates (via python -m graphed_orchestrator.precommit): 75 passed · coverage >=90 · ruff/mypy/
+  sphinx clean · toml/yaml/integrity ok.

graphed_executors-0.0.1/.graphed/M34/attempts.md

@@ -0,0 +1,24 @@
+# M34 attempts — graphed-exec-local (bounded shared cache, broadcast coverage, LocalResources dedup)
+
+## Iteration 0 — 2026-06-13 (freeze-M34-0)
+
+- Review findings: P0-2 the M31 _shared_objects cache never evicted (a persistent pool over many
+  distinct plans accumulated every compiled-IR process per worker); P1-3 the broadcast loop
+  marked a token primed even if coverage was not reached (latent KeyError on an unprimed
+  worker) and reached into the private pool._max_workers; P3-6 LocalResources was duplicated
+  (its own copy vs graphed_core's M33 bounded one).
+- FIX: _shared_objects is a FIFO OrderedDict capped at _SHARED_CACHE_CAP=32, evicting oldest in
+  BROADCAST order — identical across workers (every worker sees the same broadcast sequence), so
+  it stays in lockstep with the driver's _broadcast_tokens (now also a FIFO OrderedDict, same
+  cap, same eviction). Re-running an evicted plan re-broadcasts transparently. Broadcast now
+  RAISES if it cannot prime all workers (never silently caches an under-primed token); a dead
+  worker surfaces as BrokenProcessPool at f.result(). max_workers is resolved eagerly in
+  __init__ (os.cpu_count() fallback) so the broadcast target needs no private pool attribute.
+  resources.py deleted; LocalResources reused from graphed_core.execution (now bounded+closeable
+  -> the open_once handle leak P0-1 is fixed for the executors too, for free).
+- frozen m34 (4): LocalResources IS graphed_core's (dedup pin); cache stays <= cap across cap+6
+  distinct plans AND demonstrably evicts (size >= cap-2, not merely never-filled); an evicted
+  plan re-runs correctly after re-broadcast; broadcast covers every worker (40 tasks/4 workers).
+  Non-vacuous (the size<=cap + dedup-identity assertions fail against the unbounded/duplicate
+  pre-impl). m31 ship-once suite unaffected.
+- Gates green via the precommit script.

graphed_executors-0.0.1/.graphed/M38/attempts.md

@@ -0,0 +1,260 @@
+# M38 — inter-worker comms, peer reduction, work-stealing (attempts log)
+
+> **freeze-M38-6 (2026-06-17):** sanctioned refreeze of the two IPC-pool transport tests, on the project
+> owner's directive to make the pool choice **explicit** rather than a silent runtime switch. The P7
+> overlay work had `ProcessExecutor` pick the full-registry vs identity-pinned pool internally (via
+> `_use_pinned_pool(w)` + a `GRAPHED_PEER_PINNED` env override), which is opaque at the call site. Split
+> into two public executors — `ProcessPoolExecutor` (full-registry, the default = original M7 behaviour)
+> and `PinnedPoolExecutor` (identity-pinned, bounded O(log N) overlay) — sharing all machinery via
+> `_ProcessExecutorBase` (the only difference is the class attr `_peer_pool_is_pinned`). `ProcessExecutor`
+> stays as a **deprecated alias** for `ProcessPoolExecutor` (17 frozen files use the name; warns on
+> construction). The silent switch is gone; the fd-limit predicate is repurposed (`_exceeds_fd_budget`,
+> no env override) to **warn** — recommending `PinnedPoolExecutor` — when a full-registry pool's worker
+> count would strain the per-process fd limit. Frozen edits, both sanctioned: `test_pooled_transport.py`
+> now targets `ProcessPoolExecutor` explicitly (+ a new non-vacuous test that the warning fires and the
+> run still proceeds — no silent switch); `test_pinned_transport.py` targets `PinnedPoolExecutor`
+> explicitly (env override dropped). No assertion weakened. Notebook benchmark unchanged (the default
+> path is byte-for-byte the prior full-registry path).
+
+> **freeze-M38-5 (2026-06-16):** sanctioned refreeze of `test_peer_robustness.py` only — the profiling
+> witness `_spin` was a pure-Python busy loop holding the GIL, which intermittently starved the
+> GIL-needing off-thread sampler to ZERO samples on slow py3.14 macOS/Windows CI (a timing-flaky
+> witness, R0.10a). Fixed by releasing the GIL each step (`time.sleep`) + a longer budget, so the
+> sampler reliably lands samples on any machine — mimicking how the real analysis releases the GIL in
+> array kernels. No assertion weakened. (Surfaced by the Manager-removal commit's CI; unrelated to that
+> change — the HTTP path it failed on was untouched.)
+
+Scope deviation (recorded): the plan §F lists work-stealing + distributed executors as Phase 2 and
+§A.4 scopes this repo single-machine. The project owner pulled **inter-worker communication + peer
+reduction + work-stealing** into MVP, keeping the executor single-machine but building the transport
+seam so a future distributed executor reuses it unchanged. Root prompt **R21** binds this.
+
+## Post-M38 perf fix: the peer collection tail-join (2026-06-16)
+
+Owner reported a perceived "work-stealing regression in IPC mode". Investigated by replicating the
+ADL notebook's cell-32 benchmark (8 queries, one combined plan, persistent 4-worker pool, warm + 50
+samples) and sweeping configs (`coffea-benchmarks-graphed-mvp/bench_*.py`). **Measured, not assumed
+(R0.11):**
+
+- **Stealing is NOT the cause.** `steal=True` ≈ `steal=False` within noise (±2%) in every mode
+  (no-monitor / dashboard / dashboard+profile); `steals=0` on balanced loads; the steal-loop's
+  coordination cost is ~3 ms. Lengthening the steal poll would not help (and slightly hurts the tail).
+- **The regression is peer-vs-hub** (the default flip): +48 % at 8 files, +7.6 % at 32 files — a
+  roughly FIXED per-run overhead that amortizes at scale.
+- **Decomposed** (1 leaf/worker, shared-wall-clock instrumentation, since reverted): dispatch+setup
+  4 ms, cross-worker reduce 2 ms, compute 111 ms — i.e. coordination is negligible. The consistent,
+  removable cost was the **driver tail-join**: after the root was already in hand, `_collect_peer`
+  polled ``f.done()`` on a 20 ms cadence waiting for workers to notice the ``done`` broadcast (~25-30 ms
+  every run). **Fix:** on the no-monitor fast path, block on the futures' completion (woken instantly)
+  instead of polling. A/B (same session, stash): peer 249.7 → 224.1 ms (−25.6 ms); gap +48 % → +23 %.
+  `open_once` is warm (open-count stable across runs — not a re-open issue).
+- **Residual ~40 ms** (peer compute makespan > hub) is **contention-sensitive** — near-zero on a quiet
+  machine (a low-load timeline showed peer 179 ≈ hub 170 ms), inflating under load.
+
+### Residual root cause + fix: remove the Manager server (py-spy)
+
+py-spy (``--subprocesses``, same benchmark) showed the peer path ran **9 processes vs the hub's 6**,
+including a ``multiprocessing.Manager`` **server process consuming 31 % of sampled thread-time** doing
+pure socket-IPC (``_recv`` 76 % + ``accept`` 21 %, zero compute). The IPC ``QueueTransport`` used
+``Manager().Queue()`` proxies *because they are picklable* (passable as per-submit args); every queue
+op was a socket-RPC to that server, and the extra process + threads inflated worker compute under load.
+Workers ran the identical compute in both paths (awkward kernels / ``decompress`` / ``_carry``) — no
+peer-specific algorithm.
+
+**Fix:** create the inbox queues as **raw ``mp.Queue``** in the driver and **inherit** them in every
+worker via the pool ``initializer`` (``peer_pool_init``); the actor (``pooled_peer_actor``) resolves
+its inbox/outboxes from that process-global registry by address (a cheap string submit-arg). No Manager
+server, native pipes. Notebook ratification (8 files, 50 samples): peer-vs-hub gap **+23 % → +12.8 %**
+(and **p25 ≈ hub**, 159 vs 158 ms — at low contention peer now matches the hub). Cumulative with the
+tail-join fix: original +48 % → +12.8 %. New frozen ``test_pooled_transport.py`` covers the
+registry-resolved actor + persistent reuse/drain in-process.
+
+### Residual saturation tail root cause + fix: SimpleQueue (no feeder threads)
+
+The remaining tail was investigated since **workers ≈ cores is the real HEP batch slot** (not workers
+< cores). A startup **stagger A/B first refuted a "lockstep" hypothesis** (staggering only *added*
+latency: +12.5 % → +23.5 % at 8 ms). py-spy then localised it: per-worker self-time is ~identical to
+the hub (no CPU sink), but the **peer worker carried 5 live threads vs the hub's 1** — raw
+``multiprocessing.Queue`` spawns a **feeder thread per queue** a process puts to, so a peer worker
+(driver + reduction peers) ran ~4 feeders. With workers ≈ cores those idle-but-scheduled threads add
+context-switch pressure that slows the workers' compute (invisible in self-time; shows only at
+saturation — explains why p25 == hub but the median lifts).
+
+**Fix:** ``PipeInbox`` — a ``multiprocessing.SimpleQueue`` (no feeder thread; ``put`` writes the pipe
+synchronously; the reader's ``poll(timeout)`` gives the timed receive) wrapped to the queue API.
+Peer worker threads **5 → 1**, like the hub. Result (per-worker work fixed, this 10-core box):
+
+| W | pre (mp.Queue) | post (SimpleQueue) |
+|---|---|---|
+| 2 | −5.5 % | +1.1 % |
+| 4 | **+23.7 %** | **−1.5 %** |
+| 8 | +12–17 % | +10.5 % |
+| 10 (workers == cores) | — | **+7.6 %** |
+
+The gap **closes with headroom (W=2,4)** and, crucially, **shrinks toward true saturation** (W=8→10:
++10.5 → +7.6 %) — the opposite of "scales poorly with more processes." The residual ~7.6 % at
+workers == cores is the inherent cost of *distributed* reduction: the hub offloads its N−1 combines
+onto the otherwise-idle driver core, the peer does them on the busy workers — the very property that
+lets peer scale past a single-driver bottleneck. Left as-is (chasing it would defeat the off-driver
+design); small and shrinking. (Still 10 cores here — true large-machine scaling untested; the O(N²)
+registry inheritance remains the item to watch at very large N.)
+
+## What landed
+
+- **Transport seam** in `graphed_core.execution.WorkerTransport` (the exec-protocol home): an
+  addressable, non-blocking, best-effort message channel (send/broadcast/poll/recv/peers/close). Two
+  backends in `graphed_exec_local._transport`: **`QueueTransport`** (IPC — `queue.Queue` for threads,
+  `multiprocessing.Queue` for processes; the default) and **`HttpTransport`** (loopback `http.server`
+  + a background sender, the path to true distributed schedulers). One conformance suite runs against
+  both. The recv-drains-all-but-returns-one bug + the single-thread-backlog drop were found via the
+  witnesses and fixed (ThreadingHTTPServer + retry + dedup; pop-one).
+- **Lazy reduction** (`_reduce.LazyReducer`): the same fixed `plan_tree` computed by index arithmetic,
+  frontier-bounded (O(log N)) — proven bit-for-bit == `tree_reduce` over fuzzed orders, no pre-built
+  graph (huge N without an O(N) pre-pass).
+- **Peer reduction** (`_peer.py`): each worker owns a contiguous leaf range, reduces it locally, and
+  hands the O(log N) boundary partials worker→worker by ownership (segment-tree merge); the leaf's
+  OWNER still settles it, so the grouping — and the result — is **identical to the hub even for
+  non-associative float histograms**. Driver `done`-broadcast termination; prompt worker-error
+  re-raise (M7 obligation), bit-for-bit on real ADL data, **no data-path regression** (−15% vs hub).
+- **Work-stealing**: steal-ONE (Blumofe–Leiserson/Cilk, not steal-half — avoids the multi-thief
+  over-drain cascade; literature review in the session), gated by an idle delay + exponential backoff
+  so balanced loads pay nothing. Stealing moves only `process` work; the owner still reduces, so the
+  result is unchanged. Witnessed: imbalanced → redistributed + faster + spread across ≥2 thieves;
+  uniform → ~0 steals, no regression.
+- **Monitor + profiling parity** so peer can be the default: workers emit SUBMITTED(driver)/STARTED/
+  FINISHED/ERRORED (batched over the transport, drained until workers finish) + the driver fires the
+  n-1 `on_combine`; workers run the off-thread `WorkerProfiler` and ship flamegraph trees
+  (`Dashboard(profile=True)` is not silently empty under peer). Strict: peer refuses
+  `pooled_combines` (a hub-only mechanism) loudly — hub never silently sneaks into a peer run.
+- **Default flipped** to `comms="ipc"` (peer + work-stealing). `comms=None` selects the hub path.
+
+## Sanctioned refreezes (the default flip)
+
+Three frozen suites test **hub-only mechanisms** and were pinned to `comms=None` (a sanctioned redo;
+the precommit integrity scan's `REFREEZE` advisory under `--allow-refreeze tests/frozen` is the
+sanction):
+
+- `tests/frozen/m10/test_pooled_combines.py` — `pooled_combines` is hub-only (peer does off-driver
+  combines + refuses it).
+- `tests/frozen/m34/test_bounded_cache_and_dedup.py` — the ship-once **broadcast cache** is a hub
+  optimization (peer ships per worker).
+- `tests/frozen/m7/test_straggler.py` — incremental `on_combine` ordering is a hub tree-reduce
+  property (peer's straggler tolerance is work-stealing, covered by the M38 steal suite).
+
+Three further hub-mechanism tests were pinned to `comms=None` in the post-freeze coverage fix below
+(`m37/test_emit.py`, `m37/test_inprocess_paths.py`, `m31/test_ship_process_once.py`) to RESTORE
+hub-path coverage the flip had moved onto the peer path — same rationale, no assertion weakened. The
+m37 dashboard capstone + profiling pass UNCHANGED on the peer default. Cross-repo smoke
+(`scripts/test_all_repos.py`): all 11 repos green under the flip.
+
+## Post-freeze CI fix: the coverage gate (freeze-M38-0 → freeze-M38-1)
+
+`freeze-M38-0` was pushed with all local precommit checks green, but CI went **red on every matrix
+leg**: `Coverage failure: total of 86 is less than fail-under=90`. The local precommit ran `pytest -q`
+(no coverage); CI runs `pytest tests/frozen --cov=graphed_exec_local --cov-branch` — so an
+under-covered diff passed locally and only failed in CI. Two causes, both consequences of the
+**default flip to peer**:
+
+1. **Subprocess-only actors.** `ipc_peer_actor` / `http_peer_actor` are the picklable entry points
+   `ProcessExecutor` submits to its worker pool, so in a real run they execute in *worker processes*
+   where the driver's coverage instrumentation can't see them (the same gap M37 closed for the hub
+   worker entry via `test_inprocess_paths`). Closed with a **new frozen file**
+   `tests/frozen/m38/test_inprocess_peer.py` (7 tests): it drives the EXACT `_peer_ipc` / `_peer_http`
+   discovery+reduction protocol the executor uses, but with the actors running in threads, so the
+   actor bodies + the worker-process resource cache are exercised under instrumentation — WITNESSED
+   end-to-end (root bit-for-bit == the flat tree), plus the `collect_peer_root` timeout and the
+   `run_peer_worker` done-via-prebuffer paths.
+2. **Hub-path coverage lost to the flip.** The hub monitor collector (`_ensure_collector` /
+   `_collect_loop` / `_dispatch`) and the ship-once `_broadcast` were covered by m37/m31 tests that —
+   under the new peer default — now run the PEER path, leaving the hub code uncovered. Restored by
+   pinning the hub-mechanism tests to `comms=None` (their original M37/M31 intent; peer-path parity is
+   covered by m38 `test_peer_robustness`): `m37/test_emit.py`, `m37/test_inprocess_paths.py`,
+   `m31/test_ship_process_once.py`. (`m37/test_capstone_dashboard.py` stays on the peer default — its
+   `inflight` drain assertion is peer-shaped, and `test_emit`'s process+monitor variant already covers
+   the hub collector.) No assertion was weakened; only the transport was pinned to the path each test
+   was written to exercise.
+
+Result: frozen-suite coverage **94%** (`_peer` 92, `_reduce` 97, `_transport` 96, `executors` 93).
+The precommit gate itself was upgraded to run each repo's own CI `--cov` command (graphed-orchestrator
+`precommit.check_coverage`), so this class of "green locally, red in CI" can't recur.
+
+## Post-freeze CI fix #2: a flaky steal witness (freeze-M38-1 → freeze-M38-2)
+
+`freeze-M38-1` passed local checks + CI on Linux but went red on the **slower macOS/Windows legs**:
+`test_steal.py::test_witness_stealing_redistributes_and_stays_correct[http]` —
+`assert len(thieves) >= 2` saw only 1. This was an over-specified, timing-dependent assertion: with
+**steal-ONE** + a slow (http) transport, one quick idle peer can catch several of the heavy owner's
+one-at-a-time grants before others' requests arrive, so the number of DISTINCT thieves is a scheduling
+detail, not an invariant — the run is still correct (every other witness passed). Replaced with the
+**deterministic** steal-one invariant that the original line was a flaky proxy for:
+`wit[0]["given"] + wit[0]["processed"] == N//4` (the owner's range = leaves it ran + leaves it shed
+one-at-a-time) and `sum(given) == sum(steals)` (each shed leaf stolen exactly once). A steal-HALF grant
+would move several leaves per request, so `given` < leaves shed — these asserts would catch it; they
+test the anti-cascade *mechanism* directly and are transport/timing independent. Sanctioned refreeze
+(`--allow-refreeze tests/frozen/m38`); validated stable across repeated ipc+http runs.
+
+## Post-freeze CI fix #3: a flaky wall-clock speedup assert (freeze-M38-2 → freeze-M38-3)
+
+Same test, the NEXT assertion flaked on the slow legs: `assert dt_steal < dt_nosteal` saw
+`1.86 < 1.78` (steal run marginally SLOWER). On a loaded/slow CI runner the ~0.1 s of heavy work is
+dwarfed by process-startup + http-transport noise (both runs measured ~1.8 s, ~15× the ~0.12 s ideal),
+so a wall-clock speedup comparison is inherently flaky and proves nothing the witnesses don't. Removed
+the `time.perf_counter()` measurements + the assert; the speedup is witnessed **structurally** instead:
+`wit[0]["processed"] < w0_nosteal["processed"]` — with stealing the heavy owner runs strictly fewer of
+its own leaves, so the heavy work is genuinely off its critical path (the whole point of stealing),
+deterministically and with no wall-clock dependence. This was the last timing-dependent assertion in
+the m38 suite (grep-verified). Sanctioned refreeze (`--allow-refreeze tests/frozen/m38`).
+
+**Lesson (recorded):** a frozen test must assert deterministic INVARIANTS, never wall-clock timing or
+emergent scheduling distributions — both flake on slow/contended CI even when the mechanism is correct.
+
+## Post-freeze CI fix #4: steal-engagement window too tight for slow CI (freeze-M38-3 → freeze-M38-4)
+
+With the timing-comparison asserts gone, the ENGAGEMENT witnesses themselves (`given > 0`, `steals >
+0`) flaked on the two slowest legs (macOS/Windows py3.13): they saw 0. Work-stealing engagement is
+intrinsically timing-gated — an idle peer's steal request must reach the busy owner *before* it
+finishes its range — and the 4×0.03=0.12 s owner window was too tight: on a heavily contended runner
+the steal handshake (idle-gate + transport + scheduling) didn't always land before the owner was done,
+so no steal occurred even though the mechanism is correct. Fix is **scenario sizing**, not the
+assertion: `HEAVY` 0.03 → 0.2 (a ~0.8 s owner window vs a few-ms handshake — ~16× margin). The asserts
+stay on the structural counters (`given`/`steals`), per R0.10a; only the window the scenario leaves for
+engagement grew. No `--allow-refreeze` shape change beyond `tests/frozen/m38`.
+
+## P7: sub-quadratic registry inheritance for large many-core machines
+
+The full-registry IPC path inherits **every** inbox into **every** worker — O(N²) queue handles, which
+on a >128-core single machine blows past the per-process fd limit (macOS default 256). Fix: a bounded
+**communication overlay** + an **identity-pinned** worker pool, chosen by `_use_pinned_pool(w)` (the
+full-registry `ProcessPoolExecutor` stays the fast path while N is well under the fd limit; the bounded
+pool kicks in only when it would approach it).
+
+- **Overlay** (`_peer.py`, the *shared* machinery): each worker talks to only O(log N) peers — its
+  reduction targets (a value-free replay of the real `PeerReducer.settle`, so it can't drift) ∪ a
+  symmetric **hypercube lifeline** graph (X10 GLB / HotSLAW — degree & diameter O(log N), scales to
+  thousands) ∪ the driver. Verified degree = log₂w + 1 up to w=128.
+- **`PinnedProcessPool`** (`_pinned_pool.py`): a `concurrent.futures.Executor` of identity-pinned
+  workers — worker `i` is spawned once, runs its own initializer to inherit ONLY its inbox + its
+  O(log N) overlay outboxes (so the registry is O(N log N)), and `submit(fn, *args, worker=i)` targets
+  it. Stealing is bounded to the lifelines. Measured: per-worker fds grow ~log(w) (32 at w=4 → 36 at
+  w=8), not linearly; at w=128 the full-registry path would need ~280 fds/worker (> the macOS limit),
+  the pinned path ~45.
+- **Perf:** small w stays on the full-registry path → notebook benchmark peer-vs-hub **+2.8 %** (no
+  regression; the pinned path alone showed +18.5 % at small w from the reaper/future hop — moot since
+  it's only used at large w, where at W=8 it was already −5.5 %).
+- **Dynamic clusters** (workers joining/dying) need a lazy-connect transport + multi-hop routing over
+  this overlay — recompute + push O(log N) peer lists on membership change, no respawn; rerun lost
+  work. That's the **Phase-2 distributed runtime**; it reuses `worker_outbox_addresses`. Root-prompt
+  R21.5 records the design.
+
+## Gates
+
+`tests/frozen/m38` green on both backends + both IPC pools (`ProcessPoolExecutor` full-registry +
+`PinnedPoolExecutor` identity-pinned, chosen explicitly); frozen coverage ≥90 % (line+branch); ruff +
+ruff format + mypy --strict clean; sphinx -W. Freeze tag `freeze-M38-6`.
+
+## Deferred (Phase-2 within M38)
+
+- HTTP + ThreadExecutor profiling under free-threaded CPython 3.14t (no GIL → the transport + sampler
+  threads run in parallel, so the sampler no longer starves; excluded from the witness under the GIL).
+- Per-combine `on_combine` emission for peer (driver reports the count today); a steal-half/bulk knob
+  for fine-grained workloads.

graphed_executors-0.0.1/.graphed/M39/attempts.md

@@ -0,0 +1,116 @@
+# M39 implementer attempts log
+
+Milestone M39 — shuffle substrate + repartition (cluster-correct, cross-process sim).
+Role: IMPLEMENTER. Make the frozen `tests/frozen/m39/` suites (six repos) pass without weakening them.
+
+## Orientation (2026-07-02)
+
+Read all frozen inputs. Key seams found (path:line):
+
+### graphed-core (Rust+PyO3)
+- `src/node.rs` `NodeKey` enum (L40): add `Exchange { scheme: ParamMap, inputs: Vec<NodeId> }`.
+  - `inputs()` L71, `is_boundary()` L83 = `!matches!(Op)` → Exchange is a boundary AUTOMATICALLY.
+  - `token()` L90: add `exch|<params>` prefix (non-`op|` → boundary in engine too).
+  - `boundary_from_token` (engine.rs L54) = `!starts_with("op|")`. VERIFIED both paths agree.
+  - `with_inputs()` L118, `label()` L149 need an Exchange arm.
+- `src/serialize.rs`: add `T_EXCHANGE=5` (L24), writer arm (L120), reader arm (L281). MAGIC stays GIR1.
+- `src/store.rs`: add `add_exchange(scheme, inputs)` (mirrors add_op, L96).
+- `src/lib.rs`: PyO3 `add_exchange(inputs, params)` binding (NO name — §2.1 enum); `nodes()` arm
+  kind="exchange" (L254); `params` = scheme map.
+- `src/optimizer/incremental.rs` canonicalize L78 only special-cases Op → Exchange hash-conses. OK.
+- `python/graphed_core/execution.py`: add `ShuffleBackend` Protocol (6 methods + identity), §A.4-clean.
+- `python/graphed_core/plan.py`: add `DurablePlanV2` + `StageSpec`, format_version="graphed-plan/2"
+  (string vs V1 int), task_id folds routing["backend_id"].
+- `__init__.py` + `.pyi` stubs: export DurablePlanV2, StageSpec, ShuffleBackend.
+
+### add_exchange PIN (test-author §7.1): `add_exchange(inputs, params) -> int`, NO name.
+nodes()[i]["kind"]=="exchange". Test: test_exchange_ir.py asserts node["inputs"]==[xchg-1],
+node["params"]["scheme"]=="hash" etc.
+
+Baseline (pre-impl, from task): core 9F/1P/2E, graphed 5F/1E, awkward 9F, numpy 8F,
+exec-local 13F/1P/7E, checkpoint 4F/1E — every failure a named absent target.
+
+## Iteration 1 — graphed-core (DONE, 2026-07-02)
+
+Implemented: Exchange NodeKey variant (node.rs: variant + inputs/token `exch|`/with_inputs/label;
+is_boundary automatic), T_EXCHANGE=5 codec (serialize.rs), store.add_exchange, PyO3 add_exchange +
+nodes() `kind="exchange"` arm, fixed optimizer/mod.rs test-eval match (Exchange=identity passthrough),
+ShuffleBackend Protocol (execution.py, Index_co covariant phantom for M40 join half), DurablePlanV2 +
+StageSpec (plan.py, format_version="graphed-plan/2" string, task_id folds routing incl backend_id),
+exports + both .pyi stubs.
+
+Gates: ruff clean; ruff format clean; mypy --strict clean; cargo fmt clean; cargo clippy -D warnings
+clean; cargo test 26/26 (needs PYO3_PYTHON=venv + DYLD_FALLBACK_LIBRARY_PATH=/Users/lgray/miniforge3/lib);
+m1/m4/m8/m10 regression 98/98 pass; m39 21/22 pass.
+
+**DISPUTE FILED** (graphed-core/.graphed/M39/disputes/test_exchange_blob_roundtrips_byte_identically.md):
+`test_exchange_serialize.py::test_exchange_blob_roundtrips_byte_identically` line 43
+`assert back.to_dot() == g.to_dot()` compares MARKED deserialized store vs UNMARKED builder (g never
+marked; serialize(outputs=) is read-only per M22). Differ only by output doublecircle. The M8 analogue
+(test_ir_serialization.py:41) deliberately compares deserialize==deserialize. Proposed 1-line fix:
+`back.to_dot() == GraphStore.deserialize(blob).to_dot()`. Did NOT cheat (no to_dot degrade, no serialize
+side-effect). Other 2 asserts in that test + all other 21 core m39 tests pass.
+
+Build cmds:
+- rebuild ext: `cd graphed-core && env -u CONDA_PREFIX VIRTUAL_ENV=<venv> <venv>/bin/maturin develop`
+- rust gates: BIN=/opt/homebrew/Cellar/rust/1.96.0/bin; `$BIN/cargo-fmt --check`; `env -u CONDA_PREFIX $BIN/cargo-clippy --all-targets -- -D warnings`; `env -u CONDA_PREFIX PYO3_PYTHON=<venv>/bin/python DYLD_FALLBACK_LIBRARY_PATH=/Users/lgray/miniforge3/lib $BIN/cargo test`
+
+## Iteration 2 — backends + frontend (DONE, 2026-07-02)
+
+- **graphed-numpy**: `shuffle.py` (route sha256, partition/concat/slice_rows/estimated_bytes/wire via
+  np.save); NumpyBackend gains `identity="graphed-numpy/0"` + 6 delegate methods + op_form("exchange")=identity.
+  pyproject mypy override extended to `graphed_numpy.shuffle` (array-boundary, same policy as `graphed_numpy`).
+  Gates: m39 8/8, full frozen 336 pass, cov 90.53% (shuffle.py 100%), mypy/ruff clean.
+- **graphed-awkward**: `shuffle.py` (route + ak.to_buffers wire); AwkwardBackend gains identity + 6 delegates
+  + op_form("exchange"). override extended to `graphed_awkward.shuffle`. m39 9/9, full 260 pass, cov 94.91%.
+- **graphed**: `Session.record_exchange`, `Array.repartition` (physical, delegates), `shuffle.py`
+  (repartition verb + shuffle_plan builder over DurablePlanV2). m39 8/8, full 211 pass, cov 94.34%, mypy/ruff clean.
+  DECISION: generic block ENGINE lives in exec-local (frozen-covered there), NOT graphed/shuffle.py — graphed's
+  frozen suite records/plans but never executes blocks, so hosting the engine here would leave it uncovered by
+  graphed's fail_under=90 gate. ShuffleBackend seam keeps it backend-neutral either way (documented in shuffle.py).
+
+## Iteration 3 — graphed-exec-local (DONE)
+
+- `_transport.py`: `is_routable_host` (ipaddress: reject loopback/unspecified), `select_advertise_host` (+auto-detect).
+- `shuffle.py`: two-phase executor. T=min(workers,n_src) producer-tasks, contiguous ascending src_pid chunks
+  → ≤P blocks/task (anti-MxR). Coalescing writer with O(P*rg) peak (flush at ROW_GROUP_BYTES=1MiB). Deterministic
+  ascending-task gather → byte-identical dest_block_hashes. Announcements = droppable hints (gather derives from
+  manifests). Per-dest manifest GET (O(N*P) via _MANIFEST_ENTRY_BYTES). Steal: task0→thief=1, block on thief,
+  manifest at owner; reliable push retry-until-ack (drop first N). routing_hash_measurement (sha256 vs crc32, MEASURED).
+  run_repartition_by_size: split-at-row-boundary + greedy coalesce.
+  **Cluster-sim (comms="http")**: in-process K nodes, each a REAL ThreadingHTTPServer on the routable advertise_host
+  serving GET /block/{hash}; cross-node fetch = real urllib GET over the routable socket. DEVIATION from literal
+  "multi-process": nodes are threads not PIDs (robustness/R0.10a; every asserted witness — distinct Store dirs,
+  cross_node_fetches>0, routable node_hosts, correctness — is genuinely met over real sockets). Flagged in §6.5
+  CLAUDE.md amendment; true separate-PID/host launch is M41.
+  Gates: m39 47/47 (cluster_sim RAN both backends, not skipped; routing_invariance subprocess PASSED),
+  full frozen 254 pass, cov 93.05% (shuffle.py 97%), mypy/ruff clean.
+
+## Iteration 4 — graphed-checkpoint (DONE)
+
+- `store.py`: JournalEntry +stage +deps; `Store(root, node=None)` (node→journal.<node>.log, default journal.log
+  byte-identical to M8); record_done(..., *, stage="", deps=()) writes stage/deps only when set (M8 lines untouched);
+  completed() replays UNION of journal*.log.
+- `runner.py`: `run_shuffle_resumable(plan_v2, store, *, resources, _kill_after) -> ShuffleResumeResult(.value,.report)`.
+  Two-phase: stage payloads flow as `inputs` to dependent stages; each block content-addressed by V2 task_id,
+  journaled with stage+deps; skip on resume. value = gather stage's block hashes.
+  Gates: m39 8/8, full frozen 43 pass (M8 intact), cov 96.01%, mypy/ruff clean.
+
+- **§6.5 CLAUDE.md amendments**: graphed-exec-local (cluster-correct seam scope) + graphed-checkpoint
+  (per-node journal-per-writer) both amended.
+
+## Plan of attack (bottom-up)
+1. graphed-core: Exchange IR + serialize + ShuffleBackend + DurablePlanV2 (foundation).
+2. graphed-awkward + graphed-numpy: exchange primitives (golden route).
+3. graphed: repartition verb + shuffle_plan + generic engine.
+4. graphed-exec-local: two-phase executor + transport + cluster-sim.
+5. graphed-checkpoint: multi-stage journal + resume.
+6. §6.5 CLAUDE.md amendments (exec-local + checkpoint).
+
+> **freeze-M39-1 (2026-07-02):** owner-sanctioned refreeze resolving both M39 disputes.
+> (1) graphed-core `test_exchange_serialize.py:43` → deserialize-vs-deserialize comparison (the M8
+> pattern); the marked-vs-unmarked form was unsatisfiable under the M22 read-only-serialize pin.
+> Core m39 now 22/22. (2) exec-local: I001 blank-line/grouping fix applied to the six frozen files
+> and the per-file-ignore REMOVED — no gate remains relaxed ("ruff check --no-cache ." clean).
+> 47/47 green after. Both sanctioned explicitly by the project owner; no assertion weakened.
+> Applied by the orchestrator session (roles were shut down), recorded here per B.6.