graphed 0.0.1__tar.gz

graphed-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,223 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Python frozen suites on the full OS matrix. One `-e .[dev]` builds the maturin extension and
+  # installs every backend — no more git-sibling installs. Tests run per-subtree (see
+  # scripts/run-tests.sh) so each package's suite keeps its origin-repo isolation.
+  test:
+    name: test ${{ matrix.os }} py${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+        python: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install (builds the extension)
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+      - uses: astral-sh/setup-uv@v5
+      - name: Lint + types (Python hooks via prek)
+        # this job's toolchain has no clippy/rustfmt components, so run only the Python hooks
+        run: uvx prek@0.4.5 run --all-files --show-diff-on-failure ruff-check ruff-format mypy
+      - name: Frozen suites (per-subtree)
+        shell: bash
+        run: ./scripts/run-tests.sh
+
+  # Combined branch-coverage gate (>=90%). Accumulated with --cov-append across subtrees, gated
+  # once on the union. The Rust core is measured separately (coverage-rust).
+  coverage:
+    name: coverage (>=90% combined)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install (builds the extension)
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+      - name: Combined coverage
+        shell: bash
+        run: COV=1 ./scripts/run-tests.sh
+
+  # Pure-Rust IR: formatting, clippy (deny warnings), unit tests, and the loom model.
+  rust:
+    name: rust (clippy + cargo test + loom)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+      - name: Link libpython for the test binary
+        run: echo "LD_LIBRARY_PATH=$(python -c 'import sysconfig; print(sysconfig.get_config_var(\"LIBDIR\"))')" >> "$GITHUB_ENV"
+      - uses: astral-sh/setup-uv@v5
+      - name: Rust lint (cargo fmt + clippy via prek)
+        run: uvx prek@0.4.5 run --all-files cargo-fmt cargo-clippy
+      - run: cargo test
+      - name: loom model of the locking discipline
+        run: RUSTFLAGS="--cfg loom" cargo test --lib loom_model
+
+  coverage-rust:
+    name: rust coverage (cargo llvm-cov)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools-preview
+      - uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Link libpython for the test binary
+        run: echo "LD_LIBRARY_PATH=$(python -c 'import sysconfig; print(sysconfig.get_config_var(\"LIBDIR\"))')" >> "$GITHUB_ENV"
+      - name: cargo llvm-cov (fail under 90% lines, bindings excluded)
+        run: cargo llvm-cov --ignore-filename-regex 'lib\.rs' --fail-under-lines 90
+
+  # Free-threaded Python (plan A.5), REQUIRED for the core + frontend + numpy suites: these need
+  # only numpy (which ships free-threaded wheels), so the no-GIL thread-safety of the Rust core is
+  # a hard gate. The awkward/onnx-dependent suites run advisory below (their wheels lag on 3.14t).
+  test-freethreaded:
+    name: test py3.14t (free-threaded, core+frontend+numpy)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14t"
+      - uses: dtolnay/rust-toolchain@stable
+      - run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+          python -m pip install pytest hypothesis numpy
+          # separate invocations: a single pytest over multiple subtrees collides on duplicate
+          # test basenames (test_topologies.py, test_external.py) — same reason run-tests.sh exists.
+          pytest tests/frozen/core -p no:cacheprovider
+          pytest tests/frozen/frontend -p no:cacheprovider
+          pytest tests/frozen/numpy -p no:cacheprovider
+
+  test-freethreaded-backends:
+    name: test py3.14t (free-threaded, backends — advisory)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14t"
+      - uses: dtolnay/rust-toolchain@stable
+      - shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+          ./scripts/run-tests.sh
+
+  # The live dashboard's BROWSER render in headless Chromium (M37). Heavy (a real browser), so it
+  # runs once here; the matrix's dashboard tests importorskip Playwright and skip the smoke test.
+  dashboard-smoke:
+    name: dashboard browser smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,dashboard]"
+          python -m pip install playwright
+          python -m playwright install --with-deps chromium
+          pytest tests/frozen/debug/m37/test_dashboard_browser.py -q -p no:cacheprovider
+
+  docs:
+    name: docs (sphinx -W)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Graphviz (inheritance diagrams)
+        run: sudo apt-get update && sudo apt-get install -y graphviz
+      - run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[docs,dev]"
+          sphinx-build -W -b html docs docs/_build/html
+
+  # Reproduce a preservation bundle through a REAL NVIDIA Triton server (gRPC + HTTP) with every ML
+  # framework present — the fully-realistic External-plugin cell. Serves tests/samples/triton_models.
+  triton:
+    name: real triton server (grpc + http)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Free disk space (GPU-less runner; drop unused SDKs before the Triton image pull)
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost || true
+          df -h /
+      - name: Start Triton server with the example model
+        run: |
+          docker run -d --rm --name triton -p 8000:8000 -p 8001:8001 -p 8002:8002 \
+            -v ${{ github.workspace }}/tests/samples/triton_models:/models \
+            nvcr.io/nvidia/tritonserver:25.11-pyt-python-py3 \
+            tritonserver --model-repository=/models
+      - name: Install this package + CPU-only ML frameworks + triton client (grpc + http)
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+          # The runner has no GPU: the default torch/tensorflow wheels drag the multi-GB CUDA/NVIDIA
+          # stack and exhaust the disk. Install CPU-only builds (torch from the CPU index,
+          # tensorflow-cpu; jax and xgboost are CPU by default). tritonclient is just the client libs.
+          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+          python -m pip install tensorflow-cpu jax xgboost "tritonclient[grpc,http]"
+      - name: Wait for Triton to be ready
+        run: |
+          for i in $(seq 1 90); do
+            if [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8000/v2/health/ready)" = "200" ]; then
+              echo "triton ready after ${i} tries"; exit 0
+            fi
+            sleep 2
+          done
+          echo "triton did not become ready"; docker logs triton; exit 1
+      - name: Full preserve suite against real Triton (gRPC + HTTP); nothing in m26 may skip
+        env:
+          GRAPHED_TRITON_GRPC: localhost:8001
+          GRAPHED_TRITON_HTTP: localhost:8000
+          TRITON_SERVER_URL: localhost:8000
+        run: |
+          pytest tests/frozen/preserve -p no:cacheprovider
+          pytest tests/frozen/preserve/m26 -q -rs | tee m26run.txt
+          ! grep -i "skipped" m26run.txt || (echo "m26 skipped tests on the everything-present cell" && exit 1)
+      - name: Triton logs (on failure)
+        if: failure()
+        run: docker logs triton || true

graphed-0.0.1/.github/workflows/release.yml

@@ -0,0 +1,78 @@
+name: release
+
+# Publish to PyPI on a version tag (vX.Y.Z), using PyPI Trusted Publishing (OIDC — no API token).
+#
+# BEFORE this can publish you must, once, configure a Trusted Publisher on PyPI for the `graphed`
+# project (https://pypi.org/manage/project/graphed/settings/publishing/):
+#   owner=graphed-org  repo=graphed  workflow=release.yml  environment=pypi
+# The project name `graphed` is currently free on PyPI. Until the trusted publisher is configured,
+# this workflow builds and verifies the artifacts but the publish step will no-op/fail — that is
+# intentional (nothing is uploaded without the maintainer opting in).
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: build wheels + sdist (maturin)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+        python: ["3.11", "3.12", "3.13", "3.14", "3.14t"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Add macOS universal2 targets
+        if: runner.os == 'macOS'
+        run: rustup target add x86_64-apple-darwin aarch64-apple-darwin
+      - shell: bash
+        run: |
+          python -m pip install --upgrade pip maturin
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            maturin build --release --out dist --target universal2-apple-darwin -i python
+          else
+            maturin build --release --out dist -i python
+          fi
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os }}-py${{ matrix.python }}
+          path: dist/*
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - run: |
+          python -m pip install --upgrade pip maturin
+          maturin sdist --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: publish to PyPI (trusted publishing)
+    needs: [build, sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # OIDC token for trusted publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist

graphed-0.0.1/.github/workflows/wheels.yml

@@ -0,0 +1,70 @@
+name: wheels
+
+# Build-only release artifacts (plan A.5 / M0): per-interpreter wheels via maturin on every
+# OS x arch target, including a DEDICATED free-threaded 3.14t wheel, plus the sdist. macOS builds
+# universal2 (x86_64 + arm64 in one wheel). Each wheel carries the compiled core AND all pure-Python
+# subpackages (one distribution). No PyPI upload here — see release.yml.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: wheels-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: wheel ${{ matrix.os }} py${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+        python: ["3.11", "3.12", "3.13", "3.14", "3.14t"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Add macOS universal2 targets
+        if: runner.os == 'macOS'
+        run: rustup target add x86_64-apple-darwin aarch64-apple-darwin
+      - name: Build wheel (universal2 on macOS)
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip maturin
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            maturin build --release --out dist --target universal2-apple-darwin -i python
+          else
+            maturin build --release --out dist -i python
+          fi
+      - name: Smoke-test the wheel (base install stays light)
+        shell: bash
+        run: |
+          python -m pip install dist/*.whl
+          python -c "import graphed, graphed.core; print('graphed', graphed.__version__, '/ core', graphed.core.version())"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.os }}-py${{ matrix.python }}
+          path: dist/*.whl
+
+  sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+      - run: |
+          python -m pip install --upgrade pip maturin
+          maturin sdist --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz

graphed-0.0.1/.gitignore

@@ -0,0 +1,26 @@
+# Rust (Cargo.lock IS committed for reproducible wheel builds)
+/target/
+**/*.rs.bk
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+dist/
+build/
+_build/
+docs/generated/
+*.so
+
+# Tools
+.serena/
+.DS_Store

graphed-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+# Lint + type-check hooks for the Python surface, run by prek (https://github.com/j178/prek).
+#
+# These commands live here ONCE and are consumed by every gate: CI (`uvx prek run
+# --all-files`), the graphed-orchestrator pre-commit gate, and a developer's local
+# `prek install`. No more hand-rolled `ruff`/`mypy` shell chains duplicated per workflow.
+#
+# ruff and mypy are `system` hooks on purpose: they use the versions already installed in
+# the project's env (the `dev` extra, intentionally unpinned), so there is a single source
+# of truth for tool versions rather than a second set pinned here that could drift.
+#
+# cargo fmt + clippy are hooks too (below); the dedicated `rust` CI job invokes them through prek,
+# so the Rust lint commands live here once as well. cargo test / loom / llvm-cov stay as their own
+# steps — those are tests/coverage, not lint. The matrix `test` job runs only the Python hook ids
+# (its toolchain has no clippy/rustfmt components), and the `rust` job runs only the cargo hook ids.
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-check
+        name: ruff check
+        entry: ruff check --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: ruff-format
+        name: ruff format
+        entry: ruff format --check --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: mypy
+        name: mypy (strict)
+        entry: mypy
+        language: system
+        pass_filenames: false
+        always_run: true
+      - id: cargo-fmt
+        name: cargo fmt
+        entry: cargo fmt --check
+        language: system
+        types: [rust]
+        pass_filenames: false
+      - id: cargo-clippy
+        name: cargo clippy (deny warnings)
+        entry: cargo clippy --all-targets -- -D warnings
+        language: system
+        types: [rust]
+        pass_filenames: false

graphed-0.0.1/CLAUDE.md

@@ -0,0 +1,43 @@
+# CLAUDE.md — graphed (consolidated package)
+
+Defers to the root **`graphed-project/CLAUDE.md`**; the **project plan
+(`graphed-project-plan-gated.md`) always wins.** This repository consolidates the eight
+`graphed-*-mvp` prototype packages into one pip-installable distribution, `graphed`.
+
+## What this repo is
+
+One package, `graphed`, with the former repos as subpackages:
+
+| Import            | Was                  | Guardrail (§A.4)                                        |
+|-------------------|----------------------|--------------------------------------------------------|
+| `graphed`         | `graphed` frontend   | backend-agnostic; no numpy/awkward in core types       |
+| `graphed.core`    | `graphed-core`       | Rust+PyO3 IR/optimizer/plan; **MUST NOT import awkward**|
+| `graphed.awkward` | `graphed-awkward`    | awkward typetracer backend                             |
+| `graphed.numpy`   | `graphed-numpy`      | trivial numpy backend                                  |
+| `graphed.debug`   | `graphed-debug`      | source-mapped tracebacks, lowering, dashboard          |
+| `graphed.checkpoint` | `graphed-checkpoint` | content-addressed store / resume                    |
+| `graphed.preserve`| `graphed-preserve`   | preservation bundle; invents no formats                |
+
+This is a **packaging/presentation** consolidation — **no new functionality** over the prototypes.
+
+## Hard rules (unchanged from the plan)
+
+- `graphed.core` **MUST NOT import awkward**; the base `import graphed` / `import graphed.core` stays
+  free of numpy/awkward (verified: a base install pulls only the compiled core + `executing`).
+- Reproducibility: the serializable IR — not cloudpickle — is the canonical durable representation.
+  cloudpickle is only for genuinely opaque user callables (flagged `opaque=True`).
+- Standards, not invented formats: correctionlib JSON, ONNX, UHI, HS3, content-addressed payloads.
+- `tests/frozen/**` is read-only (freeze/integrity rules §A.7/§B.6). The one-time consolidation
+  rewrote frozen imports mechanically (`graphed_core` → `graphed.core`, …) without weakening any
+  assertion; that baseline is now the freeze. Do not route around a frozen test — file a dispute.
+
+## Build & test
+
+maturin mixed layout: Rust at `src/*.rs` + `Cargo.toml` (root), Python at `python/graphed/`,
+`module-name = "graphed.core.graphed_core"`. Tests run **per-subtree** (`./scripts/run-tests.sh`) —
+see `CONTRIBUTING.md` for why a single `pytest tests/` cannot collect this tree.
+
+## Separate repos (not consolidated)
+
+`graphed-histogram`, `graphed-exec-local` (executor), and `graphed-orchestrator` stay their own
+packages; the first two depend on this consolidated `graphed`.

graphed-0.0.1/CONTRIBUTING.md

@@ -0,0 +1,57 @@
+# Contributing to `graphed`
+
+## Layout
+
+```
+Cargo.toml, Cargo.lock, src/*.rs     the Rust core crate (graphed.core.graphed_core)
+python/graphed/                      the Python package
+  __init__.py, *.py                    frontend (Session, Array, Backend, provenance)
+  core/  awkward/  numpy/  debug/  checkpoint/  preserve/   the subpackages
+tests/frozen/<pkg>/mX/               the frozen acceptance suites, per package + milestone
+tests/extra/<pkg>/                   implementer-added (non-frozen) tests
+tests/_corpus/                       vendored M0.5 fixtures (graphed_corpus) + reference data
+docs/                                one Sphinx project (per-package design pages under docs/<pkg>/)
+```
+
+The build backend is **maturin** (mixed Rust/Python): the compiled extension is nested at
+`graphed.core.graphed_core` and re-exported by `graphed.core`. `python-source = "python"` ships the
+whole `graphed/` tree.
+
+## Setup
+
+```bash
+pip install -e ".[dev]"     # builds the Rust extension and installs the full toolchain
+```
+
+## Running tests
+
+Run **one subtree at a time** — a single `pytest tests/` will not collect (duplicate test basenames
+across packages) and would break frozen guards that assert process-global state:
+
+```bash
+./scripts/run-tests.sh          # every package's suite, isolated
+COV=1 ./scripts/run-tests.sh    # + combined branch coverage, gate >=90
+pytest tests/frozen/awkward     # a single package
+```
+
+## Gates (run before pushing)
+
+```bash
+uvx prek run --all-files                 # ruff check + ruff format --check + mypy + cargo fmt + clippy
+./scripts/run-tests.sh                   # Python suites
+cargo test                               # Rust unit tests (set DYLD_FALLBACK_LIBRARY_PATH / LD_LIBRARY_PATH to python's libdir)
+RUSTFLAGS="--cfg loom" cargo test --lib loom_model
+sphinx-build -W -b html docs docs/_build/html
+python -m graphed_orchestrator.precommit .   # the project integrity/gate wrapper (see note)
+```
+
+- `mypy --strict` runs on `python/` (`files = ["python"]`); ruff/format cover `python` and `tests`.
+- The Rust `cargo test` binary links libpython — export `DYLD_FALLBACK_LIBRARY_PATH` (macOS) or
+  `LD_LIBRARY_PATH` (Linux) to `python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))'`.
+- **Integrity / freeze rule:** `tests/frozen/**` is read-only. Do not edit, skip, xfail, weaken, or
+  stub what a frozen test verifies. If a frozen test seems wrong, file a dispute — do not route
+  around it.
+- **Coverage gate note:** because the suite runs per-subtree, coverage is accumulated with
+  `--cov-append` across subtrees and gated once (`coverage report --fail-under=90`). The
+  `graphed_orchestrator.precommit` coverage sub-check expects a single whole-repo `pytest --cov`
+  line; here `COV=1 ./scripts/run-tests.sh` is the enforcing command (CI runs it directly).