ray-dispatcher 0.1.2__tar.gz

ray_dispatcher-0.1.2/.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: uv
+    directories:
+      - /
+      - /examples/toy_batch
+      - /examples/toy_batch/experiment
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      python-dependencies:
+        patterns: ['*']
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      github-actions:
+        patterns: ['*']

ray_dispatcher-0.1.2/.github/scripts/release_version.py

@@ -0,0 +1,48 @@
+"""Select releases from the version change across a push to main."""
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+from packaging.version import Version
+
+from it.unimib.data.nanofaas.ray_dispatcher import __version__
+
+
+def should_publish(previous: str, current: str, package_version: str) -> bool:
+    if current != package_version:
+        raise ValueError("pyproject.toml version must match package __version__")
+    if Version(current) < Version(previous):
+        raise ValueError("Package version must not decrease")
+    return Version(current) > Version(previous)
+
+
+def main() -> None:
+    current = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+    previous = current
+    if os.environ["GITHUB_EVENT_NAME"] == "push":
+        before = os.environ["BEFORE_SHA"]
+        if not re.fullmatch(r"[0-9a-f]{40}", before) or before == "0" * 40:
+            raise ValueError("Expected the existing main branch's previous commit SHA")
+        # The ref is validated; git receives an argument list, not a shell command.
+        result = subprocess.run(  # noqa: S603
+            ["git", "show", f"{before}:pyproject.toml"],  # noqa: S607
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        previous = tomllib.loads(result.stdout)["project"]["version"]
+    publish = should_publish(previous, current, __version__)
+    with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output:
+        output.write(f"publish={str(publish).lower()}\nversion={Version(current)}\n")
+    print(f"Version {previous} -> {current}; publish={publish}")
+
+
+if __name__ == "__main__":
+    main()

ray_dispatcher-0.1.2/.github/workflows/ci.yml

@@ -0,0 +1,80 @@
+name: checks
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+  schedule:
+    - cron: '23 6 * * 1'
+permissions:
+  contents: read
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.10', '3.13']
+    steps:
+      - uses: actions/checkout@v7.0.1
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: astral-sh/setup-uv@v10.3.0
+        with:
+          version: '0.12.24'
+      - run: uv sync --locked
+      - run: uv run --locked ruff check .
+      - run: uv run --locked ruff format --check .
+      - run: uv run --locked mypy src
+      - run: uv run --locked deptry src
+      - run: uv run --locked pytest tests/unit -q --cov --cov-report=term-missing --cov-report=xml
+      # Use the synced interpreter: uv run triggers Ray runtime-env repackaging.
+      - run: .venv/bin/python -m pytest tests/integration -q
+      - run: uv audit --locked --python-version '${{ matrix.python }}' --python-platform linux
+      - run: uv audit --locked --project examples/toy_batch --python-version 3.13 --python-platform linux
+      - if: matrix.python == '3.13'
+        run: |
+          uv sync --locked --group e2e
+          uv run --locked --group e2e python -c 'from multipass_vm_sdk import MultipassClient, MultipassVM, find_ssh_public_key'
+          uv run --locked --group e2e pytest tests/unit/test_vm_cleanup.py -q
+          uv run --locked --group e2e pytest -o addopts= --collect-only tests/e2e -q
+          uv sync --locked --project examples/toy_batch
+          uv run --locked --project examples/toy_batch pytest examples/toy_batch/tests -q
+  distribution:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v7.0.1
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: '3.13'
+      - uses: astral-sh/setup-uv@v10.3.0
+        with:
+          version: '0.12.24'
+      - run: uv sync --locked
+      - run: uv build
+      - run: uv run --locked twine check --strict dist/*
+      - name: Install and import the wheel outside the checkout
+        run: |
+          uv venv /tmp/rd-wheel
+          uv pip install --python /tmp/rd-wheel/bin/python dist/*.whl
+          cd /tmp
+          /tmp/rd-wheel/bin/python - <<'PY'
+          import importlib
+          from importlib.metadata import version
+          from pathlib import Path
+          from it.unimib.data.nanofaas import ray_dispatcher as rd
+          assert rd.Dispatcher and rd.Job
+          assert version("ray-dispatcher") == rd.__version__
+          assert Path(rd.__file__).with_name("remote_runner.py").is_file()
+          for name in ("it", "it.unimib", "it.unimib.data", "it.unimib.data.nanofaas"):
+              assert importlib.import_module(name).__file__ is None
+          assert importlib.util.find_spec("ray_dispatcher") is None
+          PY
+      - uses: actions/upload-artifact@v7.0.2
+        with:
+          name: python-distributions
+          path: dist/*
+          if-no-files-found: error

ray_dispatcher-0.1.2/.github/workflows/release.yml

@@ -0,0 +1,52 @@
+name: PyPI release
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  version:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      publish: ${{ steps.version.outputs.publish }}
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: '3.13'
+      - uses: astral-sh/setup-uv@v10.3.0
+        with:
+          version: '0.12.24'
+      - run: uv sync --locked
+      - id: version
+        run: .venv/bin/python .github/scripts/release_version.py
+        env:
+          BEFORE_SHA: ${{ github.event.before }}
+  checks:
+    needs: version
+    if: needs.version.outputs.publish == 'true' || github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/ci.yml
+  publish:
+    needs: [version, checks]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'Nanofaas/ray-dispatcher' && needs.version.outputs.publish == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    concurrency:
+      group: pypi-${{ needs.version.outputs.version }}
+      cancel-in-progress: false
+    environment:
+      name: pypi
+      url: https://pypi.org/project/ray-dispatcher/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8.0.2
+        with:
+          name: python-distributions
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

ray_dispatcher-0.1.2/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+dist/
+.coverage
+results/
+.worktrees/
+
+coverage.xml
+htmlcov/

ray_dispatcher-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ray-dispatcher contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ray_dispatcher-0.1.2/PKG-INFO

@@ -0,0 +1,158 @@
+Metadata-Version: 2.5
+Name: ray-dispatcher
+Version: 0.1.2
+Summary: Generic Ray-scheduled dispatcher for subprocess jobs on SSH-accessible VMs.
+Project-URL: Repository, https://github.com/Nanofaas/ray-dispatcher
+Project-URL: Issues, https://github.com/Nanofaas/ray-dispatcher/issues
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: fabric<4,>=3.2
+Requires-Dist: paramiko<6,>=5
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: ray<3,>=2.56
+Provides-Extra: progress
+Requires-Dist: rich<15,>=13; extra == 'progress'
+Description-Content-Type: text/markdown
+
+# ray-dispatcher
+
+Generic, Ray-scheduled dispatcher for subprocess jobs on SSH-accessible VMs.
+See `docs/superpowers/specs/` for the design and `docs/superpowers/plans/` for
+the implementation plan.
+
+## Installation and imports
+
+Install with `pip install ray-dispatcher` or `uv add ray-dispatcher`.
+Requires Python >=3.10. For development, install from a checkout with
+`uv sync --locked`.
+
+```python
+from it.unimib.data.nanofaas.ray_dispatcher import Dispatcher, Inventory, Job, Project
+```
+
+The installable distribution remains `ray-dispatcher`. Imports have migrated
+from `ray_dispatcher` to `it.unimib.data.nanofaas.ray_dispatcher`; update existing
+consumers. There is no legacy import alias. Parent packages are implicit
+PEP 420 namespaces, so other distributions can share the Nanofaas namespace.
+
+Licensed under the [MIT License](LICENSE).
+
+## Publishing
+
+The [release workflow](.github/workflows/release.yml) publishes to PyPI when a
+push to `main` increases the version. Update both `version` in `pyproject.toml`
+and `__version__` in `src/it/unimib/data/nanofaas/ray_dispatcher/__init__.py`,
+then update both locks and commit them together with the version bump:
+
+```sh
+uv lock
+uv lock --project examples/toy_batch
+```
+
+Version decreases and mismatched versions fail the workflow.
+
+Before the first release, configure a GitHub **pending trusted publisher** at
+[PyPI account Publishing](https://pypi.org/manage/account/publishing/):
+
+| Setting | Value |
+| --- | --- |
+| PyPI project name | `ray-dispatcher` |
+| GitHub owner | `Nanofaas` |
+| Repository | `ray-dispatcher` |
+| Workflow filename | `release.yml` |
+| Environment | `pypi` |
+
+See the [official setup instructions](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/).
+The pending publisher creates the PyPI project on the first successful upload;
+it does not reserve the name. No API token or repository secret is needed.
+
+Run **PyPI release → Run workflow** in GitHub Actions to verify the complete
+release pipeline without uploading anything. Download the `python-distributions`
+artifact to inspect wheel and sdist. A version bump on `main` runs the same
+checks on Python 3.10 and 3.13, including integration tests, audits, metadata
+validation and an isolated wheel install, before uploading those exact artifacts.
+Only the upload job receives OIDC permission. Unchanged versions do not publish;
+re-running an already successful release will fail because PyPI files cannot
+be overwritten. Fix failures and re-run the failed jobs before making another
+version bump.
+
+## Development checks
+
+```sh
+uv sync --locked
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy src
+uv run deptry src
+uv audit --locked
+uv run pytest tests/unit -q --cov --cov-report=term-missing
+uv build
+uv run twine check --strict dist/*
+uv run pytest tests/unit -q
+.venv/bin/python -m pytest tests/integration -q
+.venv/bin/python -m pytest -q
+```
+
+CI runs lint, formatting, strict type checking and the complete local test suite
+on Python 3.10 and 3.13. Integration tests start a local Ray runtime; unit tests
+do not start Ray. Run integration tests with the interpreter created by
+`uv sync`: wrapping them in `uv run` activates Ray’s automatic runtime-environment
+repackaging, which is unnecessary for this exclusively local runtime.
+
+Real-VM tests require Python >=3.11, a running Multipass daemon and an SSH public
+key. They create disposable VMs and delete them after the session:
+
+```sh
+uv sync --locked --group e2e
+uv run --group e2e pytest -o addopts= tests/e2e -m e2e -q
+```
+
+The example and E2E fixtures use the PyPI distribution `multipass-vm-sdk==0.6.0`
+and import `multipass_vm_sdk`.
+
+## Lifecycle and API changes
+
+`running_hosts()` returns `{(batch_id, job_id): host}`. Update consumers from
+`hosts.get(job_id)` to `hosts.get((handle.batch_id, handle.job_id))` to distinguish
+jobs with the same name in separate batches.
+
+`teardown()` confirms worker completion and orphan cleanup before releasing
+session ownership. If cleanup cannot be confirmed within 30 seconds, it raises
+`DispatcherError`, keeps the session heartbeats and job metadata, and rejects new
+submissions. Restore connectivity and retry `teardown()`; do not discard the
+Dispatcher while it still protects unfinished work. A process crash ends those
+heartbeats, so ownership protection then lasts only until the session lock TTL.
+
+Child logs stream to remote files and are collected into local attempt logs.
+The dispatcher disables the runner's duplicate forwarding over SSH to avoid
+capturing unbounded child output in worker memory. The standalone runner still
+forwards output by default.
+
+
+## Quality gates
+
+Use uv 0.12.24, pinned in CI, and `uv sync --locked`. Ruff checks security rules
+(`S`) and cyclomatic complexity (`C901`) in addition to formatting and existing
+lint rules. Complexity is capped at the current maximum of 16; lower the cap
+as lifecycle functions are simplified. Security exceptions are scoped to
+intentional subprocess execution, test assertions and fixture credentials.
+
+Unit tests measure statement and branch coverage with a minimum of 80%
+(current baseline: 81%). Ray integration and real-VM tests run separately;
+the coverage number does not include remote processes. `deptry` checks direct,
+transitive, unused and development-only imports. The unused legacy `progress`
+extra is retained for installation compatibility and explicitly excluded from
+the unused-dependency check.
+
+`uv audit --locked` checks known vulnerabilities in the resolved dependencies.
+CI audits the root and example locks and runs weekly as well as on pushes and
+pull requests, so newly disclosed vulnerabilities are detected without a code
+change. Dependabot proposes weekly Python and GitHub Actions updates; updates
+require review and passing checks. Secret scanning and push protection are
+enabled on GitHub.
+
+CI builds wheel and sdist, validates their metadata with Twine and installs the
+wheel in a clean environment outside the checkout. The release workflow reuses
+these checks before publishing a version bump. See the
+[repository](https://github.com/Nanofaas/ray-dispatcher).

ray_dispatcher-0.1.2/README.md

@@ -0,0 +1,141 @@
+# ray-dispatcher
+
+Generic, Ray-scheduled dispatcher for subprocess jobs on SSH-accessible VMs.
+See `docs/superpowers/specs/` for the design and `docs/superpowers/plans/` for
+the implementation plan.
+
+## Installation and imports
+
+Install with `pip install ray-dispatcher` or `uv add ray-dispatcher`.
+Requires Python >=3.10. For development, install from a checkout with
+`uv sync --locked`.
+
+```python
+from it.unimib.data.nanofaas.ray_dispatcher import Dispatcher, Inventory, Job, Project
+```
+
+The installable distribution remains `ray-dispatcher`. Imports have migrated
+from `ray_dispatcher` to `it.unimib.data.nanofaas.ray_dispatcher`; update existing
+consumers. There is no legacy import alias. Parent packages are implicit
+PEP 420 namespaces, so other distributions can share the Nanofaas namespace.
+
+Licensed under the [MIT License](LICENSE).
+
+## Publishing
+
+The [release workflow](.github/workflows/release.yml) publishes to PyPI when a
+push to `main` increases the version. Update both `version` in `pyproject.toml`
+and `__version__` in `src/it/unimib/data/nanofaas/ray_dispatcher/__init__.py`,
+then update both locks and commit them together with the version bump:
+
+```sh
+uv lock
+uv lock --project examples/toy_batch
+```
+
+Version decreases and mismatched versions fail the workflow.
+
+Before the first release, configure a GitHub **pending trusted publisher** at
+[PyPI account Publishing](https://pypi.org/manage/account/publishing/):
+
+| Setting | Value |
+| --- | --- |
+| PyPI project name | `ray-dispatcher` |
+| GitHub owner | `Nanofaas` |
+| Repository | `ray-dispatcher` |
+| Workflow filename | `release.yml` |
+| Environment | `pypi` |
+
+See the [official setup instructions](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/).
+The pending publisher creates the PyPI project on the first successful upload;
+it does not reserve the name. No API token or repository secret is needed.
+
+Run **PyPI release → Run workflow** in GitHub Actions to verify the complete
+release pipeline without uploading anything. Download the `python-distributions`
+artifact to inspect wheel and sdist. A version bump on `main` runs the same
+checks on Python 3.10 and 3.13, including integration tests, audits, metadata
+validation and an isolated wheel install, before uploading those exact artifacts.
+Only the upload job receives OIDC permission. Unchanged versions do not publish;
+re-running an already successful release will fail because PyPI files cannot
+be overwritten. Fix failures and re-run the failed jobs before making another
+version bump.
+
+## Development checks
+
+```sh
+uv sync --locked
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy src
+uv run deptry src
+uv audit --locked
+uv run pytest tests/unit -q --cov --cov-report=term-missing
+uv build
+uv run twine check --strict dist/*
+uv run pytest tests/unit -q
+.venv/bin/python -m pytest tests/integration -q
+.venv/bin/python -m pytest -q
+```
+
+CI runs lint, formatting, strict type checking and the complete local test suite
+on Python 3.10 and 3.13. Integration tests start a local Ray runtime; unit tests
+do not start Ray. Run integration tests with the interpreter created by
+`uv sync`: wrapping them in `uv run` activates Ray’s automatic runtime-environment
+repackaging, which is unnecessary for this exclusively local runtime.
+
+Real-VM tests require Python >=3.11, a running Multipass daemon and an SSH public
+key. They create disposable VMs and delete them after the session:
+
+```sh
+uv sync --locked --group e2e
+uv run --group e2e pytest -o addopts= tests/e2e -m e2e -q
+```
+
+The example and E2E fixtures use the PyPI distribution `multipass-vm-sdk==0.6.0`
+and import `multipass_vm_sdk`.
+
+## Lifecycle and API changes
+
+`running_hosts()` returns `{(batch_id, job_id): host}`. Update consumers from
+`hosts.get(job_id)` to `hosts.get((handle.batch_id, handle.job_id))` to distinguish
+jobs with the same name in separate batches.
+
+`teardown()` confirms worker completion and orphan cleanup before releasing
+session ownership. If cleanup cannot be confirmed within 30 seconds, it raises
+`DispatcherError`, keeps the session heartbeats and job metadata, and rejects new
+submissions. Restore connectivity and retry `teardown()`; do not discard the
+Dispatcher while it still protects unfinished work. A process crash ends those
+heartbeats, so ownership protection then lasts only until the session lock TTL.
+
+Child logs stream to remote files and are collected into local attempt logs.
+The dispatcher disables the runner's duplicate forwarding over SSH to avoid
+capturing unbounded child output in worker memory. The standalone runner still
+forwards output by default.
+
+
+## Quality gates
+
+Use uv 0.12.24, pinned in CI, and `uv sync --locked`. Ruff checks security rules
+(`S`) and cyclomatic complexity (`C901`) in addition to formatting and existing
+lint rules. Complexity is capped at the current maximum of 16; lower the cap
+as lifecycle functions are simplified. Security exceptions are scoped to
+intentional subprocess execution, test assertions and fixture credentials.
+
+Unit tests measure statement and branch coverage with a minimum of 80%
+(current baseline: 81%). Ray integration and real-VM tests run separately;
+the coverage number does not include remote processes. `deptry` checks direct,
+transitive, unused and development-only imports. The unused legacy `progress`
+extra is retained for installation compatibility and explicitly excluded from
+the unused-dependency check.
+
+`uv audit --locked` checks known vulnerabilities in the resolved dependencies.
+CI audits the root and example locks and runs weekly as well as on pushes and
+pull requests, so newly disclosed vulnerabilities are detected without a code
+change. Dependabot proposes weekly Python and GitHub Actions updates; updates
+require review and passing checks. Secret scanning and push protection are
+enabled on GitHub.
+
+CI builds wheel and sdist, validates their metadata with Twine and installs the
+wheel in a clean environment outside the checkout. The release workflow reuses
+these checks before publishing a version bump. See the
+[repository](https://github.com/Nanofaas/ray-dispatcher).