omnirun 0.2.1__tar.gz → 0.2.3__tar.gz

{omnirun-0.2.1 → omnirun-0.2.3}/.github/workflows/publish.yml

@@ -22,6 +22,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v7
 
+      - name: Check tag matches package version
+        run: ./scripts/check-version.sh "${GITHUB_REF_NAME}"
+
       - name: Install uv
         uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
         with:

{omnirun-0.2.1 → omnirun-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omnirun
-Version: 0.2.1
+Version: 0.2.3
 Summary: Run jobs from your repo anywhere: Slurm over SSH, any SSH box, Kaggle, Colab, or marketplace GPUs
 Author-email: Dmitrii Mukhutdinov <flyingleafe@gmail.com>
 License: MIT
@@ -140,7 +140,9 @@ section name is yours (you can have several of the same type). All backends
 accept `enabled = false` to keep the config without probing it. Every backend
 also accepts `project_root` (point it at an existing checkout to reuse its
 `.git`/`.venv`) and `max_parallel` (queue-daemon cap on concurrent jobs, default
-`1`).
+`1`). `project_root` may be a single path (applied to every repo) or a table
+mapping repo slug -> path, with an optional `default` key as fallback, so one
+backend can serve several repos (each getting the right checkout).
 
 ```toml
 [policy]
@@ -159,6 +161,7 @@ poll_interval_s = 10.0       # scheduler tick: refresh running + place pending
 type = "local"
 # root = "$HOME/.omnirun"    # OMNIRUN_ROOT: where jobs live
 # project_root = "$HOME/proj"  # reuse an existing checkout's .git and .venv
+# project_root = { my-repo = "$HOME/proj" }  # or per-repo (by slug), else default path
 # max_parallel = 2           # queue daemon: run up to 2 jobs here at once
 
 # ---- any machine you can ssh into ----

{omnirun-0.2.1 → omnirun-0.2.3}/README.md

@@ -120,7 +120,9 @@ section name is yours (you can have several of the same type). All backends
 accept `enabled = false` to keep the config without probing it. Every backend
 also accepts `project_root` (point it at an existing checkout to reuse its
 `.git`/`.venv`) and `max_parallel` (queue-daemon cap on concurrent jobs, default
-`1`).
+`1`). `project_root` may be a single path (applied to every repo) or a table
+mapping repo slug -> path, with an optional `default` key as fallback, so one
+backend can serve several repos (each getting the right checkout).
 
 ```toml
 [policy]
@@ -139,6 +141,7 @@ poll_interval_s = 10.0       # scheduler tick: refresh running + place pending
 type = "local"
 # root = "$HOME/.omnirun"    # OMNIRUN_ROOT: where jobs live
 # project_root = "$HOME/proj"  # reuse an existing checkout's .git and .venv
+# project_root = { my-repo = "$HOME/proj" }  # or per-repo (by slug), else default path
 # max_parallel = 2           # queue daemon: run up to 2 jobs here at once
 
 # ---- any machine you can ssh into ----

{omnirun-0.2.1 → omnirun-0.2.3}/flake.nix

@@ -71,6 +71,9 @@
 
           shellHook = ''
             ${pre-commit-check.shellHook}
+            # Guard against tagging a release without bumping the version.
+            # pre-commit owns only the pre-commit hook, so pre-push is ours.
+            ln -sf "$PWD/scripts/pre-push" "$(git rev-parse --git-path hooks)/pre-push"
             export LD_LIBRARY_PATH=${pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib pkgs.zlib ]}:$LD_LIBRARY_PATH
             export UV_PYTHON=${python}/bin/python
             uv sync --quiet

{omnirun-0.2.1 → omnirun-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "omnirun"
-version = "0.2.1"
+version = "0.2.3"
 description = "Run jobs from your repo anywhere: Slurm over SSH, any SSH box, Kaggle, Colab, or marketplace GPUs"
 readme = "README.md"
 requires-python = ">=3.12"

omnirun-0.2.3/scripts/check-version.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Verify the package version is internally consistent, and optionally that it
+# matches an expected version (e.g. a release tag).
+#
+#   scripts/check-version.sh            # pyproject.toml == src/omnirun/__init__.py
+#   scripts/check-version.sh v0.2.2     # ...and both equal 0.2.2
+#
+# Used by both the publish workflow and the pre-push hook so the rule lives in
+# one place.
+set -euo pipefail
+
+root="$(cd "$(dirname "$0")/.." && pwd)"
+
+extract() { grep -m1 -E "$2" "$root/$1" | sed -E 's/.*"([^"]+)".*/\1/'; }
+
+pyproject="$(extract pyproject.toml '^version *= *"')"
+init="$(extract src/omnirun/__init__.py '^__version__ *= *"')"
+
+if [ "$pyproject" != "$init" ]; then
+  echo "version mismatch: pyproject.toml=$pyproject src/omnirun/__init__.py=$init" >&2
+  exit 1
+fi
+
+if [ "$#" -ge 1 ]; then
+  expected="${1#v}"
+  if [ "$pyproject" != "$expected" ]; then
+    echo "version mismatch: expected $expected but package version is $pyproject" >&2
+    exit 1
+  fi
+fi
+
+echo "version OK: $pyproject"

omnirun-0.2.3/scripts/pre-push

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Refuse to push a release tag whose commit does not declare a matching version.
+#
+# Installed as .git/hooks/pre-push by the devShell (see flake.nix). It inspects
+# the *pushed commit's* content (via `git show <sha>:path`), so it catches a
+# stale worktree too. CI (publish.yml) enforces the same rule server-side.
+set -euo pipefail
+
+version_at() { git show "$1:$2" | grep -m1 -E "$3" | sed -E 's/.*"([^"]+)".*/\1/'; }
+
+status=0
+while read -r _local_ref local_sha remote_ref _remote_sha; do
+  case "$remote_ref" in
+    refs/tags/v[0-9]*) ;;
+    *) continue ;;
+  esac
+  # Deleting a tag pushes the zero sha; nothing to check.
+  case "$local_sha" in
+    0000000000000000000000000000000000000000) continue ;;
+  esac
+
+  tag="${remote_ref#refs/tags/}"
+  pyproject="$(version_at "$local_sha" pyproject.toml '^version *= *"')"
+  init="$(version_at "$local_sha" src/omnirun/__init__.py '^__version__ *= *"')"
+
+  if [ "$tag" != "v$pyproject" ]; then
+    echo "pre-push: tag $tag != pyproject version $pyproject (commit ${local_sha:0:12})" >&2
+    status=1
+  fi
+  if [ "$pyproject" != "$init" ]; then
+    echo "pre-push: pyproject $pyproject != __init__ $init (commit ${local_sha:0:12})" >&2
+    status=1
+  fi
+done
+
+if [ "$status" -ne 0 ]; then
+  echo "pre-push: bump the version before tagging (see scripts/check-version.sh)" >&2
+fi
+exit "$status"

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/__init__.py

@@ -1,3 +1,3 @@
 """omnirun — run jobs from your repo anywhere."""
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/base.py

@@ -22,6 +22,14 @@ class BackendError(RuntimeError):
     """Raised for backend-level failures with a user-actionable message."""
 
 
+#: Optional hook submit() calls the instant it creates a *billable* resource,
+#: before the (possibly minutes-long) wait for it to become usable. It hands the
+#: client a partial JobHandle so a recovery stub can be persisted immediately —
+#: an interrupted submit then stays visible to `ps`/`gc` instead of orphaning a
+#: running instance with no local record.
+ProvisioningSink = Callable[[JobHandle], None]
+
+
 class Backend(ABC):
     """One configured execution target (a cluster, a machine, a provider account).
 
@@ -46,7 +54,16 @@ class Backend(ABC):
     def probe(self, res: ResourceSpec) -> list[Offer]: ...
 
     @abstractmethod
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle: ...
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
+        """Run the job and return a handle. If a billable resource is created
+        before the handle is ready, call on_provisioning with a partial handle
+        first so the client can persist a recovery stub (see ProvisioningSink)."""
+        ...
 
     @abstractmethod
     def status(self, handle: JobHandle) -> StatusReport: ...

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/colab.py

@@ -33,7 +33,7 @@ from pathlib import Path
 from typing import Any
 
 from omnirun.backends import jobdir, tarsafe
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import (
     HEARTBEAT_STALE_S,
     BootstrapParams,
@@ -329,7 +329,12 @@ class ColabBackend(Backend):
 
     # ---- submit -----------------------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         session = self._session(spec.job_id)
         job_dir = f"{COLAB_ROOT}/jobs/{spec.job_id}"
 
@@ -360,7 +365,9 @@ class ColabBackend(Backend):
                 BootstrapParams(
                     omnirun_root=COLAB_ROOT,
                     project_root=jobdir.project_root_of(
-                        COLAB_ROOT, spec.repo.slug, self.config.project_root
+                        COLAB_ROOT,
+                        spec.repo.slug,
+                        self.config.project_root_for(spec.repo.slug),
                     ),
                     code=code,
                 ),

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/kaggle.py

@@ -33,7 +33,7 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.backends import jobdir, tarsafe
 from omnirun.bootstrap import (
     BootstrapParams,
@@ -393,7 +393,12 @@ class KaggleBackend(Backend):
             "the backend's `max_source_bytes` if Kaggle accepts more"
         )
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         api = self._api()
         user = self._username(api)
         job_id = spec.job_id
@@ -435,7 +440,9 @@ class KaggleBackend(Backend):
                 BootstrapParams(
                     omnirun_root=KAGGLE_ROOT,
                     project_root=jobdir.project_root_of(
-                        KAGGLE_ROOT, spec.repo.slug, self.config.project_root
+                        KAGGLE_ROOT,
+                        spec.repo.slug,
+                        self.config.project_root_for(spec.repo.slug),
                     ),
                     code=code,
                 ),

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/local.py

@@ -13,7 +13,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, register
+from omnirun.backends.base import Backend, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import shell_quote
 from omnirun.execlayer.local import LocalExec
@@ -110,10 +110,18 @@ class LocalBackend(Backend):
 
     # --- lifecycle ---------------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         root = jobdir.remote_root(self.exec, self.config.root)
         project_root = jobdir.resolve_project_root(
-            self.exec, root, spec.repo.slug, self.config.project_root
+            self.exec,
+            root,
+            spec.repo.slug,
+            self.config.project_root_for(spec.repo.slug),
         )
         params = BootstrapParams(
             omnirun_root=root,

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/marketplace.py

@@ -7,6 +7,13 @@ the job (client-side git push + bootstrap.sh) -> detached launch (setsid+nohup,
 pidfile), exactly like the plain ssh backend.
 
 Termination policy (billable resources must not leak):
+- **Provisioning stub** (issue #7): the instant ``submit`` rents an instance —
+  before the minutes-long provisioning wait — it calls the ``on_provisioning``
+  hook with a partial handle carrying the instance id, so the client persists a
+  recovery record *first*. A submit killed mid-wait then stays visible to
+  ``ps``/``status`` (reported as PROVISIONING with a billing warning) and
+  reclaimable by ``omnirun gc --all``, instead of orphaning a running instance
+  with no local trace. The final handle overwrites the stub on success.
 - **Client-side auto-terminate** (``auto_terminate``, default true): the instance
   is destroyed in ``pull_outputs`` (after a successful pull), in ``cancel``, and
   in ``gc``. ``status`` never terminates — when the job is terminal it appends a
@@ -52,7 +59,7 @@ import httpx
 from pydantic import BaseModel, Field
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import Exec, shell_quote
 from omnirun.repo import local_root_of
@@ -270,9 +277,26 @@ class MarketplaceBackend(Backend, ABC):
             unfit_reasons=[reason],
         )
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         inst = self._create_instance(spec, offer)
         instance_id = inst.instance_id
+        # The instance is now billing but the handle isn't ready yet. Hand the
+        # client a stub carrying the instance id *before* the provisioning wait,
+        # so an interrupted submit (a kill mid-wait) stays visible to ps/gc
+        # instead of orphaning a running instance with no local record (#7).
+        if on_provisioning is not None:
+            on_provisioning(
+                JobHandle(
+                    backend=self.name,
+                    job_id=spec.job_id,
+                    data={"instance_id": instance_id, "provisioning": True},
+                )
+            )
         try:
             inst = self._wait_provisioned(instance_id)
             assert inst.ssh_target is not None
@@ -286,7 +310,7 @@ class MarketplaceBackend(Backend, ABC):
             self._wait_ssh(ex)
             root = jobdir.remote_root(ex, self.config.root)
             project_root = jobdir.resolve_project_root(
-                ex, root, spec.repo.slug, self.config.project_root
+                ex, root, spec.repo.slug, self.config.project_root_for(spec.repo.slug)
             )
             params = BootstrapParams(
                 omnirun_root=root,
@@ -379,6 +403,31 @@ class MarketplaceBackend(Backend, ABC):
 
     def status(self, handle: JobHandle) -> StatusReport:
         instance_id = handle.data["instance_id"]
+        if not handle.data.get("job_dir"):
+            # A provisioning stub: submit was interrupted before it staged the
+            # job, so only the instance id was persisted. Report whether the
+            # rented instance is still billing so the user knows to reclaim it.
+            try:
+                inst = self._get_instance(instance_id)
+            except BackendError as e:
+                return StatusReport(
+                    status=JobStatus.PROVISIONING,
+                    detail=f"submit was interrupted while provisioning instance "
+                    f"{instance_id}; could not reach the {self.name} API to check "
+                    f"it ({e}) — verify in the provider console, run `omnirun gc`",
+                )
+            if inst is None:
+                return StatusReport(
+                    status=JobStatus.LOST,
+                    detail=f"instance {instance_id} never finished provisioning and "
+                    "no longer exists",
+                )
+            return StatusReport(
+                status=JobStatus.PROVISIONING,
+                detail=f"submit was interrupted after renting instance {instance_id} "
+                f"(now {inst.status or 'unknown'}) — it is still billing; run "
+                f"`omnirun gc --all` (or cancel {handle.job_id}) to destroy it",
+            )
         inst: Instance | None = None
         api_error: str | None = None
         try:
@@ -436,16 +485,17 @@ class MarketplaceBackend(Backend, ABC):
         return paths
 
     def cancel(self, handle: JobHandle) -> None:
-        try:  # best-effort remote kill; instance dies right after anyway
-            ex = self._exec_from_handle(handle)
-            q = shell_quote(handle.data["job_dir"])
-            ex.run(
-                f"if [ -f {q}/pid ]; then p=$(cat {q}/pid); "
-                f'kill -TERM -- "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null; fi; true',
-                timeout=30,
-            )
-        except Exception:
-            pass
+        if handle.data.get("job_dir"):  # a provisioning stub has nothing to kill
+            try:  # best-effort remote kill; instance dies right after anyway
+                ex = self._exec_from_handle(handle)
+                q = shell_quote(handle.data["job_dir"])
+                ex.run(
+                    f"if [ -f {q}/pid ]; then p=$(cat {q}/pid); "
+                    f'kill -TERM -- "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null; fi; true',
+                    timeout=30,
+                )
+            except Exception:
+                pass
         instance_id = handle.data["instance_id"]
         if self._get_instance(instance_id) is not None:
             self._terminate(instance_id)

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/slurm.py

@@ -17,7 +17,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams, generate_bootstrap
 from omnirun.config import BackendConfig
 from omnirun.execlayer.base import Exec, ExecError, shell_quote
@@ -181,6 +181,8 @@ class SlurmBackend(Backend):
 
         reasons: list[str] = []
         notes: list[str] = []
+        if res.wants_gpu() and self.config.extra("has_gpus", True) is False:
+            reasons.append(f"backend {self.name} is CPU-only (has_gpus = false)")
         if res.wants_gpu() and res.gpu_type and self.config.gpu_map:
             if _gpu_map_lookup(self.config.gpu_map, res.gpu_type) is None:
                 reasons.append(
@@ -258,7 +260,7 @@ class SlurmBackend(Backend):
         root = self.config.root
         job_dir = jobdir.job_dir_of(root, spec.job_id)
         project_root = jobdir.project_root_of(
-            root, spec.repo.slug, self.config.project_root
+            root, spec.repo.slug, self.config.project_root_for(spec.repo.slug)
         )
         sbatch = render_sbatch(spec, self.config, job_dir, root)
         bootstrap = generate_bootstrap(
@@ -277,11 +279,16 @@ class SlurmBackend(Backend):
         )
         return f"{sbatch}{sep}{bootstrap}"
 
-    def submit(self, spec: JobSpec, offer: Offer | None = None) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer | None = None,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         ex = self.exec_
         root = jobdir.remote_root(ex, self.config.root)
         project_root = jobdir.resolve_project_root(
-            ex, root, spec.repo.slug, self.config.project_root
+            ex, root, spec.repo.slug, self.config.project_root_for(spec.repo.slug)
         )
         params = BootstrapParams(
             omnirun_root=root,

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/ssh.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from pathlib import Path
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import Exec, ExecError, shell_quote
 from omnirun.execlayer.ssh import RECONNECT_HINT, SSHExec
@@ -213,11 +213,16 @@ class SshBackend(Backend):
 
     # --- submit / lifecycle ---------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer | None = None) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer | None = None,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         ex = self.exec_
         root = jobdir.remote_root(ex, self.config.root)
         project_root = jobdir.resolve_project_root(
-            ex, root, spec.repo.slug, self.config.project_root
+            ex, root, spec.repo.slug, self.config.project_root_for(spec.repo.slug)
         )
         params = BootstrapParams(
             omnirun_root=root,

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/backends/vast.py

@@ -37,23 +37,25 @@ OFFER_LIMIT = 5
 MIN_RELIABILITY = 0.95
 
 # normalized name -> vast gpu_name values (their catalog naming)
+# The REST /bundles/ endpoint matches gpu_name with spaces ("A100 SXM4");
+# underscore forms match nothing (the vast CLI translates them, the API doesn't).
 VAST_GPU_NAMES: dict[str, list[str]] = {
     "H200": ["H200"],
-    "H100": ["H100_SXM", "H100_PCIE", "H100_NVL"],
-    "A100": ["A100_SXM4", "A100_PCIE", "A100X"],
-    "A100-80": ["A100_SXM4", "A100_PCIE"],
-    "A6000": ["RTX_A6000"],
+    "H100": ["H100 SXM", "H100 PCIE", "H100 NVL"],
+    "A100": ["A100 SXM4", "A100 PCIE", "A100X"],
+    "A100-80": ["A100 SXM4", "A100 PCIE"],
+    "A6000": ["RTX A6000"],
     "L40": ["L40", "L40S"],
     "L4": ["L4"],
-    "V100": ["Tesla_V100"],
-    "4090": ["RTX_4090"],
-    "3090": ["RTX_3090"],
-    "5090": ["RTX_5090"],
+    "V100": ["Tesla V100"],
+    "4090": ["RTX 4090"],
+    "3090": ["RTX 3090"],
+    "5090": ["RTX 5090"],
 }
 
 
 def vast_gpu_names(gpu_type: str) -> list[str]:
-    return VAST_GPU_NAMES.get(gpu_type, [gpu_type.replace(" ", "_").replace("-", "_")])
+    return VAST_GPU_NAMES.get(gpu_type, [gpu_type.replace("_", " ").replace("-", " ")])
 
 
 def normalize_vast_gpu(name: str, gpu_ram_mb: float | None = None) -> str:
@@ -84,6 +86,10 @@ class VastBackend(MarketplaceBackend):
         }
         if res.gpu_type is not None:
             filt["gpu_name"] = {"in": vast_gpu_names(res.gpu_type)}
+            if res.gpu_type == "A100-80":
+                # cheapest-first + OFFER_LIMIT would otherwise fill the page
+                # with 40GB A100s that normalization then discards
+                filt["gpu_ram"] = {"gte": 70_000}
         elif res.min_vram_gb is not None:
             # vast reports gpu_ram in MB (verify live)
             filt["gpu_ram"] = {"gte": res.min_vram_gb * 1024}

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/cli.py

@@ -27,6 +27,7 @@ from omnirun.config import (
 )
 from omnirun.models import (
     EnvSpec,
+    JobHandle,
     JobRecord,
     JobSpec,
     JobStatus,
@@ -451,15 +452,24 @@ def submit(
         _render_payload(backend_obj, spec, picked.offer)
         return
 
-    handle = backend_obj.submit(spec, picked.offer)
-    JobStore().save(
-        JobRecord(
-            spec=spec,
-            handle=handle,
-            offer=picked.offer,
-            submitted_at=datetime.now(timezone.utc),
+    store = JobStore()
+    picked_offer = picked.offer
+
+    def _persist(h: JobHandle) -> None:
+        # Called once with a provisioning stub (if the backend rents something
+        # before the handle is ready) and again with the final handle, so an
+        # interrupted submit still leaves a reclaimable record (#7).
+        store.save(
+            JobRecord(
+                spec=spec,
+                handle=h,
+                offer=picked_offer,
+                submitted_at=datetime.now(timezone.utc),
+            )
         )
-    )
+
+    handle = backend_obj.submit(spec, picked_offer, on_provisioning=_persist)
+    _persist(handle)
     console.print(f"[green]submitted[/green] {spec.job_id} -> {picked.offer.label}")
     console.print(f"follow logs with: omnirun logs -f {spec.job_id}")
 

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/config.py

@@ -48,8 +48,10 @@ class BackendConfig(BaseModel, extra="allow"):
     env_setup: list[str] = Field(default_factory=list)  # module loads, exports
     # Where a project's shared checkout + .venv live on the worker. Default
     # "$OMNIRUN_ROOT/projects/<slug>"; point at an existing clone to reuse it
-    # (its .git becomes the object store, its .venv the shared env).
-    project_root: str | None = None
+    # (its .git becomes the object store, its .venv the shared env). A str
+    # applies to every repo; a dict maps repo slug -> path (with an optional
+    # "default" key as fallback) so one backend can serve several repos.
+    project_root: str | dict[str, str] | None = None
 
     # static capability declaration (ssh backend; probe checks live via nvidia-smi)
     gpus: list[GpuDecl] = Field(default_factory=list)
@@ -74,6 +76,15 @@ class BackendConfig(BaseModel, extra="allow"):
     def extra(self, key: str, default: Any = None) -> Any:
         return (self.model_extra or {}).get(key, default)
 
+    def project_root_for(self, slug: str) -> str | None:
+        """Configured project root for a repo. A str applies to every repo; a
+        dict is keyed by slug, falling back to a "default" key, then to None
+        (the built-in "$OMNIRUN_ROOT/projects/<slug>" path)."""
+        pr = self.project_root
+        if isinstance(pr, dict):
+            return pr.get(slug) or pr.get("default")
+        return pr
+
 
 class DaemonConfig(BaseModel):
     host: str = "127.0.0.1"  # localhost socket; bind 0.0.0.0 + auth to go remote

{omnirun-0.2.1 → omnirun-0.2.3}/src/omnirun/daemon.py

@@ -26,6 +26,7 @@ from omnirun import chooser
 from omnirun.backends.base import Backend, make_backend
 from omnirun.config import Config
 from omnirun.models import (
+    JobHandle,
     JobRecord,
     JobSpec,
     JobStatus,
@@ -433,19 +434,25 @@ class Daemon:
         if be is None:
             self._fail_submit(qid, f"backend {backend_name!r} unavailable")
             return
+
+        def _persist(h: JobHandle) -> None:
+            # Persist a stub the instant a billable resource is created, then
+            # the final handle — an interrupted submit stays reclaimable (#7).
+            self._jobstore.save(
+                JobRecord(
+                    spec=spec,
+                    handle=h,
+                    offer=offer,
+                    submitted_at=datetime.now(timezone.utc),
+                )
+            )
+
         try:
-            handle = be.submit(spec, offer)
+            handle = be.submit(spec, offer, on_provisioning=_persist)
         except Exception as e:
             self._retry_or_fail(qid, str(e))
             return
-        self._jobstore.save(
-            JobRecord(
-                spec=spec,
-                handle=handle,
-                offer=offer,
-                submitted_at=datetime.now(timezone.utc),
-            )
-        )
+        _persist(handle)
         with self._lock:
             entry = self._store.get(qid)
             if entry is None: