omnirun 0.2.2__tar.gz → 0.2.3__tar.gz

{omnirun-0.2.2 → omnirun-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omnirun
-Version: 0.2.2
+Version: 0.2.3
 Summary: Run jobs from your repo anywhere: Slurm over SSH, any SSH box, Kaggle, Colab, or marketplace GPUs
 Author-email: Dmitrii Mukhutdinov <flyingleafe@gmail.com>
 License: MIT

{omnirun-0.2.2 → omnirun-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "omnirun"
-version = "0.2.2"
+version = "0.2.3"
 description = "Run jobs from your repo anywhere: Slurm over SSH, any SSH box, Kaggle, Colab, or marketplace GPUs"
 readme = "README.md"
 requires-python = ">=3.12"

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/__init__.py

@@ -1,3 +1,3 @@
 """omnirun — run jobs from your repo anywhere."""
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/base.py

@@ -22,6 +22,14 @@ class BackendError(RuntimeError):
     """Raised for backend-level failures with a user-actionable message."""
 
 
+#: Optional hook submit() calls the instant it creates a *billable* resource,
+#: before the (possibly minutes-long) wait for it to become usable. It hands the
+#: client a partial JobHandle so a recovery stub can be persisted immediately —
+#: an interrupted submit then stays visible to `ps`/`gc` instead of orphaning a
+#: running instance with no local record.
+ProvisioningSink = Callable[[JobHandle], None]
+
+
 class Backend(ABC):
     """One configured execution target (a cluster, a machine, a provider account).
 
@@ -46,7 +54,16 @@ class Backend(ABC):
     def probe(self, res: ResourceSpec) -> list[Offer]: ...
 
     @abstractmethod
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle: ...
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
+        """Run the job and return a handle. If a billable resource is created
+        before the handle is ready, call on_provisioning with a partial handle
+        first so the client can persist a recovery stub (see ProvisioningSink)."""
+        ...
 
     @abstractmethod
     def status(self, handle: JobHandle) -> StatusReport: ...

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/colab.py

@@ -33,7 +33,7 @@ from pathlib import Path
 from typing import Any
 
 from omnirun.backends import jobdir, tarsafe
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import (
     HEARTBEAT_STALE_S,
     BootstrapParams,
@@ -329,7 +329,12 @@ class ColabBackend(Backend):
 
     # ---- submit -----------------------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         session = self._session(spec.job_id)
         job_dir = f"{COLAB_ROOT}/jobs/{spec.job_id}"
 

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/kaggle.py

@@ -33,7 +33,7 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.backends import jobdir, tarsafe
 from omnirun.bootstrap import (
     BootstrapParams,
@@ -393,7 +393,12 @@ class KaggleBackend(Backend):
             "the backend's `max_source_bytes` if Kaggle accepts more"
         )
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         api = self._api()
         user = self._username(api)
         job_id = spec.job_id

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/local.py

@@ -13,7 +13,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, register
+from omnirun.backends.base import Backend, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import shell_quote
 from omnirun.execlayer.local import LocalExec
@@ -110,7 +110,12 @@ class LocalBackend(Backend):
 
     # --- lifecycle ---------------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         root = jobdir.remote_root(self.exec, self.config.root)
         project_root = jobdir.resolve_project_root(
             self.exec,

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/marketplace.py

@@ -7,6 +7,13 @@ the job (client-side git push + bootstrap.sh) -> detached launch (setsid+nohup,
 pidfile), exactly like the plain ssh backend.
 
 Termination policy (billable resources must not leak):
+- **Provisioning stub** (issue #7): the instant ``submit`` rents an instance —
+  before the minutes-long provisioning wait — it calls the ``on_provisioning``
+  hook with a partial handle carrying the instance id, so the client persists a
+  recovery record *first*. A submit killed mid-wait then stays visible to
+  ``ps``/``status`` (reported as PROVISIONING with a billing warning) and
+  reclaimable by ``omnirun gc --all``, instead of orphaning a running instance
+  with no local trace. The final handle overwrites the stub on success.
 - **Client-side auto-terminate** (``auto_terminate``, default true): the instance
   is destroyed in ``pull_outputs`` (after a successful pull), in ``cancel``, and
   in ``gc``. ``status`` never terminates — when the job is terminal it appends a
@@ -52,7 +59,7 @@ import httpx
 from pydantic import BaseModel, Field
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import Exec, shell_quote
 from omnirun.repo import local_root_of
@@ -270,9 +277,26 @@ class MarketplaceBackend(Backend, ABC):
             unfit_reasons=[reason],
         )
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         inst = self._create_instance(spec, offer)
         instance_id = inst.instance_id
+        # The instance is now billing but the handle isn't ready yet. Hand the
+        # client a stub carrying the instance id *before* the provisioning wait,
+        # so an interrupted submit (a kill mid-wait) stays visible to ps/gc
+        # instead of orphaning a running instance with no local record (#7).
+        if on_provisioning is not None:
+            on_provisioning(
+                JobHandle(
+                    backend=self.name,
+                    job_id=spec.job_id,
+                    data={"instance_id": instance_id, "provisioning": True},
+                )
+            )
         try:
             inst = self._wait_provisioned(instance_id)
             assert inst.ssh_target is not None
@@ -379,6 +403,31 @@ class MarketplaceBackend(Backend, ABC):
 
     def status(self, handle: JobHandle) -> StatusReport:
         instance_id = handle.data["instance_id"]
+        if not handle.data.get("job_dir"):
+            # A provisioning stub: submit was interrupted before it staged the
+            # job, so only the instance id was persisted. Report whether the
+            # rented instance is still billing so the user knows to reclaim it.
+            try:
+                inst = self._get_instance(instance_id)
+            except BackendError as e:
+                return StatusReport(
+                    status=JobStatus.PROVISIONING,
+                    detail=f"submit was interrupted while provisioning instance "
+                    f"{instance_id}; could not reach the {self.name} API to check "
+                    f"it ({e}) — verify in the provider console, run `omnirun gc`",
+                )
+            if inst is None:
+                return StatusReport(
+                    status=JobStatus.LOST,
+                    detail=f"instance {instance_id} never finished provisioning and "
+                    "no longer exists",
+                )
+            return StatusReport(
+                status=JobStatus.PROVISIONING,
+                detail=f"submit was interrupted after renting instance {instance_id} "
+                f"(now {inst.status or 'unknown'}) — it is still billing; run "
+                f"`omnirun gc --all` (or cancel {handle.job_id}) to destroy it",
+            )
         inst: Instance | None = None
         api_error: str | None = None
         try:
@@ -436,16 +485,17 @@ class MarketplaceBackend(Backend, ABC):
         return paths
 
     def cancel(self, handle: JobHandle) -> None:
-        try:  # best-effort remote kill; instance dies right after anyway
-            ex = self._exec_from_handle(handle)
-            q = shell_quote(handle.data["job_dir"])
-            ex.run(
-                f"if [ -f {q}/pid ]; then p=$(cat {q}/pid); "
-                f'kill -TERM -- "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null; fi; true',
-                timeout=30,
-            )
-        except Exception:
-            pass
+        if handle.data.get("job_dir"):  # a provisioning stub has nothing to kill
+            try:  # best-effort remote kill; instance dies right after anyway
+                ex = self._exec_from_handle(handle)
+                q = shell_quote(handle.data["job_dir"])
+                ex.run(
+                    f"if [ -f {q}/pid ]; then p=$(cat {q}/pid); "
+                    f'kill -TERM -- "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null; fi; true',
+                    timeout=30,
+                )
+            except Exception:
+                pass
         instance_id = handle.data["instance_id"]
         if self._get_instance(instance_id) is not None:
             self._terminate(instance_id)

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/slurm.py

@@ -17,7 +17,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams, generate_bootstrap
 from omnirun.config import BackendConfig
 from omnirun.execlayer.base import Exec, ExecError, shell_quote
@@ -279,7 +279,12 @@ class SlurmBackend(Backend):
         )
         return f"{sbatch}{sep}{bootstrap}"
 
-    def submit(self, spec: JobSpec, offer: Offer | None = None) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer | None = None,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         ex = self.exec_
         root = jobdir.remote_root(ex, self.config.root)
         project_root = jobdir.resolve_project_root(

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/backends/ssh.py

@@ -10,7 +10,7 @@ from collections.abc import Iterator
 from pathlib import Path
 
 from omnirun.backends import jobdir
-from omnirun.backends.base import Backend, BackendError, register
+from omnirun.backends.base import Backend, BackendError, ProvisioningSink, register
 from omnirun.bootstrap import BootstrapParams
 from omnirun.execlayer.base import Exec, ExecError, shell_quote
 from omnirun.execlayer.ssh import RECONNECT_HINT, SSHExec
@@ -213,7 +213,12 @@ class SshBackend(Backend):
 
     # --- submit / lifecycle ---------------------------------------------------
 
-    def submit(self, spec: JobSpec, offer: Offer | None = None) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer | None = None,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         ex = self.exec_
         root = jobdir.remote_root(ex, self.config.root)
         project_root = jobdir.resolve_project_root(

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/cli.py

@@ -27,6 +27,7 @@ from omnirun.config import (
 )
 from omnirun.models import (
     EnvSpec,
+    JobHandle,
     JobRecord,
     JobSpec,
     JobStatus,
@@ -451,15 +452,24 @@ def submit(
         _render_payload(backend_obj, spec, picked.offer)
         return
 
-    handle = backend_obj.submit(spec, picked.offer)
-    JobStore().save(
-        JobRecord(
-            spec=spec,
-            handle=handle,
-            offer=picked.offer,
-            submitted_at=datetime.now(timezone.utc),
+    store = JobStore()
+    picked_offer = picked.offer
+
+    def _persist(h: JobHandle) -> None:
+        # Called once with a provisioning stub (if the backend rents something
+        # before the handle is ready) and again with the final handle, so an
+        # interrupted submit still leaves a reclaimable record (#7).
+        store.save(
+            JobRecord(
+                spec=spec,
+                handle=h,
+                offer=picked_offer,
+                submitted_at=datetime.now(timezone.utc),
+            )
         )
-    )
+
+    handle = backend_obj.submit(spec, picked_offer, on_provisioning=_persist)
+    _persist(handle)
     console.print(f"[green]submitted[/green] {spec.job_id} -> {picked.offer.label}")
     console.print(f"follow logs with: omnirun logs -f {spec.job_id}")
 

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/daemon.py

@@ -26,6 +26,7 @@ from omnirun import chooser
 from omnirun.backends.base import Backend, make_backend
 from omnirun.config import Config
 from omnirun.models import (
+    JobHandle,
     JobRecord,
     JobSpec,
     JobStatus,
@@ -433,19 +434,25 @@ class Daemon:
         if be is None:
             self._fail_submit(qid, f"backend {backend_name!r} unavailable")
             return
+
+        def _persist(h: JobHandle) -> None:
+            # Persist a stub the instant a billable resource is created, then
+            # the final handle — an interrupted submit stays reclaimable (#7).
+            self._jobstore.save(
+                JobRecord(
+                    spec=spec,
+                    handle=h,
+                    offer=offer,
+                    submitted_at=datetime.now(timezone.utc),
+                )
+            )
+
         try:
-            handle = be.submit(spec, offer)
+            handle = be.submit(spec, offer, on_provisioning=_persist)
         except Exception as e:
             self._retry_or_fail(qid, str(e))
             return
-        self._jobstore.save(
-            JobRecord(
-                spec=spec,
-                handle=handle,
-                offer=offer,
-                submitted_at=datetime.now(timezone.utc),
-            )
-        )
+        _persist(handle)
         with self._lock:
             entry = self._store.get(qid)
             if entry is None:

{omnirun-0.2.2 → omnirun-0.2.3}/src/omnirun/repo.py

@@ -11,6 +11,7 @@ import os
 import re
 import shutil
 import subprocess
+import tempfile
 from pathlib import Path
 
 from omnirun.models import RepoRef
@@ -32,6 +33,60 @@ def _git(root: Path, *args: str) -> subprocess.CompletedProcess[str]:
     )
 
 
+# Identity stamped on wip commits so `git commit-tree` never fails on a repo
+# without a configured user.name/email; the commit is ephemeral anyway.
+_WIP_IDENTITY = {
+    "GIT_AUTHOR_NAME": "omnirun",
+    "GIT_AUTHOR_EMAIL": "wip@omnirun.invalid",
+    "GIT_COMMITTER_NAME": "omnirun",
+    "GIT_COMMITTER_EMAIL": "wip@omnirun.invalid",
+}
+
+
+def _wip_commit(root: Path, parent_sha: str) -> str:
+    """Create a dangling commit capturing the working tree so a `--dirty` submit
+    runs exactly what's on disk: tracked modifications, staged changes,
+    deletions, and untracked non-ignored files. The user's index, HEAD, and
+    working tree are left untouched — a scratch index is used — and gitignored
+    files (e.g. `.env`) stay out, matching a normal commit. Its parent is
+    `parent_sha` (HEAD); returns the new commit's sha."""
+    with tempfile.TemporaryDirectory() as td:
+        env = {**os.environ, **_WIP_IDENTITY, "GIT_INDEX_FILE": str(Path(td) / "index")}
+
+        def run(*args: str) -> subprocess.CompletedProcess[str]:
+            return subprocess.run(
+                ["git", "-C", str(root), *args],
+                capture_output=True,
+                text=True,
+                env=env,
+                timeout=600,
+            )
+
+        # Seed the scratch index from HEAD, then stage every working-tree change.
+        for step in (("read-tree", parent_sha), ("add", "-A")):
+            r = run(*step)
+            if r.returncode != 0:
+                raise RepoError(
+                    f"capturing --dirty working tree failed at `git {step[0]}`:\n"
+                    f"{r.stderr.strip()}"
+                )
+        tree = run("write-tree")
+        if tree.returncode != 0:
+            raise RepoError(
+                "capturing --dirty working tree failed at `git write-tree`:\n"
+                f"{tree.stderr.strip()}"
+            )
+        commit = run(
+            "commit-tree", tree.stdout.strip(), "-p", parent_sha, "-m", "omnirun wip"
+        )
+        if commit.returncode != 0:
+            raise RepoError(
+                "capturing --dirty working tree failed at `git commit-tree`:\n"
+                f"{commit.stderr.strip()}"
+            )
+        return commit.stdout.strip()
+
+
 def find_repo_root(start: Path | None = None) -> Path:
     """Toplevel of the git repo containing `start` (default: cwd)."""
     start = start or Path.cwd()
@@ -111,6 +166,13 @@ def capture_repo_state(
                     f"run `git push origin {branch}` first or pass --push"
                 )
 
+    # A --dirty submit must run the working tree, not plain HEAD: snapshot it
+    # into a wip commit and ship that sha (pushed to refs/omnirun/<sha12> for the
+    # SSH family, bundled for notebooks). `dirty` is only true here when
+    # allow_dirty was set, else we raised above.
+    if dirty:
+        sha = _wip_commit(root, sha)
+
     return RepoRef(
         remote_url=remote_url,
         sha=sha,

{omnirun-0.2.2 → omnirun-0.2.3}/tests/test_cli.py

@@ -11,7 +11,12 @@ from typing import ClassVar
 import pytest
 from typer.testing import CliRunner
 
-from omnirun.backends.base import Backend, BackendError, register  # noqa: E402
+from omnirun.backends.base import (  # noqa: E402
+    Backend,
+    BackendError,
+    ProvisioningSink,
+    register,
+)
 from omnirun.cli import app  # noqa: E402
 from omnirun.models import (  # noqa: E402
     JobHandle,
@@ -47,7 +52,12 @@ class StubBackend(Backend):
             )
         ]
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         type(self).submitted[spec.job_id] = (spec, offer)
         return JobHandle(
             backend=self.name, job_id=spec.job_id, data={"token": f"t-{spec.job_id}"}
@@ -96,7 +106,12 @@ class UnreachableBackend(Backend):
             )
         ]
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:  # pragma: no cover
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:  # pragma: no cover
         raise AssertionError("dry-run must never submit")
 
     def status(self, handle: JobHandle) -> StatusReport:  # pragma: no cover
@@ -116,6 +131,47 @@ class UnreachableBackend(Backend):
         return []
 
 
+@register("provfail")
+class ProvisionThenFailBackend(Backend):
+    """Rents a resource (emits a provisioning stub) then dies before returning a
+    handle — models an interrupted marketplace submit (issue #7)."""
+
+    def probe(self, res: ResourceSpec) -> list[Offer]:
+        return [Offer(backend=self.name, label=f"{self.name}: ok", fits=True)]
+
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
+        if on_provisioning is not None:
+            on_provisioning(
+                JobHandle(
+                    backend=self.name,
+                    job_id=spec.job_id,
+                    data={"instance_id": "inst-42", "provisioning": True},
+                )
+            )
+        raise BackendError("killed mid-provision")
+
+    def status(self, handle: JobHandle) -> StatusReport:  # pragma: no cover
+        raise BackendError("n/a")
+
+    def logs(
+        self, handle: JobHandle, follow: bool = False
+    ) -> Iterator[str]:  # pragma: no cover
+        yield ""
+
+    def cancel(self, handle: JobHandle) -> None:  # pragma: no cover
+        pass
+
+    def pull_outputs(
+        self, handle: JobHandle, dest: Path
+    ) -> list[Path]:  # pragma: no cover
+        return []
+
+
 BASE_CONFIG = """\
 [policy]
 auto_wait_threshold = "15m"
@@ -126,6 +182,9 @@ type = "stub"
 
 [backends.offline]
 type = "unreachable"
+
+[backends.provfail]
+type = "provfail"
 """
 
 
@@ -172,6 +231,21 @@ def submit_one(*extra: str) -> str:
 # ------------------------------------------------------------------ submit
 
 
+def test_submit_interrupted_after_provision_leaves_reclaimable_stub(env):
+    """A submit that dies after renting still persists a stub record carrying the
+    instance id, so `ps`/`gc` can see and reclaim the orphan (issue #7)."""
+    result = runner.invoke(
+        app, ["submit", "--yes", "--backend", "provfail", "--", "python", "train.py"]
+    )
+    assert result.exit_code != 0  # the submit failed...
+
+    ids = JobStore().list_ids()
+    assert len(ids) == 1  # ...but a record survived
+    rec = JobStore().load(ids[0])
+    assert rec is not None and rec.handle is not None
+    assert rec.handle.data == {"instance_id": "inst-42", "provisioning": True}
+
+
 def test_submit_yes_happy_path(env):
     result = runner.invoke(
         app,

{omnirun-0.2.2 → omnirun-0.2.3}/tests/test_marketplaces.py

@@ -309,6 +309,119 @@ def test_runpod_submit_happy_path(spec, fake_ssh, fake_stage):
     assert f"sleep {24 * 3600}" in watcher
 
 
+@respx.mock
+def test_submit_emits_provisioning_stub_before_wait(spec, fake_ssh, fake_stage):
+    """on_provisioning fires with the instance id the instant it's rented —
+    before provisioning finishes — so an interrupted submit stays reclaimable."""
+    respx.post(f"{REST_BASE}/pods").mock(
+        return_value=httpx.Response(200, json={"id": "pod123"})
+    )
+    respx.get(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "desiredStatus": "RUNNING",
+                "publicIp": "1.2.3.4",
+                "portMappings": {"22": 40022},
+            },
+        )
+    )
+    stubs: list[JobHandle] = []
+    handle = runpod_backend().submit(spec, h100_offer(), on_provisioning=stubs.append)
+
+    # Exactly one stub, carrying the instance id and flagged as provisioning,
+    # with none of the not-yet-known connection details.
+    (stub,) = stubs
+    assert stub.backend == "runpod"
+    assert stub.job_id == spec.job_id
+    assert stub.data == {"instance_id": "pod123", "provisioning": True}
+    # The final handle is the full one (superset).
+    assert handle.data["instance_id"] == "pod123"
+    assert handle.data["job_dir"].endswith(spec.job_id)
+
+
+@respx.mock
+def test_submit_stub_persists_when_interrupted(spec, fake_ssh, monkeypatch):
+    """If the submit dies after renting (here: staging raises), the stub was
+    already handed to the client, so the orphaned instance is recorded."""
+    respx.post(f"{REST_BASE}/pods").mock(
+        return_value=httpx.Response(200, json={"id": "pod123"})
+    )
+    respx.get(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "desiredStatus": "RUNNING",
+                "publicIp": "1.2.3.4",
+                "portMappings": {"22": 40022},
+            },
+        )
+    )
+    respx.delete(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(200, json={})
+    )
+
+    def boom(*a, **kw):
+        raise RuntimeError("git push exploded")
+
+    monkeypatch.setattr(jobdir, "stage_job", boom)
+    stubs: list[JobHandle] = []
+    with pytest.raises(BackendError, match="git push exploded"):
+        runpod_backend().submit(spec, h100_offer(), on_provisioning=stubs.append)
+    assert stubs and stubs[0].data["instance_id"] == "pod123"
+
+
+@respx.mock
+def test_status_on_provisioning_stub_flags_billing_instance(fake_ssh):
+    """status() of a stub whose instance is still up reports PROVISIONING and
+    tells the user it is billing (so ps surfaces the orphan)."""
+    respx.get(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(200, json={"desiredStatus": "RUNNING"})
+    )
+    stub = JobHandle(
+        backend="runpod",
+        job_id="train-abc123",
+        data={"instance_id": "pod123", "provisioning": True},
+    )
+    report = runpod_backend().status(stub)
+    assert report.status is JobStatus.PROVISIONING
+    assert "still billing" in report.detail
+    assert "pod123" in report.detail
+
+
+@respx.mock
+def test_status_on_provisioning_stub_lost_when_instance_gone(fake_ssh):
+    """A stub whose instance no longer exists is LOST, not a crash."""
+    respx.get(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(404, json={"error": "not found"})
+    )
+    stub = JobHandle(
+        backend="runpod",
+        job_id="train-abc123",
+        data={"instance_id": "pod123", "provisioning": True},
+    )
+    report = runpod_backend().status(stub)
+    assert report.status is JobStatus.LOST
+
+
+@respx.mock
+def test_gc_reclaims_provisioning_stub(fake_ssh):
+    """gc terminates the instance recorded in an interrupted-submit stub."""
+    respx.get(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(200, json={"desiredStatus": "RUNNING"})
+    )
+    delete = respx.delete(f"{REST_BASE}/pods/pod123").mock(
+        return_value=httpx.Response(200, json={})
+    )
+    stub = JobHandle(
+        backend="runpod",
+        job_id="train-abc123",
+        data={"instance_id": "pod123", "provisioning": True},
+    )
+    runpod_backend().gc(stub)
+    assert delete.called
+
+
 @respx.mock
 def test_runpod_submit_provision_timeout_terminates(spec, fake_ssh, fake_stage):
     respx.post(f"{REST_BASE}/pods").mock(

{omnirun-0.2.2 → omnirun-0.2.3}/tests/test_queue.py

@@ -7,7 +7,7 @@ import time
 from collections.abc import Iterator
 from pathlib import Path
 
-from omnirun.backends.base import Backend
+from omnirun.backends.base import Backend, ProvisioningSink
 from omnirun.config import BackendConfig, Config, DaemonConfig
 from omnirun.daemon import Daemon, daemon_address, send_request
 from omnirun.models import (
@@ -68,7 +68,12 @@ class FakeBackend(Backend):
             )
         ]
 
-    def submit(self, spec: JobSpec, offer: Offer) -> JobHandle:
+    def submit(
+        self,
+        spec: JobSpec,
+        offer: Offer,
+        on_provisioning: ProvisioningSink | None = None,
+    ) -> JobHandle:
         if self.fail_submit:
             raise RuntimeError("submit boom")
         self.submitted.append(spec.job_id)

{omnirun-0.2.2 → omnirun-0.2.3}/tests/test_repo.py

@@ -80,10 +80,61 @@ def test_capture_dirty_untracked(sample_repo: Path) -> None:
         capture_repo_state(sample_repo)
 
 
-def test_capture_allow_dirty(sample_repo: Path) -> None:
+def test_capture_allow_dirty_snapshots_working_tree(sample_repo: Path) -> None:
+    """--dirty ships a wip commit whose tree is the working tree, not plain HEAD,
+    and leaves the user's HEAD/index/working tree untouched (issue #6)."""
+    head = git(sample_repo, "rev-parse", "HEAD")
     (sample_repo / "job.py").write_text("changed\n")
+    (sample_repo / "scratch.txt").write_text("wip\n")  # untracked, non-ignored
+
     ref = capture_repo_state(sample_repo, allow_dirty=True)
+
     assert ref.dirty is True
+    # A real, distinct commit parented on HEAD — not HEAD itself.
+    assert ref.sha != head
+    assert git(sample_repo, "rev-parse", f"{ref.sha}^") == head
+    # The wip tree carries the uncommitted edits and the untracked file.
+    assert git(sample_repo, "show", f"{ref.sha}:job.py") == "changed"
+    assert git(sample_repo, "show", f"{ref.sha}:scratch.txt") == "wip"
+    # The user's repo is not mutated: HEAD unchanged, tree still dirty on disk.
+    assert git(sample_repo, "rev-parse", "HEAD") == head
+    assert (sample_repo / "job.py").read_text() == "changed\n"
+    assert git(sample_repo, "status", "--porcelain") != ""
+
+
+def test_capture_allow_dirty_excludes_gitignored(sample_repo: Path) -> None:
+    """A gitignored file (e.g. .env) rides out-of-band, never in the wip tree."""
+    (sample_repo / ".gitignore").write_text(".env\n")
+    git(sample_repo, "add", ".gitignore")
+    git(sample_repo, "commit", "-q", "-m", "add gitignore")
+    (sample_repo / ".env").write_text("SECRET=1\n")
+    (sample_repo / "job.py").write_text("changed\n")
+
+    ref = capture_repo_state(sample_repo, allow_dirty=True)
+
+    assert git(sample_repo, "show", f"{ref.sha}:job.py") == "changed"
+    missing = subprocess.run(
+        ["git", "-C", str(sample_repo), "cat-file", "-e", f"{ref.sha}:.env"],
+        capture_output=True,
+    )
+    assert missing.returncode != 0  # .env is absent from the wip tree
+
+
+def test_capture_allow_dirty_captures_deletion(sample_repo: Path) -> None:
+    """A deleted tracked file is absent from the wip tree."""
+    (sample_repo / "job.py").unlink()
+    ref = capture_repo_state(sample_repo, allow_dirty=True)
+    gone = subprocess.run(
+        ["git", "-C", str(sample_repo), "cat-file", "-e", f"{ref.sha}:job.py"],
+        capture_output=True,
+    )
+    assert gone.returncode != 0
+
+
+def test_capture_allow_dirty_clean_tree_is_head(sample_repo: Path) -> None:
+    """--dirty on a clean tree is a no-op: sha stays HEAD, no wip commit."""
+    ref = capture_repo_state(sample_repo, allow_dirty=True)
+    assert ref.dirty is False
     assert ref.sha == git(sample_repo, "rev-parse", "HEAD")
 
 

{omnirun-0.2.2 → omnirun-0.2.3}/uv.lock

@@ -595,7 +595,7 @@ wheels = [
 
 [[package]]
 name = "omnirun"
-version = "0.2.0"
+version = "0.2.3"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },