openrunner-sdk 2.40.1__tar.gz → 2.41.1__tar.gz

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.40.1
+Version: 2.41.1
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/openrunner/__init__.py

@@ -181,7 +181,7 @@ launch.from_run = _launch_from_run  # type: ignore[attr-defined]
 # openrunner.trace.patch_openai() syntax
 trace.patch_openai = _patch_openai  # type: ignore[attr-defined]
 
-__version__ = "2.40.1"
+__version__ = "2.41.1"
 
 logger = logging.getLogger("openrunner")
 

openrunner_sdk-2.41.1/openrunner/_retry_transport.py

@@ -0,0 +1,94 @@
+"""A drop-in httpx transport that transparently retries transient failures.
+
+Wraps the default ``httpx.HTTPTransport`` and retries:
+  * connection/timeout/protocol errors (the request never reached the app), and
+  * ``502 / 503`` responses (bad gateway / unavailable — e.g. the API container
+    being recreated by a deploy, which the app also never processed), plus
+  * ``504`` and ``429`` — ``504`` only for idempotent methods (GET/HEAD/PUT/
+    DELETE/OPTIONS) so a POST that may have executed is never silently repeated;
+    ``429`` honours ``Retry-After``.
+
+Backoff is exponential with jitter, capped. This lets long-running SDK jobs
+(dataset materialise, bulk upload, run logging) ride out a transient blip instead
+of crashing — the failure mode that corrupted an EarningsCall fetch mid-download.
+"""
+from __future__ import annotations
+
+import random
+import time
+
+import httpx
+
+_IDEMPOTENT = frozenset({"GET", "HEAD", "PUT", "DELETE", "OPTIONS", "TRACE"})
+# 502/503: app never handled the request -> safe to retry for any method.
+# 504: request may have executed -> retry idempotent methods only.
+_RETRY_ALWAYS = frozenset({502, 503})
+
+
+class RetryTransport(httpx.BaseTransport):
+    """httpx transport retrying transient 5xx/429/network errors with backoff."""
+
+    def __init__(
+        self,
+        wrapped: httpx.BaseTransport | None = None,
+        *,
+        max_retries: int = 4,
+        base_delay: float = 1.0,
+        max_delay: float = 30.0,
+    ) -> None:
+        self._wrapped = wrapped if wrapped is not None else httpx.HTTPTransport()
+        self.max_retries = max_retries
+        self.base_delay = base_delay
+        self.max_delay = max_delay
+
+    def _sleep(self, attempt: int, retry_after: float = 0.0) -> None:
+        delay = min(self.base_delay * (2 ** attempt), self.max_delay)
+        delay = max(delay, min(retry_after, self.max_delay))
+        time.sleep(delay * (0.5 + random.random()))  # 0.5x–1.5x jitter
+
+    def _should_retry_status(self, code: int, method: str) -> bool:
+        if code in _RETRY_ALWAYS or code == 429:
+            return True
+        if code == 504 and method.upper() in _IDEMPOTENT:
+            return True
+        return False
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        last_exc: Exception | None = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = self._wrapped.handle_request(request)
+            except (
+                httpx.ConnectError,
+                httpx.ConnectTimeout,
+                httpx.ReadTimeout,
+                httpx.WriteTimeout,
+                httpx.PoolTimeout,
+                httpx.RemoteProtocolError,
+            ) as exc:
+                last_exc = exc
+                if attempt >= self.max_retries:
+                    raise
+                self._sleep(attempt)
+                continue
+
+            if attempt < self.max_retries and self._should_retry_status(
+                response.status_code, request.method
+            ):
+                retry_after = 0.0
+                try:
+                    retry_after = float(response.headers.get("Retry-After", 0) or 0)
+                except (TypeError, ValueError):
+                    retry_after = 0.0
+                response.close()
+                self._sleep(attempt, retry_after)
+                continue
+            return response
+
+        # Only reached if the last attempt raised then the loop fell through.
+        if last_exc is not None:
+            raise last_exc
+        raise httpx.HTTPError("request failed after retries")  # pragma: no cover
+
+    def close(self) -> None:
+        self._wrapped.close()

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/openrunner/datasets.py

@@ -194,10 +194,12 @@ class DatasetsClient:
                or _g.get("base_url") or "http://localhost:8000")
         if not key:
             raise ValueError("API key required. Set OPENRUNNER_API_KEY, pass api_key=, or run `openrunner login`.")
+        from ._retry_transport import RetryTransport
         self._http = httpx.Client(
             base_url=f"{url.rstrip('/')}/api/v1",
             headers={"Authorization": f"Bearer {key}"},
             timeout=timeout,
+            transport=RetryTransport(),
         )
 
     # -- resolution ----------------------------------------------------------

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/openrunner/gpu/api_integration.py

@@ -408,40 +408,29 @@ class GpuAPIClient:
             logger.warning("get_job failed: %s", e)
             return None
 
-    def ovh_training_run(
-        self, *, image: str, command: str, gpu: int = 1,
-        name: str | None = None, project_id: str | None = None,
-    ) -> dict[str, Any] | None:
-        """POST /api/v1/gpu/ai-training/run -- submit an OVH AI Training job
-        server-side (the OVH credential never leaves the API). Returns
-        {job_id,status} or None."""
+    def ovh_training_run_stream(
+        self, *, image: str, command: str, gpu: int = 1, name: str | None = None,
+        project_id: str | None = None,
+    ):
+        """POST /api/v1/gpu/ai-training/run and YIELD the job's log lines. The
+        server runs `ovhai job run` with the org's stored user/password (never
+        sent here) and streams logs. A ``__ENGINE_EXIT__ <code>`` line ends it."""
         org = self.org_id()
         body: dict[str, Any] = {"image": image, "command": command, "gpu": gpu}
         if name:
             body["name"] = name
         if project_id:
             body["project_id"] = project_id
-        try:
-            r = self._client.post("/gpu/ai-training/run", params={"org_id": org}, json=body)
-            r.raise_for_status()
-            return r.json()
-        except Exception as e:
-            detail = getattr(getattr(e, "response", None), "text", "") or str(e)
-            logger.warning("ovh_training_run failed: %s", detail)
-            return None
-
-    def ovh_training_status(self, job_id: str) -> dict[str, Any] | None:
-        """GET /api/v1/gpu/ai-training/{id} -- poll status + logs."""
-        try:
-            r = self._client.get(
-                f"/gpu/ai-training/{job_id}", params={"org_id": self.org_id()}
-            )
+        with self._client.stream(
+            "POST", "/gpu/ai-training/run", params={"org_id": org}, json=body,
+        ) as r:
             if r.status_code >= 400:
-                return None
-            return r.json()
-        except Exception as e:
-            logger.warning("ovh_training_status failed: %s", e)
-            return None
+                r.read()
+                yield f"[openrunner] server error {r.status_code}: {r.text[:300]}"
+                yield "__ENGINE_EXIT__ 1"
+                return
+            for line in r.iter_lines():
+                yield line
 
     # ------------------------------------------------------------------
     # Lifecycle

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/openrunner/gpu/runner.py

@@ -534,13 +534,11 @@ def run_ovh_training_sync(
     project: str | None = None,
     name: str | None = None,
     emit: Callable[[str], None] | None = None,
-    poll_interval: float = 5.0,
 ) -> dict[str, Any]:
-    """Governed OVH AI Training run — SERVER-SIDE. Submits a job (the OVH keys
-    stay in the API vault, never on this machine), then polls status + logs until
-    it finishes. COMMAND runs in IMAGE on `gpu` GPUs. Data comes from OpenRunner
-    (datasets/artifacts) — no OVH object-storage volumes. Returns
-    ``{"job_id","exit_code"}``."""
+    """Governed OVH AI Training run — SERVER-SIDE. The API runs `ovhai job run`
+    on the engine-executor with the org's stored user/password (never on this
+    machine) and streams the job's logs. COMMAND runs in IMAGE on `gpu` GPUs. Data
+    comes from OpenRunner (no OVH object-storage volumes). Returns {"exit_code"}."""
     from openrunner.gpu import _get_api_client
 
     say = emit or (lambda m: print(m, flush=True))
@@ -548,33 +546,20 @@ def run_ovh_training_sync(
     if client is None:
         raise RuntimeError("Not logged in. Run 'openrunner login' first.")
 
-    say(f"[openrunner] submitting OVH AI Training job (image={image}, gpu={gpu}) "
-        "— OVH keys stay server-side ...")
-    res = client.ovh_training_run(image=image, command=command, gpu=gpu,
-                                  name=name, project_id=project)
-    if not res or not res.get("job_id"):
-        raise RuntimeError(
-            "failed to submit OVH AI Training job — need an org 'ovhcloud' "
-            "credential (openrunner gpu credentials add --provider ovhcloud ...)."
-        )
-    jid = res["job_id"]
-    say(f"[openrunner] OVH job {jid} ({res.get('status')}) — streaming ...")
-
-    seen = 0
-    while True:
-        time.sleep(poll_interval)
-        st = client.ovh_training_status(jid) or {}
-        logs = st.get("logs") or ""
-        if len(logs) > seen:
-            say(logs[seen:].rstrip("\n"))
-            seen = len(logs)
-        status = st.get("status")
-        if status in ("terminated", "error"):
-            code = st.get("exit_code")
-            if code is None:
-                code = 0 if status == "terminated" else 1
-            say(f"[openrunner] OVH AI Training job {status} (exit={code}).")
-            return {"job_id": jid, "exit_code": int(code)}
+    say(f"[openrunner] launching OVH AI Training (image={image}, gpu={gpu}) "
+        "server-side — OVH user/password stays in OpenRunner ...")
+    exit_code = 1
+    for line in client.ovh_training_run_stream(
+        image=image, command=command, gpu=gpu, name=name, project_id=project,
+    ):
+        if line.startswith("__ENGINE_EXIT__"):
+            try:
+                exit_code = int(line.split()[1])
+            except Exception:
+                exit_code = 1
+            continue
+        say(line.rstrip("\n"))
+    return {"exit_code": exit_code}
 
 
 def run_engine_serverside_sync(

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/openrunner/install_commands.py

@@ -3599,13 +3599,15 @@ mentions a "distant GPU":
   SCRIPT (GPU is declared in `@app.function(gpu=...)`, so no `--gpu`), the Run
   auto-terminates when the function returns. Don't hunt for an SSH host/runner.
   For native Forge use, the `forge-deploy` skill covers install/auth/write/run.
-- OVHcloud **AI Training** (managed GPU batch jobs): governed, submitted
-  SERVER-SIDE with the org's OVH credential (the `ovhcloud` app/secret/consumer
-  keys, never on the client). Run:
+- OVHcloud **AI Training** (managed GPU batch jobs): governed, run SERVER-SIDE
+  via the `ovhai` CLI on the engine-executor using the org's stored OVH
+  **user/password** (an OpenStack user with the AI Training Operator +
+  ObjectStore operator roles) -- the password never reaches the client. Store it
+  in Settings (provider `ovhai`: username + password), then run:
   `openrunner gpu run --provider ovhai --image <img> --gpu-count N "<train cmd>"`
-  -- COMMAND runs in the image on N GPUs; poll streams logs; it auto-stops when
-  the command exits. **Do NOT use OVH object-storage `--volume` mounts** -- data
-  comes from OpenRunner (datasets/artifacts). Needs an `ovhcloud` credential.
+  -- COMMAND runs in the image on N GPUs; the job's logs stream back; it
+  auto-stops when the command exits. **Do NOT use OVH object-storage `--volume`
+  mounts** -- data comes from OpenRunner (datasets/artifacts).
 - Provider keys are stored encrypted with `openrunner gpu credentials add`
   (e.g. `--provider forge --api-key fg_..`); list/kill pods with
   `openrunner gpu ls` / `openrunner gpu terminate`. Governed compute -- the key

{openrunner_sdk-2.40.1 → openrunner_sdk-2.41.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.40.1"
+version = "2.41.1"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}

openrunner_sdk-2.41.1/tests/test_gpu/test_ovh_training.py

@@ -0,0 +1,29 @@
+"""SDK OVH AI Training governed run — streams the server's job logs."""
+from __future__ import annotations
+from unittest.mock import patch
+from openrunner.gpu import runner as R
+
+
+class _Fake:
+    def __init__(self, lines): self.args=None; self._lines=lines
+    def ovh_training_run_stream(self, **kwargs):
+        self.args=kwargs
+        yield from self._lines
+
+
+def test_ovh_training_streams_and_parses_exit():
+    fake=_Fake(["epoch 1","epoch 2","__ENGINE_EXIT__ 0"])
+    out=[]
+    with patch("openrunner.gpu._get_api_client", return_value=fake):
+        res=R.run_ovh_training_sync(command="python train.py",
+                                    image="ovhcom/ai-training-pytorch", gpu=2, emit=out.append)
+    assert res=={"exit_code":0}
+    assert fake.args["image"]=="ovhcom/ai-training-pytorch" and fake.args["gpu"]==2
+    assert fake.args["command"]=="python train.py"
+    assert any("epoch 2" in l for l in out) and not any("__ENGINE_EXIT__" in l for l in out)
+
+
+def test_ovh_training_nonzero_exit():
+    with patch("openrunner.gpu._get_api_client", return_value=_Fake(["boom","__ENGINE_EXIT__ 4"])):
+        res=R.run_ovh_training_sync(command="x", image="i", emit=lambda _l:None)
+    assert res["exit_code"]==4

openrunner_sdk-2.41.1/tests/test_retry_transport.py

@@ -0,0 +1,66 @@
+"""Tests for RetryTransport — the SDK-level transient-failure retry."""
+import httpx
+import pytest
+
+from openrunner._retry_transport import RetryTransport
+
+
+def _client(responder, **kw):
+    mock = httpx.MockTransport(responder)
+    return httpx.Client(base_url="http://t", transport=RetryTransport(mock, base_delay=0.0, **kw))
+
+
+def test_retries_502_then_succeeds():
+    calls = {"n": 0}
+    def responder(req):
+        calls["n"] += 1
+        return httpx.Response(502 if calls["n"] < 3 else 200, text="ok")
+    r = _client(responder).get("/x")
+    assert r.status_code == 200 and calls["n"] == 3
+
+
+def test_gives_up_after_max_retries():
+    calls = {"n": 0}
+    def responder(req):
+        calls["n"] += 1
+        return httpx.Response(503)
+    r = _client(responder, max_retries=2).get("/x")
+    assert r.status_code == 503 and calls["n"] == 3  # 1 + 2 retries
+
+
+def test_4xx_not_retried():
+    calls = {"n": 0}
+    def responder(req):
+        calls["n"] += 1
+        return httpx.Response(404)
+    r = _client(responder).get("/x")
+    assert r.status_code == 404 and calls["n"] == 1
+
+
+def test_504_retried_for_get_but_not_post():
+    for method, expect in [("GET", 3), ("POST", 1)]:
+        calls = {"n": 0}
+        def responder(req):
+            calls["n"] += 1
+            return httpx.Response(504)
+        _client(responder, max_retries=2).request(method, "/x")
+        assert calls["n"] == expect, method
+
+
+def test_connect_error_retried_then_raises():
+    calls = {"n": 0}
+    def responder(req):
+        calls["n"] += 1
+        raise httpx.ConnectError("boom")
+    with pytest.raises(httpx.ConnectError):
+        _client(responder, max_retries=2).get("/x")
+    assert calls["n"] == 3
+
+
+def test_429_honored_and_retried():
+    calls = {"n": 0}
+    def responder(req):
+        calls["n"] += 1
+        return httpx.Response(429, headers={"Retry-After": "0"}) if calls["n"] < 2 else httpx.Response(200)
+    r = _client(responder).get("/x")
+    assert r.status_code == 200 and calls["n"] == 2

openrunner_sdk-2.40.1/tests/test_gpu/test_ovh_training.py

@@ -1,47 +0,0 @@
-"""SDK OVH AI Training governed run (server-side; OVH keys never on client)."""
-
-from __future__ import annotations
-
-from unittest.mock import patch
-
-from openrunner.gpu import runner as R
-
-
-class _Fake:
-    def __init__(self, statuses):
-        self.run_args = None
-        self._st = list(statuses)
-
-    def ovh_training_run(self, **kwargs):
-        self.run_args = kwargs
-        return {"job_id": "ai-1", "status": "queued"}
-
-    def ovh_training_status(self, job_id):
-        return self._st.pop(0) if self._st else {"status": "terminated", "exit_code": 0}
-
-
-def test_ovh_training_submits_and_streams():
-    fake = _Fake([
-        {"status": "running", "logs": "epoch 1\n"},
-        {"status": "running", "logs": "epoch 1\nepoch 2\n"},
-        {"status": "terminated", "exit_code": 0, "logs": "epoch 1\nepoch 2\ndone\n"},
-    ])
-    lines: list[str] = []
-    with patch("openrunner.gpu._get_api_client", return_value=fake):
-        res = R.run_ovh_training_sync(
-            command="python train.py", image="ovhcom/ai-training-pytorch",
-            gpu=2, emit=lines.append, poll_interval=0,
-        )
-    assert res == {"job_id": "ai-1", "exit_code": 0}
-    assert fake.run_args["image"] == "ovhcom/ai-training-pytorch"
-    assert fake.run_args["gpu"] == 2
-    assert fake.run_args["command"] == "python train.py"
-    assert any("done" in ln for ln in lines)
-
-
-def test_ovh_training_nonzero_exit():
-    fake = _Fake([{"status": "error", "exit_code": 7, "logs": "boom\n"}])
-    with patch("openrunner.gpu._get_api_client", return_value=fake):
-        res = R.run_ovh_training_sync(command="x", image="i", emit=lambda _l: None,
-                                      poll_interval=0)
-    assert res["exit_code"] == 7