openrunner-sdk 2.41.8__tar.gz → 2.41.10__tar.gz

{openrunner_sdk-2.41.8 → openrunner_sdk-2.41.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: openrunner-sdk
-Version: 2.41.8
+Version: 2.41.10
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/gladiaio/openrunner
 Project-URL: Repository, https://github.com/gladiaio/openrunner
@@ -23,7 +23,7 @@ Requires-Dist: click<9.0,>=8.1
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: mcp<2.0,>=1.0
 Requires-Dist: nvidia-ml-py<14.0,>=12.0
-Requires-Dist: pillow<13.0,>=10.0
+Requires-Dist: pillow<14,>=10.0
 Requires-Dist: psutil<8.0,>=6.0
 Provides-Extra: accelerate
 Requires-Dist: accelerate<1.0,>=0.24; extra == 'accelerate'
@@ -35,7 +35,7 @@ Provides-Extra: dev
 Requires-Dist: matplotlib<5.0,>=3.7; extra == 'dev'
 Requires-Dist: num2words2<2.0,>=0.1; extra == 'dev'
 Requires-Dist: numpy<3.0,>=1.24; extra == 'dev'
-Requires-Dist: pillow<13.0,>=10.0; extra == 'dev'
+Requires-Dist: pillow<14,>=10.0; extra == 'dev'
 Requires-Dist: pytest-asyncio<1.0,>=0.23; extra == 'dev'
 Requires-Dist: pytest-cov<6.0,>=5.0; extra == 'dev'
 Requires-Dist: pytest<9.0,>=8.0; extra == 'dev'

{openrunner_sdk-2.41.8 → openrunner_sdk-2.41.10}/openrunner/CLAUDE.md

@@ -88,9 +88,7 @@ the LanceDB table on the dataset's own connection.
 
 Beyond FTS, the client exposes semantic / lexical search and **derivation by
 query** — build a child dataset from a SUBSET of a parent selected by a search
-(only matching primary keys stored; no media copied; de-dup + lineage automatic).
-These delegate to the native Rust core (`openrunner-core` wheel) when installed,
-falling back to pure-Python httpx otherwise:
+(only matching primary keys stored; no media copied; de-dup + lineage automatic):
 
 ```python
 ord.search_dataset(repo, q, mode="mixed"|"exact", lang="fr")   # BM25 / phrase, lang filter

{openrunner_sdk-2.41.8 → openrunner_sdk-2.41.10}/openrunner/__init__.py

@@ -72,6 +72,7 @@ from pathlib import Path
 from typing import Any
 
 from openrunner.artifact import Artifact
+from openrunner.errors import ArtifactUploadError, DatasetUploadError, UploadError
 from openrunner.model_card import (
     MODEL_CARD_STANDARD,
     model_card_standard,
@@ -166,7 +167,7 @@ from openrunner.reload import (
     dataset_handle,
 )
 from openrunner.sync import sync_environment, scan_environment
-from openrunner.projects import create_project
+from openrunner.projects import create_project, rename_project
 from openrunner.run import Run
 from openrunner.settings import SDKSettings, resolve_project, pin_project
 from openrunner.summary import Summary
@@ -861,7 +862,10 @@ def log_artifact(
     type: str | None = None,
     aliases: list[str] | None = None,
 ) -> dict[str, Any] | None:
-    """Upload an artifact to the active run. Never raises.
+    """Upload an artifact to the active run.
+
+    Raises :class:`ArtifactUploadError` when the upload fails (unless
+    ``OPENRUNNER_ARTIFACT_STRICT=0``); other SDK errors are logged, never raised.
 
     Args:
         artifact: An Artifact with files added via add_file/add_dir, OR a path
@@ -880,6 +884,9 @@ def log_artifact(
         return _active_run.log_artifact(
             artifact, name=name, type=type, aliases=aliases
         )
+    except ArtifactUploadError:
+        # An upload failure must reach the caller (see OPENRUNNER_ARTIFACT_STRICT).
+        raise
     except Exception as e:
         logger.warning("openrunner.log_artifact() failed: %s", e)
         return None
@@ -903,6 +910,8 @@ def finish_artifact(artifact: Artifact) -> dict[str, Any] | None:
             logger.warning("openrunner.finish_artifact(): no active run")
             return None
         return _active_run.finish_artifact(artifact)
+    except ArtifactUploadError:
+        raise
     except Exception as e:
         logger.warning("openrunner.finish_artifact() failed: %s", e)
         return None
@@ -928,6 +937,8 @@ def link_artifact(
             logger.warning("openrunner.link_artifact(): no active run")
             return None
         return _active_run.link_artifact(artifact, aliases=aliases)
+    except ArtifactUploadError:
+        raise
     except Exception as e:
         logger.warning("openrunner.link_artifact() failed: %s", e)
         return None
@@ -1079,6 +1090,10 @@ def push(
     Returns:
         The created version info dict, or None on failure.
 
+    Raises:
+        ArtifactUploadError: when any file fails to upload or the version
+            cannot be confirmed (unless ``OPENRUNNER_ARTIFACT_STRICT=0``).
+
     Example::
 
         openrunner.push("model.safetensors", "gladia/my-project/my-model",
@@ -1103,6 +1118,8 @@ def push(
             except Exception as e:  # noqa: BLE001
                 logger.warning("openrunner.push(): set_model_card failed: %s", e)
         return result
+    except ArtifactUploadError:
+        raise
     except Exception as e:  # noqa: BLE001
         logger.warning("openrunner.push() failed: %s", e)
         return None

openrunner_sdk-2.41.10/openrunner/_multipart.py

@@ -0,0 +1,116 @@
+"""Client side of S3 multipart uploads (presigned part URLs).
+
+The server signs one PUT URL per part (``/.../multipart/start``); this module
+PUTs the parts straight to object storage -- each part read from disk on
+demand, retried independently, a few in parallel -- and returns the ETags the
+server needs for ``CompleteMultipartUpload``. A single dropped connection now
+costs one 100 MiB part instead of the whole multi-GB file.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+import httpx
+
+from .errors import UploadError
+
+logger = logging.getLogger("openrunner")
+
+MIB = 1024 * 1024
+
+
+def _env_mb(name: str, default: int) -> int:
+    try:
+        return max(1, int(os.environ.get(name, "") or default))
+    except ValueError:
+        return default
+
+
+def multipart_threshold() -> int:
+    """Files at or above this size (bytes) use multipart. Env:
+    ``OPENRUNNER_MULTIPART_THRESHOLD_MB`` (default 100)."""
+    return _env_mb("OPENRUNNER_MULTIPART_THRESHOLD_MB", 100) * MIB
+
+
+def part_size() -> int:
+    """Requested part size (bytes). Env: ``OPENRUNNER_MULTIPART_PART_MB``
+    (default 100; the server clamps to S3 limits)."""
+    return _env_mb("OPENRUNNER_MULTIPART_PART_MB", 100) * MIB
+
+
+def part_workers() -> int:
+    """Parts uploaded concurrently. Env: ``OPENRUNNER_MULTIPART_WORKERS``
+    (default 4 -> ~4 x part size of RAM)."""
+    return _env_mb("OPENRUNNER_MULTIPART_WORKERS", 4)
+
+
+def _put_part(url: str, data: bytes, headers: dict[str, str], retries: int,
+              timeout: httpx.Timeout) -> str:
+    last: Exception | None = None
+    for attempt in range(retries):
+        try:
+            resp = httpx.put(url, content=data, headers=headers, timeout=timeout)
+            if resp.status_code >= 400:
+                raise UploadError(
+                    f"HTTP {resp.status_code}: {(resp.text or '')[:300]}"
+                )
+            etag = resp.headers.get("ETag") or resp.headers.get("etag")
+            if not etag:
+                raise UploadError(
+                    "storage returned no ETag for the part (bucket CORS must "
+                    "expose the ETag header)"
+                )
+            return etag
+        except Exception as e:  # noqa: BLE001
+            last = e
+            if attempt < retries - 1:
+                delay = min(1.0 * (2 ** attempt), 20.0) * (0.5 + random.random())
+                logger.warning(
+                    "part upload failed (attempt %d/%d), retrying in %.1fs: %s",
+                    attempt + 1, retries, delay, e,
+                )
+                time.sleep(delay)
+    raise UploadError(str(last) if last else "part upload failed")
+
+
+def upload_parts(
+    file_path: str,
+    parts: list[dict[str, Any]],
+    part_size: int,
+    *,
+    headers: dict[str, str] | None = None,
+    workers: int | None = None,
+    retries: int = 4,
+) -> list[dict[str, Any]]:
+    """PUT every part of ``file_path`` to its presigned URL.
+
+    ``parts`` is the server's ``[{"part_number", "url"}, ...]``. Returns
+    ``[{"part_number", "etag"}, ...]``; raises :class:`UploadError` naming the
+    first part that exhausted its retries.
+    """
+    size = os.path.getsize(file_path)
+    hdrs = dict(headers or {})
+    timeout = httpx.Timeout(connect=30.0, read=300.0, write=300.0, pool=30.0)
+
+    def _one(p: dict[str, Any]) -> dict[str, Any]:
+        n = int(p["part_number"])
+        offset = (n - 1) * part_size
+        length = max(0, min(part_size, size - offset))
+        with open(file_path, "rb") as fh:
+            fh.seek(offset)
+            data = fh.read(length)
+        try:
+            etag = _put_part(p["url"], data, hdrs, retries, timeout)
+        except UploadError as e:
+            raise UploadError(f"part {n}/{len(parts)} of {os.path.basename(file_path)} failed: {e}") from e
+        return {"part_number": n, "etag": etag}
+
+    with ThreadPoolExecutor(max_workers=max(1, workers or part_workers())) as ex:
+        results = list(ex.map(_one, parts))
+    return sorted(results, key=lambda r: r["part_number"])

{openrunner_sdk-2.41.8 → openrunner_sdk-2.41.10}/openrunner/api_client.py

@@ -19,6 +19,8 @@ from typing import Any
 
 import httpx
 
+from . import _multipart as _mp
+
 logger = logging.getLogger("openrunner")
 
 # -- Retry configuration ------------------------------------------------------
@@ -77,6 +79,38 @@ def _retry_request(fn, *args, **kwargs):
     raise last_error
 
 
+def _raise_for_status(response: httpx.Response) -> None:
+    """raise_for_status() that keeps the server's error detail in the message."""
+    if response.status_code < 400:
+        return
+    detail = ""
+    try:
+        body = response.json()
+        detail = body.get("detail", body) if isinstance(body, dict) else body
+    except Exception:  # noqa: BLE001
+        detail = (response.text or "")[:300]
+    raise httpx.HTTPStatusError(
+        f"HTTP {response.status_code} for {response.request.method} "
+        f"{response.request.url.path}: {detail}",
+        request=response.request,
+        response=response,
+    )
+
+
+def _describe(e: BaseException) -> str:
+    """Human-readable cause (``TimeoutException`` str() is often empty)."""
+    text = str(e).strip()
+    return f"{type(e).__name__}: {text}" if text else type(e).__name__
+
+
+def _fmt_bytes(n: int) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+    return f"{n} B"
+
+
 class APIClient:
     """HTTP client for the OpenRunner server API.
 
@@ -779,6 +813,7 @@ class APIClient:
         metadata: dict[str, Any] | None = None,
         base_version_id: str | None = None,
         message: str | None = None,
+        raise_on_error: bool = False,
     ) -> dict[str, Any] | None:
         """POST /runs/{run_id}/artifacts -- create a new artifact version.
 
@@ -807,27 +842,56 @@ class APIClient:
                 body["base_version_id"] = base_version_id
             if message is not None:
                 body["message"] = message
-            response = self._request("post",f"/runs/{run_id}/artifacts", json=body)
-            response.raise_for_status()
+            # The server does per-file dedup lookups + layer building on this
+            # call; under load it can exceed the 30s default read timeout, and a
+            # timed-out POST that is retried races the still-running first
+            # request (version-number unique constraint -> 500). Give it room.
+            response = self._request(
+                "post", f"/runs/{run_id}/artifacts", json=body,
+                timeout=httpx.Timeout(30.0, read=180.0),
+            )
+            _raise_for_status(response)
             return response.json()
         except Exception as e:
+            if raise_on_error:
+                from .errors import ArtifactUploadError
+
+                raise ArtifactUploadError(
+                    f"could not create artifact version {name!r}: {_describe(e)}"
+                ) from e
             logger.warning("create_artifact_version failed: %s", e)
             return None
 
     def confirm_artifact_version(
-        self, version_id: str
+        self, version_id: str, raise_on_error: bool = False
     ) -> dict[str, Any] | None:
         """POST /artifacts/versions/{version_id}/confirm -- confirm upload.
 
-        Returns {status, version}, or None on failure.
+        Returns {status, version}, or None on failure. With
+        ``raise_on_error=True`` a failure (including a server response whose
+        status is not ``confirmed``, e.g. 409 missing objects) raises
+        :class:`~openrunner.errors.ArtifactUploadError`.
         """
         try:
             response = self._request("post",
-                f"/artifacts/versions/{version_id}/confirm"
+                f"/artifacts/versions/{version_id}/confirm",
+                timeout=httpx.Timeout(30.0, read=180.0),
             )
-            response.raise_for_status()
-            return response.json()
-        except Exception as e:
+            _raise_for_status(response)
+            data = response.json()
+            if raise_on_error and isinstance(data, dict) and data.get("status") not in (
+                None, "confirmed"
+            ):
+                raise RuntimeError(f"server left the version in state {data.get('status')!r}")
+            return data
+        except Exception as e:
+            if raise_on_error:
+                from .errors import ArtifactUploadError
+
+                raise ArtifactUploadError(
+                    f"could not confirm artifact version {version_id}: {_describe(e)}",
+                    version_id=version_id,
+                ) from e
             logger.warning("confirm_artifact_version failed: %s", e)
             return None
 
@@ -1129,52 +1193,124 @@ class APIClient:
             return []
 
     def upload_file_to_presigned_url(
-        self, presigned_url: str, file_path: str, run_id: str = ""
+        self,
+        presigned_url: str,
+        file_path: str,
+        run_id: str = "",
+        *,
+        version_id: str | None = None,
+        name: str | None = None,
+        raise_on_error: bool = False,
     ) -> bool:
-        """PUT raw file bytes to a presigned URL.
-
-        Handles three cases:
-        1. Relative proxy URL (/api/v1/runs/...) — use authenticated client
-        2. Absolute URL — try direct PUT, fall back to proxy on ConnectError
-        3. ConnectError fallback — PUT via /runs/{run_id}/files/{name}
+        """Upload a local file for an artifact version.
+
+        Handles:
+        1. Large files (>= ``OPENRUNNER_MULTIPART_THRESHOLD_MB``, default 100)
+           when ``version_id`` + ``name`` are given and storage is directly
+           reachable: S3 multipart via presigned part URLs (parts retried
+           independently). Falls back to a single PUT on servers without the
+           multipart endpoints.
+        2. Relative proxy URL (/api/v1/runs/...) — streamed PUT via the
+           authenticated client.
+        3. Absolute URL — streamed direct PUT, falling back to the API proxy on
+           ConnectError.
+
+        Returns True on success. On failure returns False after logging the
+        real cause at ERROR level, or raises
+        :class:`~openrunner.errors.ArtifactUploadError` if ``raise_on_error``.
         """
-        import os
-
         try:
-            file_size = os.path.getsize(file_path)
-            # Use streaming for large files, read into memory for small ones
-            if file_size > 50 * 1024 * 1024:  # 50 MB
-                data = open(file_path, "rb")  # noqa: SIM115
-            else:
-                with open(file_path, "rb") as f:
-                    data = f.read()
+            size = os.path.getsize(file_path)
+            if (
+                version_id
+                and name
+                and not presigned_url.startswith("/")
+                and size >= _mp.multipart_threshold()
+            ):
+                if self._upload_multipart(version_id, name, file_path, size):
+                    return True
+            self._upload_single(presigned_url, file_path, size, run_id)
+            return True
+        except Exception as e:
+            label = name or os.path.basename(file_path)
+            msg = f"upload of {label!r} ({_fmt_bytes(os.path.getsize(file_path)) if os.path.exists(file_path) else '?'}) failed: {_describe(e)}"
+            if raise_on_error:
+                from .errors import ArtifactUploadError
 
-            # Scale timeout by file size (minimum 60s, ~30s per 100MB)
-            upload_timeout = max(60.0, file_size / (3 * 1024 * 1024))
+                raise ArtifactUploadError(
+                    msg, version_id=version_id, failed_files=[label]
+                ) from e
+            logger.error("upload_file_to_presigned_url: %s", msg)
+            return False
 
-            # Case 1: relative proxy URL from server
-            if presigned_url.startswith("/"):
-                api_path = presigned_url.replace("/api/v1/", "/", 1) if presigned_url.startswith("/api/v1/") else presigned_url
-                resp = self._client.put(api_path, content=data, timeout=upload_timeout)
-                return resp.status_code == 200
+    def _upload_single(self, presigned_url: str, file_path: str, size: int, run_id: str) -> None:
+        """One streamed PUT (file read in chunks, never fully in RAM). Raises."""
+        # Per-operation timeouts. The read timeout covers the wait for the
+        # response AFTER the body is sent: the API proxy hashes the body and
+        # copies it to storage before answering, so scale it with size.
+        read_timeout = max(120.0, size / (2 * 1024 * 1024))
+        timeout = httpx.Timeout(connect=30.0, read=read_timeout, write=300.0, pool=30.0)
+
+        def _put(client_put, url: str) -> httpx.Response:
+            with open(file_path, "rb") as fh:  # fresh handle per attempt
+                return client_put(url, content=fh, timeout=timeout)
 
-            # Case 2: absolute presigned URL (direct to S3)
+        if presigned_url.startswith("/"):
+            api_path = (
+                presigned_url.replace("/api/v1/", "/", 1)
+                if presigned_url.startswith("/api/v1/") else presigned_url
+            )
+            _raise_for_status(_put(self._client.put, api_path))
+            return
+        try:
+            _raise_for_status(_put(httpx.put, presigned_url))
+        except (httpx.ConnectError, httpx.ConnectTimeout):
+            if not run_id:
+                raise
+            fname = os.path.basename(file_path)
+            _raise_for_status(_put(self._client.put, f"/runs/{run_id}/files/{fname}"))
+
+    def _upload_multipart(self, version_id: str, name: str, file_path: str, size: int) -> bool:
+        """Multipart upload of one artifact file. Returns False when the server
+        cannot do multipart (old server / proxy-only storage) so the caller
+        falls back to a single PUT; raises on an actual upload failure."""
+        base = f"/artifacts/versions/{version_id}/multipart"
+        start = self._request(
+            "post", f"{base}/start",
+            json={"name": name, "size": size, "part_size": _mp.part_size()},
+        )
+        if start.status_code in (404, 405):
+            detail = ""
             try:
-                resp = httpx.put(presigned_url, content=data, timeout=upload_timeout)
-                resp.raise_for_status()
-                return True
-            except (httpx.ConnectError, httpx.ConnectTimeout):
-                # Case 3: fallback to API proxy
-                if run_id:
-                    fname = os.path.basename(file_path)
-                    proxy_resp = self._client.put(
-                        f"/runs/{run_id}/files/{fname}", content=data, timeout=upload_timeout
-                    )
-                    return proxy_resp.status_code == 200
+                detail = str(start.json().get("detail", ""))
+            except Exception:  # noqa: BLE001
+                pass
+            if start.status_code == 405 or detail in ("", "Not Found"):
+                logger.info("server has no multipart endpoint; using a single PUT for %s", name)
                 return False
-        except Exception as e:
-            logger.warning("upload_file_to_presigned_url failed: %s", e)
+        _raise_for_status(start)
+        info = start.json()
+        if not info.get("multipart"):
             return False
+        upload_id = info["upload_id"]
+        logger.info(
+            "uploading %s (%s) in %d parts", name, _fmt_bytes(size), len(info["parts"])
+        )
+        try:
+            etags = _mp.upload_parts(file_path, info["parts"], int(info["part_size"]))
+            done = self._request(
+                "post", f"{base}/complete",
+                json={"name": name, "upload_id": upload_id, "parts": etags},
+                timeout=httpx.Timeout(30.0, read=300.0),
+            )
+            _raise_for_status(done)
+        except Exception:
+            try:
+                self._client.post(f"{base}/abort", json={"name": name, "upload_id": upload_id})
+            except Exception:  # noqa: BLE001
+                pass
+            raise
+        return True
 
     def download_file_from_presigned_url(
         self, presigned_url: str