bladekit 4.5.0__py3-none-any.whl

bladekit/exceptions.py

@@ -0,0 +1,544 @@
+"""Exception hierarchy for the Blade AI Fabric SDK.
+
+Design rule (task requirement 6 / SPEC §3.3): a *remote user exception* must be
+clearly distinct from a *platform error*.
+
+- ``RemoteFunctionError`` wraps an exception raised by the user's function on the
+  remote side. When the original exception can be unpickled it is re-raised
+  directly (with the remote traceback attached); otherwise this generic wrapper
+  carries the remote traceback so the user still sees a readable message.
+- ``FabricError`` and its subclasses signal that *the platform* failed (API
+  envelope errors, run-level infra errors such as NODE_LOST / CONTAINER_FAILED /
+  VERSION_MISMATCH). These are never confused with a legitimate user exception.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any, Mapping, Optional
+
+
+class FabricError(Exception):
+    """Base class for every platform-side error surfaced by the SDK.
+
+    Was ``ForgeError`` before the AI Fabric rename (PAAS-771); that name is kept
+    as a deprecated alias of this very class, so ``except ForgeError`` still
+    catches everything.
+    """
+
+
+# Old public name -> new public name. Resolved lazily by __getattr__ below so
+# each use of an old name emits a DeprecationWarning, while returning the SAME
+# object (isinstance / except keep working across the rename).
+DEPRECATED_ALIASES = {"ForgeError": "FabricError"}
+
+
+def resolve_deprecated(name: str, *, stacklevel: int = 3) -> Any:
+    """Return the object behind a deprecated alias, warning once per call site.
+
+    ``stacklevel=3`` points the warning at the code that used the old name when
+    called from a module ``__getattr__`` (this frame, __getattr__, the caller).
+    """
+    new = DEPRECATED_ALIASES[name]
+    warnings.warn(
+        f"{name} is deprecated, use {new} instead (renamed with the bladekit "
+        "package; the old name will be removed in a future major release).",
+        DeprecationWarning,
+        stacklevel=stacklevel,
+    )
+    return globals()[new]
+
+
+def __getattr__(name: str) -> Any:
+    if name in DEPRECATED_ALIASES:
+        return resolve_deprecated(name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+# --- API envelope errors (conventions.md §2) --------------------------------
+
+
+class APIError(FabricError):
+    """An error returned in the standard JSON error envelope.
+
+    ``code`` is the stable machine code (SCREAMING_SNAKE_CASE). Unknown codes are
+    tolerated and treated like any other API error (conventions.md §2/§5).
+    """
+
+    def __init__(
+        self,
+        code: str,
+        message: str,
+        *,
+        status_code: Optional[int] = None,
+        details: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        self.code = code
+        self.api_message = message
+        self.status_code = status_code
+        self.details = dict(details or {})
+        super().__init__(f"[{code}] {message}")
+
+
+class AuthenticationError(APIError):
+    """UNAUTHENTICATED — token missing/invalid/revoked (HTTP 401)."""
+
+
+class PermissionDeniedError(APIError):
+    """PERMISSION_DENIED — token valid but operation forbidden (HTTP 403)."""
+
+
+class NotFoundError(APIError):
+    """*_NOT_FOUND — object does not exist or belongs to another user (404)."""
+
+
+class InvalidArgumentError(APIError):
+    """INVALID_ARGUMENT / INVALID_CURSOR / IMAGE_ALIAS_UNKNOWN (400)."""
+
+
+class ConflictError(APIError):
+    """409 family — BUNDLE_NOT_READY, RUN_NOT_CANCELLABLE, etc."""
+
+
+class PayloadTooLargeError(APIError):
+    """413 family — BUNDLE_TOO_LARGE / PAYLOAD_TOO_LARGE."""
+
+
+class UnavailableError(APIError):
+    """UNAVAILABLE (503) — a dependency is down; the caller may retry."""
+
+
+# --- Connection / transport -------------------------------------------------
+
+
+class ConnectionFailed(FabricError):
+    """Could not reach the control plane at all (DNS, refused, timeout)."""
+
+
+# --- Run-level infrastructure errors (conventions.md RunError registry) -----
+
+
+class RunError(FabricError):
+    """A run/call failed for an *infrastructure* reason (not a user exception).
+
+    ``code`` mirrors ``RunError.code`` from the contract (NODE_LOST,
+    CONTAINER_FAILED, IMAGE_PULL_FAILED, …). Specific subclasses give nicer,
+    actionable messages for the common cases; unknown codes fall back to this
+    base class.
+    """
+
+    code: str = "INTERNAL"
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        code: Optional[str] = None,
+        run_id: Optional[str] = None,
+        details: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        if code is not None:
+            self.code = code
+        self.run_id = run_id
+        self.details = dict(details or {})
+        super().__init__(message)
+
+
+class NodeLostError(RunError):
+    code = "NODE_LOST"
+
+
+class ContainerFailedError(RunError):
+    code = "CONTAINER_FAILED"
+
+
+class ImagePullFailedError(RunError):
+    code = "IMAGE_PULL_FAILED"
+
+
+class RunnerCrashedError(RunError):
+    code = "RUNNER_CRASHED"
+
+
+class BundleFetchFailedError(RunError):
+    code = "BUNDLE_FETCH_FAILED"
+
+
+class VersionMismatchError(RunError):
+    """PYTHON_VERSION_MISMATCH or CLOUDPICKLE_VERSION_MISMATCH (handshake §5)."""
+
+    code = "PYTHON_VERSION_MISMATCH"
+
+
+class TimeoutError_(RunError):  # noqa: N801 - explicit, avoid shadowing builtin
+    code = "TIMEOUT"
+
+
+class RunCancelled(FabricError):
+    """The run was cancelled (locally via Ctrl-C, or by the timeout watcher)."""
+
+    def __init__(self, run_id: str, *, message: Optional[str] = None) -> None:
+        self.run_id = run_id
+        super().__init__(message or f"run {run_id} was cancelled")
+
+
+class RunLostError(NodeLostError):
+    """Raised to the caller when a run/call transitions to ``lost``."""
+
+
+# --- Remote user exception --------------------------------------------------
+
+
+class RemoteError(FabricError):
+    """Carries a remote user exception that could not be reconstructed locally.
+
+    Holds the remote traceback so the message is still readable. Alias kept for
+    parity with the runner-protocol wording (``bladekit.RemoteError``).
+    """
+
+    def __init__(
+        self,
+        exc_type: str,
+        message: str,
+        remote_traceback: str,
+    ) -> None:
+        self.exc_type = exc_type
+        self.remote_message = message
+        self.remote_traceback = remote_traceback
+        super().__init__(self._format())
+
+    def _format(self) -> str:
+        return (
+            f"remote function raised {self.exc_type}: {self.remote_message}\n\n"
+            f"--- remote traceback ---\n{self.remote_traceback}"
+        )
+
+
+# Task requirement names the user-exception wrapper ``RemoteFunctionError``.
+# It is the generic carrier used when the original exception cannot be
+# re-raised (pickled is null or unpickling failed).
+RemoteFunctionError = RemoteError
+
+
+# --- image builds -----------------------------------------------------------
+
+
+class ImageBuildError(FabricError):
+    """A ``bladekit.Image`` build did not reach ``succeeded``.
+
+    Actionable: points the user at ``blade image logs <build_id>``. ``code`` is
+    the build's own error code (IMAGE_BUILD_FAILED / IMAGE_BUILD_TIMEOUT /
+    IMAGE_PUSH_FAILED / INTERNAL — conventions.md §2, extensible).
+    """
+
+    def __init__(
+        self,
+        build_id: str,
+        code: str,
+        message: str,
+        *,
+        definition_hash: Optional[str] = None,
+    ) -> None:
+        self.build_id = build_id
+        self.code = code
+        self.build_message = message
+        self.definition_hash = definition_hash
+        super().__init__(
+            f"image build {build_id} failed ({code}): {message}\n"
+            f"  see build logs with: blade image logs {build_id}"
+        )
+
+
+# --- volumes ----------------------------------------------------------------
+
+
+class VolumeError(FabricError):
+    """A volume operation is invalid (e.g. ``commit()`` on an unmounted volume)."""
+
+
+class VolumeCommitError(FabricError):
+    """``volume.commit()`` failed or was refused.
+
+    ``code`` mirrors the server (VOLUME_COMMIT_FAILED, VOLUME_READ_ONLY,
+    VOLUME_COMMIT_IN_PROGRESS, NODE_LOST, RUNNER_CRASHED, …). Never fatal to the
+    container — the caller may retry (runner-protocol.md §11).
+    """
+
+    def __init__(self, code: str, message: str) -> None:
+        self.code = code
+        super().__init__(f"[{code}] {message}")
+
+
+class PreemptedError(RunError):
+    """PREEMPTED — the run/call was preempted by a higher-priority job.
+
+    For map workers: the input is automatically re-queued by the server and
+    will be retried without consuming the retries budget.
+    For simple runs: the run is terminal (cancelled). Re-run it or use map()
+    for automatic re-queue on preemption.
+    """
+
+    code = "PREEMPTED"
+
+
+class InputTimeoutError(RunError):
+    """INPUT_TIMEOUT — an input exceeded its per-input timeout.
+
+    Distinct from TIMEOUT (run global). Raised when the map() call had
+    ``input_timeout_seconds`` set and a single input took too long.
+    Increase ``input_timeout_seconds=`` or optimize the function.
+    """
+
+    code = "INPUT_TIMEOUT"
+
+
+class RetriesExhaustedError(RunError):
+    """RETRIES_EXHAUSTED — an input exhausted all retry attempts.
+
+    The last error code is in ``details["last_error_code"]``. To avoid this,
+    make your function idempotent and increase ``retries=`` on the map call.
+    """
+
+    code = "RETRIES_EXHAUSTED"
+
+
+class GPUQuotaExceededError(APIError):
+    """GPU_QUOTA_EXCEEDED — the namespace quota would be exceeded (strict mode).
+
+    Only raised when the namespace is configured with ``quota_mode=enforce``.
+    In the default ``quota_mode=queue``, the run is queued instead and a
+    ``GPU_QUOTA_REACHED`` warning is emitted.
+    """
+
+
+class OrgGPUQuotaExceededError(APIError):
+    """ORG_GPU_QUOTA_EXCEEDED — the organization's concurrent-GPU quota would
+    be exceeded (strict mode).
+
+    The org tier of the GPU quota: ``details`` carries ``org_id``,
+    ``gpu_type`` (``'*'`` = the total cap across every type), ``limit``,
+    ``in_use`` and ``requested``. Only raised when the org is configured with
+    ``quota_mode=enforce``; in the default ``quota_mode=queue`` the work is
+    queued instead and an ``ORG_GPU_QUOTA_REACHED`` warning is emitted.
+    """
+
+
+# --- sandbox errors ---------------------------------------------------------
+
+
+class SandboxNotRunningError(APIError):
+    """SANDBOX_NOT_RUNNING — sandbox is not in the ``running`` state (409).
+
+    Raised when exec/put_file/get_file/ls/tunnel is called on a sandbox that is
+    creating, terminating, terminated, or failed.
+    ``details["state"]`` carries the current state.
+    """
+
+
+class SandboxExecLimitError(APIError):
+    """SANDBOX_EXEC_LIMIT — too many concurrent execs in this sandbox (429).
+
+    The sandbox has reached ``max_concurrent_execs`` (16 by default).
+    Wait for an exec to finish before starting a new one.
+    """
+
+
+class SandboxFileNotFoundError(NotFoundError):
+    """SANDBOX_FILE_NOT_FOUND — path does not exist in the sandbox rootfs (404)."""
+
+
+class SandboxFileTooLargeError(PayloadTooLargeError):
+    """SANDBOX_FILE_TOO_LARGE — file exceeds the 5 GiB per-file limit (413)."""
+
+
+class SandboxPathInvalidError(InvalidArgumentError):
+    """SANDBOX_PATH_INVALID — path is non-absolute, contains '..', or a symlink
+    hostile resolution would exit the rootfs (400).
+
+    Always use absolute paths; never construct paths from untrusted input.
+    """
+
+
+class SandboxDiskQuotaExceededError(PayloadTooLargeError):
+    """SANDBOX_DISK_QUOTA_EXCEEDED — write would exceed the sandbox's cumulative
+    disk quota (413). Distinct from SANDBOX_FILE_TOO_LARGE (per-file limit).
+
+    Remove files you no longer need, or reduce the amount of data written.
+    """
+
+
+class SandboxPortInUseError(ConflictError):
+    """SANDBOX_PORT_IN_USE — another tunnel already exposes this port (409).
+
+    Re-POST the same port returns the existing tunnel (idempotent); this error
+    only fires when the port is in use by a *different* tunnel object.
+    """
+
+
+class RuntimeUnavailableWarning(FabricError):
+    """RUNTIME_UNAVAILABLE — no schedulable node announces the requested runtime.
+
+    Not raised as an exception; surfaced as a warning in the Sandbox.create()
+    response. The sandbox stays ``creating`` until a node with the runtime
+    appears. This is the safe default: a ``runsc`` sandbox on a runc-only fleet
+    waits rather than falling back to runc silently.
+    """
+
+
+class SandboxTerminatedError(FabricError):
+    """SANDBOX_TERMINATED — the sandbox died while an exec was in flight.
+
+    Raised when a WebSocket exec stream receives an ``error`` frame with code
+    ``SANDBOX_TERMINATED`` (timeout, terminate(), preemption, node lost, OOM).
+    The sandbox's exec is in state ``terminated`` (no exit_code).
+    """
+
+    def __init__(self, sandbox_id: str, code: str, message: str) -> None:
+        self.sandbox_id = sandbox_id
+        self.code = code
+        super().__init__(
+            f"sandbox {sandbox_id} terminated during exec ({code}): {message}\n"
+            "The sandbox can no longer accept exec calls. Create a new sandbox."
+        )
+
+
+class SandboxFailedError(FabricError):
+    """The sandbox transitioned to ``failed`` before becoming running.
+
+    ``code`` is the infrastructure error code (IMAGE_PULL_FAILED, CONTAINER_FAILED,
+    GPU_QUOTA_EXCEEDED, INTERNAL, …). These reuse the run error registry
+    (sandbox-protocol.md §7.1 / conventions.md §2).
+    """
+
+    def __init__(self, sandbox_id: str, code: str, message: str) -> None:
+        self.sandbox_id = sandbox_id
+        self.code = code
+        super().__init__(
+            f"sandbox {sandbox_id} failed ({code}): {message}"
+        )
+
+
+# Map RunError.code -> SDK exception class for friendly, actionable messages.
+RUN_ERROR_CLASSES = {
+    "NODE_LOST": NodeLostError,
+    "CONTAINER_FAILED": ContainerFailedError,
+    "IMAGE_PULL_FAILED": ImagePullFailedError,
+    "RUNNER_CRASHED": RunnerCrashedError,
+    "RUNNER_INTERNAL": RunnerCrashedError,
+    "BUNDLE_FETCH_FAILED": BundleFetchFailedError,
+    "PYTHON_VERSION_MISMATCH": VersionMismatchError,
+    "CLOUDPICKLE_VERSION_MISMATCH": VersionMismatchError,
+    "TIMEOUT": TimeoutError_,
+    "PREEMPTED": PreemptedError,
+    "INPUT_TIMEOUT": InputTimeoutError,
+    "RETRIES_EXHAUSTED": RetriesExhaustedError,
+}
+
+
+def run_error_from_payload(
+    payload: Mapping[str, Any], *, run_id: Optional[str] = None
+) -> RunError:
+    """Build the most specific RunError subclass for an ``error`` object."""
+    code = str(payload.get("code", "INTERNAL"))
+    message = str(payload.get("message", code))
+    details = payload.get("details") or {}
+    cls = RUN_ERROR_CLASSES.get(code, RunError)
+    actionable = _actionable_hint(code, details)
+    if actionable:
+        message = f"{message}\n{actionable}"
+    return cls(message, code=code, run_id=run_id, details=details)
+
+
+def run_lost_from_payload(
+    payload: Mapping[str, Any], *, run_id: Optional[str] = None
+) -> "RunLostError":
+    """Build a ``RunLostError`` for a run/call that transitioned to ``lost``.
+
+    A ``lost`` outcome is the node monitor declaring the node silent (>60 s); it is
+    always a NODE_LOST-class condition. ``RunLostError`` subclasses ``NodeLostError``
+    so existing ``except NodeLostError`` handlers still catch it, while the more
+    specific type is the one actually raised (it is documented/exported).
+    """
+    message = str(payload.get("message", "run lost"))
+    details = payload.get("details") or {}
+    hint = _actionable_hint("NODE_LOST", details)
+    if hint:
+        message = f"{message}\n{hint}"
+    return RunLostError(
+        message, code=str(payload.get("code", "NODE_LOST")), run_id=run_id, details=details
+    )
+
+
+def _actionable_hint(code: str, details: Mapping[str, Any]) -> str:
+    """Return a short, actionable hint for known infra error codes."""
+    if code == "PYTHON_VERSION_MISMATCH":
+        exp = details.get("expected")
+        act = details.get("actual")
+        hint = details.get("image_ref_hint")
+        base = (
+            f"The remote Python ({act}) does not match the one declared by the "
+            f"client ({exp})."
+        )
+        if hint:
+            base += f" {hint}"
+        else:
+            base += " Pick a catalogue image whose Python matches your client."
+        return base
+    if code == "CLOUDPICKLE_VERSION_MISMATCH":
+        return (
+            "Client and remote cloudpickle major.minor differ. Pin the same "
+            "cloudpickle version locally (pip install 'cloudpickle==<remote>')."
+        )
+    if code == "NODE_LOST":
+        return (
+            "The GPU node went silent for more than 60s; the run was lost. "
+            "This is usually transient — re-run it."
+        )
+    if code == "CONTAINER_FAILED":
+        return (
+            "The container never started or vanished. Check the image ref and "
+            "the run logs for an interpreter/startup error."
+        )
+    if code == "IMAGE_PULL_FAILED":
+        return (
+            "The node could not pull the image. Check the image alias/registry "
+            "ref and that it is reachable from the fleet."
+        )
+    if code == "BUNDLE_FETCH_FAILED":
+        return "The runner could not download/verify your code bundle. Re-run to re-upload."
+    if code == "TIMEOUT":
+        return "The run exceeded its timeout. Increase timeout= on @app.function."
+    if code == "PREEMPTED":
+        return (
+            "The run/worker was preempted by a higher-priority job. "
+            "For map() batches the input is automatically re-queued. "
+            "For simple runs: re-run with a higher priority= or wait for the "
+            "high-priority job to finish."
+        )
+    if code == "INPUT_TIMEOUT":
+        return (
+            "An input exceeded its per-input timeout. Increase input_timeout_seconds= "
+            "on the map() call, or optimize the function."
+        )
+    if code == "RETRIES_EXHAUSTED":
+        last = details.get("last_error_code", "")
+        base = "The input exhausted all retry attempts."
+        if last:
+            base += f" Last infra error: {last}."
+        base += " Increase retries= or fix the underlying error."
+        return base
+    if code == "GPU_QUOTA_EXCEEDED":
+        ns = details.get("namespace", "")
+        gpu_type = details.get("gpu_type", "")
+        hint = f"Namespace {ns!r} quota for {gpu_type!r} is full (quota_mode=enforce)."
+        hint += " Wait for quota to free up or ask an admin to raise the limit."
+        return hint
+    if code == "ORG_GPU_QUOTA_EXCEEDED":
+        gpu_type = details.get("gpu_type", "")
+        in_use = details.get("in_use", "?")
+        limit = details.get("limit", "?")
+        scope = "GPUs" if gpu_type in ("", "*") else f"GPUs of type {gpu_type!r}"
+        hint = f"Your organization is using {in_use}/{limit} {scope}."
+        hint += " Wait for a run to finish or ask an admin to raise the org limit."
+        return hint
+    return ""