gpuhedge 0.1.0__py3-none-any.whl

gpuhedge/__init__.py

@@ -0,0 +1,32 @@
+"""GPUHedge — speculative execution across serverless GPU clouds.
+
+First valid result wins. Loser cancellation is audited.
+
+The public surface is intentionally small; the benchmark harness lives under
+``gpuhedge.benchmark`` and provider adapters under ``gpuhedge.backends``.
+"""
+
+from __future__ import annotations
+
+from gpuhedge.config import BenchmarkConfig, default_config_path, load_config
+
+
+def __getattr__(name: str):
+    # Lazy: Router pulls in telemetry/policy modules; keep bare import light.
+    if name in ("Router", "RouterOutcome"):
+        from gpuhedge import router
+
+        return getattr(router, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "BenchmarkConfig",
+    "load_config",
+    "default_config_path",
+    "Router",
+    "RouterOutcome",
+]

gpuhedge/__main__.py

@@ -0,0 +1,8 @@
+"""Enable ``python -m gpuhedge`` as an alias for the ``gpuhedge`` console script."""
+
+from __future__ import annotations
+
+from gpuhedge.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

gpuhedge/auth.py

@@ -0,0 +1,100 @@
+"""Provider login verification, reused by ``gpuhedge login-check``.
+
+Each check does a real authenticated round-trip (not just "is a token file
+present"): Modal lists apps, RunPod calls ``get_user``, Cerebrium lists
+projects. Everything is timeboxed and degrades to a clear "not logged in"
+rather than raising, so the CLI can render a status table.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import shutil
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass
+class AuthStatus:
+    provider: str
+    logged_in: bool
+    identity: str | None = None      # profile / user id / project
+    detail: str = ""
+    config_path: str | None = None
+
+    @property
+    def mark(self) -> str:
+        return "OK" if self.logged_in else "NOT LOGGED IN"
+
+
+def _run(cmd: list[str], timeout: float) -> tuple[int, str]:
+    try:
+        proc = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=timeout, check=False
+        )
+        return proc.returncode, (proc.stdout + proc.stderr)
+    except FileNotFoundError:
+        return 127, "command not found"
+    except subprocess.TimeoutExpired:
+        return 124, f"timed out after {timeout}s"
+
+
+def check_modal(timeout: float = 45.0) -> AuthStatus:
+    cfg = pathlib.Path.home() / ".modal.toml"
+    if shutil.which("modal") is None:
+        return AuthStatus("modal", False, detail="modal CLI not installed")
+    code, out = _run(["modal", "profile", "current"], timeout)
+    profile = out.strip().splitlines()[-1].strip() if code == 0 and out.strip() else None
+    if code != 0:
+        return AuthStatus("modal", False, detail=out.strip()[:200], config_path=str(cfg))
+    # Live auth: listing apps requires a valid token.
+    code2, out2 = _run(["modal", "app", "list"], timeout)
+    ok = code2 == 0
+    return AuthStatus(
+        "modal", ok, identity=profile,
+        detail="apps listed" if ok else out2.strip()[:200],
+        config_path=str(cfg),
+    )
+
+
+def check_runpod(timeout: float = 45.0) -> AuthStatus:
+    cfg = pathlib.Path.home() / ".runpod" / "config.toml"
+    try:
+        from gpuhedge.backends.runpod_backend import load_runpod_api_key
+
+        api_key = load_runpod_api_key()
+    except Exception as exc:  # noqa: BLE001
+        return AuthStatus("runpod", False, detail=str(exc)[:200], config_path=str(cfg))
+    try:
+        import runpod
+
+        runpod.api_key = api_key
+        user = runpod.get_user()
+        uid = user.get("id") if isinstance(user, dict) else None
+        return AuthStatus("runpod", True, identity=uid, detail="get_user ok",
+                          config_path=str(cfg))
+    except Exception as exc:  # noqa: BLE001
+        return AuthStatus("runpod", False, detail=str(exc)[:200], config_path=str(cfg))
+
+
+def check_cerebrium(timeout: float = 45.0) -> AuthStatus:
+    cfg = pathlib.Path.home() / ".cerebrium" / "config.yaml"
+    if shutil.which("cerebrium") is None:
+        return AuthStatus("cerebrium", False, detail="cerebrium CLI not installed",
+                          config_path=str(cfg))
+    code, out = _run(["cerebrium", "projects", "list", "--no-color"], timeout)
+    if code != 0:
+        return AuthStatus("cerebrium", False, detail=out.strip()[:200], config_path=str(cfg))
+    # Grab the current project context if available.
+    code2, out2 = _run(["cerebrium", "projects", "current", "--no-color"], timeout)
+    project = None
+    if code2 == 0:
+        for line in out2.splitlines():
+            if "projectId" in line:
+                project = line.split(":", 1)[-1].strip()
+    return AuthStatus("cerebrium", True, identity=project, detail="projects listed",
+                      config_path=str(cfg))
+
+
+def check_all(timeout: float = 45.0) -> list[AuthStatus]:
+    return [check_modal(timeout), check_runpod(timeout), check_cerebrium(timeout)]

gpuhedge/backends/__init__.py

@@ -0,0 +1,66 @@
+"""Provider adapters and the factory that builds one from config.
+
+Adapters are imported lazily so that ``gpuhedge`` (and CI) can import the
+package without every cloud SDK installed — ``pip install gpuhedge[runpod]``
+etc. pulls only what a given provider needs.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from gpuhedge.backends.base import (
+    Backend,
+    BackendError,
+    CancellationEvidence,
+    CancellationReceipt,
+    JobHandle,
+    JobState,
+    LifecycleEvent,
+    NotDeployedError,
+    ProviderResult,
+)
+from gpuhedge.config import Provider
+
+_REGISTRY = {
+    "runpod": ("gpuhedge.backends.runpod_backend", "RunPodBackend"),
+    "modal": ("gpuhedge.backends.modal_backend", "ModalBackend"),
+    "cerebrium": ("gpuhedge.backends.cerebrium_backend", "CerebriumBackend"),
+    "sim": ("gpuhedge.backends.sim_backend", "SimBackend"),
+    "http": ("gpuhedge.backends.http_backend", "HttpBackend"),
+}
+
+
+def build_backend(provider: Provider, request: dict[str, Any]) -> Backend:
+    """Construct the adapter for the provider.
+
+    The adapter is selected by ``extra.adapter`` when present (``sim``,
+    ``http``, or any registry key), else by the provider key itself — so a
+    config can declare arbitrarily-named providers backed by the generic
+    adapters."""
+
+    import importlib
+
+    adapter = provider.extra.get("adapter", provider.key)
+    if adapter not in _REGISTRY:
+        raise BackendError(
+            f"no adapter {adapter!r} for provider {provider.key!r} "
+            f"(known: {sorted(_REGISTRY)})"
+        )
+    module_name, cls_name = _REGISTRY[adapter]
+    module = importlib.import_module(module_name)
+    return getattr(module, cls_name)(provider, request)
+
+
+__all__ = [
+    "Backend",
+    "BackendError",
+    "NotDeployedError",
+    "JobHandle",
+    "JobState",
+    "LifecycleEvent",
+    "ProviderResult",
+    "CancellationEvidence",
+    "CancellationReceipt",
+    "build_backend",
+]

gpuhedge/backends/base.py

@@ -0,0 +1,183 @@
+"""Uniform provider abstraction: submit -> result / status / cancel, plus the
+structured cancellation receipt that is GPUHedge's core differentiator.
+
+Every adapter implements the same small surface so the benchmark controller and
+the (future) production router treat RunPod, Modal, and Cerebrium identically.
+Provider capability differences (e.g. fine-grained lifecycle events) are exposed
+honestly via ``LifecycleEvent`` lists that may be coarse for some providers.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from gpuhedge.config import Provider
+
+
+class JobState(str, Enum):
+    PENDING = "PENDING"
+    QUEUED = "IN_QUEUE"
+    IN_PROGRESS = "IN_PROGRESS"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    CANCELLED = "CANCELLED"
+    TIMEOUT = "TIMEOUT"       # right-censored: exceeded the per-model cap
+    UNKNOWN = "UNKNOWN"
+
+    @property
+    def terminal(self) -> bool:
+        return self in {
+            JobState.COMPLETED,
+            JobState.FAILED,
+            JobState.CANCELLED,
+            JobState.TIMEOUT,
+        }
+
+
+def now_ms() -> float:
+    """Monotonic milliseconds — safe for durations, never wall-clock arithmetic."""
+
+    return time.monotonic() * 1000.0
+
+
+@dataclass
+class LifecycleEvent:
+    """One observed transition, ms relative to submit (round-relative if shared)."""
+
+    t_ms: float
+    stage: str            # submitted|queued|in_progress|model_load|generating|result|...
+    detail: str = ""
+
+
+@dataclass
+class ProviderResult:
+    provider: str
+    state: JobState
+    wall_s: float                                  # submit -> valid result / terminal
+    audio: bytes | None = None                     # decoded WAV bytes (winner only)
+    sample_rate: int | None = None
+    provider_metrics: dict[str, Any] = field(default_factory=dict)  # load_seconds, gpu, ...
+    events: list[LifecycleEvent] = field(default_factory=list)
+    error: str | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.state is JobState.COMPLETED and self.audio is not None
+
+
+class CancellationEvidence(str, Enum):
+    """How much proof the adapter obtained that the loser actually stopped.
+
+    Ordered strongest to weakest. An adapter must EARN a level: acknowledgment
+    of the cancel call and terminal-state confirmation are distinct events,
+    and an unconfirmed cancellation never defaults to success
+    (docs/cancellation-semantics.md).
+    """
+
+    CONFIRMED_TERMINAL = "confirmed_terminal"        # polled to a terminal state
+    PROVIDER_ACK = "provider_ack"                    # cancel call acknowledged only
+    REQUEST_CHANNEL_CLOSED = "request_channel_closed"  # in-flight request broke
+    NO_EVIDENCE = "no_evidence"                      # nothing observable
+
+
+@dataclass
+class CancellationReceipt:
+    """Audit record for a loser cancellation — the evidence trail that
+    separates GPUHedge from an ``asyncio.wait(FIRST_COMPLETED)`` demo
+    (docs/cancellation-semantics.md).
+
+    Conservative by default: ``evidence`` starts at NO_EVIDENCE and ``leaked``
+    starts True. Adapters flip them only when they observe proof."""
+
+    provider: str
+    job_id: str | None
+    was_running: bool                       # cancelled IN_PROGRESS vs merely IN_QUEUE
+    cancel_sent_ms: float
+    cancel_ack_ms: float | None = None      # provider acknowledged the cancel call
+    terminal_ms: float | None = None        # job reached a terminal state on poll
+    terminal_status: str | None = None      # set ONLY from an observed terminal state
+    last_gpu_activity_ms: float | None = None
+    execution_ms_before_cancel: float | None = None
+    estimated_cost_usd: float | None = None
+    reconciled_cost_usd: float | None = None
+    evidence: str = CancellationEvidence.NO_EVIDENCE.value
+    cancel_scope: str = "unknown"           # queued_job | request | container | unknown
+    confirmed_terminal: bool = False        # terminal state observed, not assumed
+    billing_stop_confirmed: bool = False    # provider-reported final billing captured
+    leaked: bool = True                     # True until stopping is evidenced
+    note: str = ""
+
+    def ack_latency_ms(self) -> float | None:
+        if self.cancel_ack_ms is None:
+            return None
+        return self.cancel_ack_ms - self.cancel_sent_ms
+
+    def cancel_to_terminal_ms(self) -> float | None:
+        if self.terminal_ms is None:
+            return None
+        return self.terminal_ms - self.cancel_sent_ms
+
+
+class BackendError(RuntimeError):
+    pass
+
+
+class NotDeployedError(BackendError):
+    """Raised when a provider's endpoint has not been deployed yet
+    (``deployed: false`` in the config — e.g. Cerebrium before Stage 1)."""
+
+
+class JobHandle:
+    """A submitted job on one provider. Adapters subclass and implement the
+    async methods; base tracks submit time so wall/relative timings are uniform."""
+
+    provider: str
+
+    def __init__(self, provider: str, *, submit_ms: float) -> None:
+        self.provider = provider
+        self.key = provider          # adapters build ProviderResult(provider=self.key)
+        self.submit_ms = submit_ms
+        self.events: list[LifecycleEvent] = [LifecycleEvent(0.0, "submitted")]
+
+    def _record(self, stage: str, detail: str = "") -> None:
+        self.events.append(LifecycleEvent(now_ms() - self.submit_ms, stage, detail))
+
+    def job_id(self) -> str | None:  # pragma: no cover - overridden
+        return None
+
+    async def result(self, timeout_s: float) -> ProviderResult:  # pragma: no cover
+        raise NotImplementedError
+
+    async def status(self) -> JobState:  # pragma: no cover
+        raise NotImplementedError
+
+    async def cancel(  # pragma: no cover
+        self, *, reason: str = "lost the race"
+    ) -> CancellationReceipt:
+        raise NotImplementedError
+
+
+class Backend:
+    """Factory + provider-level operations (submit, force cold)."""
+
+    key: str
+
+    def __init__(self, provider: Provider, request: dict[str, Any]) -> None:
+        self.provider = provider
+        self.key = provider.key
+        self.request = request
+
+    def available(self) -> bool:
+        return self.provider.deployed
+
+    async def submit(self) -> JobHandle:  # pragma: no cover - overridden
+        raise NotImplementedError
+
+    async def scale_to_zero(self) -> None:
+        """Best-effort force the endpoint into its normal cold state before a
+        round. Default no-op; adapters override where the platform allows it."""
+
+        return None