freesolo-flash-dev 0.2.25__py3-none-any.whl

flash/providers/hyperstack/jobs/builders.py

@@ -0,0 +1,122 @@
+"""Pure, monkeypatch-free building blocks for the Hyperstack run lifecycle.
+
+The Hyperstack-specific leaf of ``flash.providers.hyperstack.jobs``: the normalized dataclasses
+(``HyperstackInstance``, ``HyperstackJobHandle``) and the image accessor. The cross-provider
+pieces (sweep label, bootstrap payload, cloud-init ``user_data``) are shared with Lambda in
+``flash.providers._instance`` and re-exported here.
+
+This module MUST NOT import the ``jobs`` package ``__init__`` (it is imported BY it).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from flash.providers._instance import (
+    build_payload as _shared_build_payload,
+)
+from flash.providers._instance import (
+    build_user_data as _shared_build_user_data,
+)
+from flash.providers._instance import (
+    instance_label,
+    run_label_prefix,
+)
+
+__all__ = [
+    "HyperstackInstance",
+    "HyperstackJobHandle",
+    "build_payload",
+    "build_user_data",
+    "hyperstack_image",
+    "instance_label",
+    "run_label_prefix",
+]
+
+
+@dataclass(frozen=True)
+class HyperstackInstance:
+    """A launchable (region, flavor, $/hr) for a managed GPU class (the Hyperstack analog of a
+    Lambda instance candidate)."""
+
+    gpu: str  # canonical class name (GPU_INFO key)
+    flavor: str  # Hyperstack flavor name (e.g. "n3-L40x1")
+    region: str
+    environment: str  # default-<region>
+    vram_gb: int
+    price_usd_hr: float
+
+
+@dataclass
+class HyperstackJobHandle:
+    """Persisted in RunStatus.remote so any process can reattach/cancel (cf. base.JobHandle)."""
+
+    vm_id: str
+    flavor: str
+    region: str
+    name: str  # the sweep-matchable VM name (run-derived; see ``instance_label``)
+    gpu: str
+    hourly_usd: float
+    attempt: int
+    started_ts: float
+
+    def to_dict(self) -> dict:
+        return {
+            "provider": "hyperstack",
+            "vm_id": self.vm_id,
+            "flavor": self.flavor,
+            "region": self.region,
+            "name": self.name,
+            "gpu": self.gpu,
+            "hourly_usd": self.hourly_usd,
+            "attempt": self.attempt,
+            "started_ts": self.started_ts,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> HyperstackJobHandle:
+        return cls(
+            vm_id=str(d["vm_id"]),
+            flavor=str(d.get("flavor") or ""),
+            region=str(d.get("region") or ""),
+            name=str(d.get("name") or ""),
+            gpu=str(d.get("gpu") or ""),
+            hourly_usd=float(d.get("hourly_usd") or 0),
+            attempt=int(d.get("attempt") or 0),
+            started_ts=float(d.get("started_ts") or 0),
+        )
+
+
+def hyperstack_image(gpu: str | None = None) -> str:
+    """Docker image the cloud-init runs on the Hyperstack host: the prebuilt, PUBLIC ``WORKER_IMAGE``
+    (the byte-identical training stack RunPod bakes). ``FLASH_WORKER_IMAGE`` overrides it; when the
+    operator opts into per-SM warmed images (``FLASH_WORKER_IMAGE_PER_SM`` /
+    ``FLASH_WORKER_IMAGE_TEMPLATE``), the GPU class selects the matching ``-smXX`` tag so the worker's
+    baked kernel cache matches the rented GPU's arch (the same selector RunPod uses). NB: this is the
+    *container* image; the Hyperstack VM *boot* image (Docker-preinstalled Ubuntu/CUDA) is chosen
+    separately in ``api.docker_image_for_region``."""
+    from flash.providers.runpod.train import WORKER_IMAGE, worker_image_for_gpu
+
+    # allow_default=True -> always a concrete image to docker-pull (override / per-sm tag / base).
+    return worker_image_for_gpu(gpu, allow_default=True) or WORKER_IMAGE
+
+
+def build_payload(
+    spec, seed: int, attempt: int, runtime_secrets: dict | None = None,
+    cache_host_mount: str | None = None, cache_block_device: bool = False,
+    mode: str | None = None, models: list | None = None,
+) -> dict:
+    """The Hyperstack bootstrap payload (shared builder, arm='hyperstack'). ``cache_host_mount`` (the
+    host path the attached block volume is formatted+mounted at) points HF_HOME at it;
+    ``cache_block_device`` enables the cloud-init wait-for-device/format/mount preamble.
+    ``mode='preload'`` + ``models`` makes it a download-only warm payload (no worker)."""
+    return _shared_build_payload(
+        spec, seed, attempt, arm="hyperstack", runtime_secrets=runtime_secrets,
+        cache_host_mount=cache_host_mount, cache_block_device=cache_block_device,
+        mode=mode, models=models,
+    )
+
+
+def build_user_data(payload: dict, *, gpu: str | None = None) -> str:
+    """The Hyperstack cloud-init user_data (shared builder, runs the worker WORKER_IMAGE)."""
+    return _shared_build_user_data(payload, image=hyperstack_image(gpu))

flash/providers/hyperstack/preflight.py

@@ -0,0 +1,23 @@
+"""Fail-fast credential checks for the Hyperstack substrate (operator-side).
+
+Mirrors ``providers/lambdalabs/preflight.py``. Hyperstack is OPT-IN (the allocator only reaches for
+it when ``HYPERSTACK_API_KEY`` is set), so the only Hyperstack-specific requirement is the key;
+HF_TOKEN is the shared run requirement checked once centrally.
+"""
+
+from __future__ import annotations
+
+from flash.providers.hyperstack.auth import load_api_key
+
+
+def missing_credentials(require_hf: bool = True) -> list[str]:
+    """Hyperstack-related operator config that is missing (empty list == ready).
+
+    ``require_hf`` is accepted only for signature parity with the RunPod check and is ignored.
+    """
+    problems: list[str] = []
+    if not load_api_key():
+        problems.append(
+            "  - HYPERSTACK_API_KEY: the operator's Hyperstack API key (for the hyperstack provider)"
+        )
+    return problems

flash/providers/hyperstack/pricing.py

@@ -0,0 +1,26 @@
+"""Hyperstack $/hr: static list price per class (the flavors API carries no price field).
+
+Unlike Lambda's ``/instance-types`` (which carries live prices), Hyperstack's ``/core/flavors``
+exposes capacity (``stock_available``) but not price, so rates come from the published list-price
+snapshot below. Offline-safe and stable (Hyperstack prices are fixed list rates, not an auction).
+"""
+
+from __future__ import annotations
+
+# Hyperstack list prices (snapshot 2026-06-25, hyperstack.cloud/gpu-pricing), per single GPU.
+_STATIC_RATES: dict[str, float] = {
+    "RTX A6000": 0.50,
+    "L40": 1.00,
+    "A100 PCIe": 1.35,
+    "H100": 1.90,
+    "RTX Pro 6000": 1.80,
+}
+
+
+def hourly_rate(gpu_name: str) -> float:
+    """$/hr for one friendly GPU name on Hyperstack (static list price)."""
+    from flash.providers.base import GPU_INFO, canonical_gpu
+
+    name = canonical_gpu(gpu_name)
+    # Prefer the Hyperstack snapshot; fall back to the class nominal rate (keeps the call total).
+    return _STATIC_RATES.get(name) or GPU_INFO[name].hourly_usd

flash/providers/hyperstack/train.py

@@ -0,0 +1,25 @@
+"""Hyperstack train submission: build the VM payload + submit a run.
+
+The worker stack/env is substrate-neutral, so the per-run worker env and dependency resolution are
+shared with RunPod; this module owns the Hyperstack-specific submission entrypoint. Provisioning,
+polling, and teardown live in ``providers/hyperstack/jobs``.
+"""
+
+from __future__ import annotations
+
+from flash.providers.hyperstack.jobs import build_payload, submit_run_hyperstack
+from flash.providers.runpod.train import (
+    WORKER_DEPS,
+    WORKER_SYSTEM_DEPS,
+    build_worker_env,
+    resolve_worker_deps,
+)
+
+__all__ = [
+    "WORKER_DEPS",
+    "WORKER_SYSTEM_DEPS",
+    "build_payload",
+    "build_worker_env",
+    "resolve_worker_deps",
+    "submit_run_hyperstack",
+]

flash/providers/lambdalabs/__init__.py

@@ -0,0 +1,139 @@
+"""Lambda Cloud provider: single-GPU instances bootstrapped via cloud-init (the instance-based
+complement to RunPod's serverless Flash endpoints).
+
+Fine-tuning runs on a Lambda Cloud GPU instance launched by Flash. The instance's cloud-init
+``user_data`` runs the prebuilt, PUBLIC ``WORKER_IMAGE`` via Docker (the byte-identical training
+stack RunPod bakes), which executes ``flash.engine.worker`` on the GPU; completion is detected
+purely from the worker's HF artifacts (no inbound network, no serverless queue). It implements the
+SAME ``base.Provider`` interface as RunPod, so the orchestrator/allocator treat the two
+interchangeably.
+
+``PROVIDER`` is the ``base.Provider`` implementation the registry hands out.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from flash.providers.base import GpuClass, JobHandle, PollResult, Provider
+
+
+class LambdaProvider:
+    """``base.Provider`` for the Lambda Cloud substrate."""
+
+    name = "lambda"
+
+    def is_configured(self) -> bool:
+        from flash.providers.lambdalabs.auth import load_api_key
+
+        # Lambda is an opt-in instance substrate: it is available only when its operator key is
+        # present. Without LAMBDA_API_KEY (tests / CI / RunPod-only operators) allocation degrades
+        # deterministically to RunPod's catalog — exactly the prior RunPod-only behavior.
+        return load_api_key() is not None
+
+    def preflight(self, require_hf: bool = True) -> list[str]:
+        from flash.providers.lambdalabs.preflight import missing_credentials
+
+        return missing_credentials(require_hf=require_hf)
+
+    def gpu_classes(self) -> list[GpuClass]:
+        from flash.providers.lambdalabs.gpus import gpu_classes
+
+        return gpu_classes()
+
+    def hourly_rate(self, gpu: str) -> float:
+        from flash.providers.lambdalabs.pricing import hourly_rate
+
+        return hourly_rate(gpu)
+
+    def submit_run(
+        self,
+        spec,
+        seed: int,
+        *,
+        log: Any = None,
+        on_handle: Any = None,
+        attempt: int = 0,
+        runtime_secrets: dict[str, str] | None = None,
+        on_last_gpu: bool = False,
+    ) -> PollResult:
+        from flash.providers.lambdalabs.jobs import submit_run_lambda
+
+        return submit_run_lambda(
+            spec,
+            seed,
+            log=log,
+            on_handle=on_handle,
+            attempt=attempt,
+            runtime_secrets=runtime_secrets,
+            on_last_gpu=on_last_gpu,
+        )
+
+    def poll(self, handle: JobHandle, spec, seed: int, *, log: Any = None) -> PollResult:
+        import contextlib
+
+        from flash.providers.lambdalabs import api as lambda_api
+        from flash.providers.lambdalabs.jobs import (
+            PROVISION_GRACE_S,
+            LambdaJobHandle,
+            poll_lambda_job,
+        )
+        from flash.providers.runpod.jobs import make_hf_heartbeat_reader
+
+        hf_repo = spec.train.hf_repo
+        prefix = f"{spec.phase}/{spec.run_id}/seed{seed}"
+        reader = make_hf_heartbeat_reader(hf_repo, prefix) if hf_repo else None
+        lh = LambdaJobHandle.from_dict(handle.to_dict())
+        if log is not None:
+            print(f"attaching: lambda instance={lh.instance_id}", file=log, flush=True)
+        # The wall-cap deadline counts from the instance's LAUNCH, not from this reattach — Lambda
+        # has no server-side execution timeout, so resetting it on every recovery would let a
+        # control-plane restart extend the billable window unbounded. The poll loop already anchors
+        # its deadline check to ``handle.started_ts`` (start = launch), so we pass the FULL
+        # launch-relative budget here; pre-subtracting elapsed too would double-count and tear down
+        # a still-valid instance the moment a recovered run is past half its window.
+        deadline = max(60.0, int(spec.gpu.max_wall_seconds) + PROVISION_GRACE_S)
+        try:
+            return poll_lambda_job(lh, spec, seed, log=log, heartbeat_reader=reader, deadline_s=deadline)
+        finally:
+            # Recovery (attach_run) has no submit_run_lambda teardown ``finally``; terminate the
+            # reattached instance here so a finished/abandoned recovered seed stops billing
+            # immediately instead of idling until the whole run ends.
+            with contextlib.suppress(Exception):
+                lambda_api.terminate_instances([lh.instance_id])
+
+    def cancel(self, handle: JobHandle) -> None:
+        # Terminating the instance both stops the job and tears down the (only) billable resource —
+        # Lambda has no separate "cancel job" vs "destroy resource".
+        from flash.providers.lambdalabs import api as lambda_api
+
+        d = handle.to_dict()
+        if d.get("instance_id"):
+            lambda_api.terminate_instances([str(d["instance_id"])])
+
+    def destroy(self, handle: JobHandle) -> None:
+        from flash.providers.lambdalabs import api as lambda_api
+
+        d = handle.to_dict()
+        if d.get("instance_id"):
+            lambda_api.terminate_instances([str(d["instance_id"])])
+
+    def gc(self, spec) -> None:
+        from flash.providers.lambdalabs.jobs import terminate_run_instances
+
+        terminate_run_instances(spec.run_id)
+
+    def sweep_orphans(
+        self, active_labels: set[str] | Callable[[], set[str]] | None = None
+    ) -> list[str]:
+        """Lambda crash-recovery sweep (called via the provider object at startup).
+
+        Lambda instance ids are opaque hex STRINGS (the ``base.Provider`` protocol widens the return
+        to ``list[int | str]`` to cover both substrates); the orchestrator only logs/counts them."""
+        from flash.providers.lambdalabs.jobs import sweep_orphans
+
+        return sweep_orphans(active_labels=active_labels)
+
+
+PROVIDER: Provider = LambdaProvider()

flash/providers/lambdalabs/api.py

@@ -0,0 +1,261 @@
+"""Thin Lambda Cloud REST client (no SDK state): instance-types + instance lifecycle.
+
+Mirrors ``providers/runpod/api.py`` / the historical Vast client: stdlib urllib only (via the
+shared ``RestClient``), hardened retries, and nothing persisted locally — a fresh process can
+list/terminate any instance using only the persisted ids + ``LAMBDA_API_KEY``.
+
+Two Lambda-specific quirks the rest of the provider relies on:
+
+* **Cloudflare WAF.** Lambda's API sits behind Cloudflare, which 403s the stdlib default
+  ``Python-urllib/<v>`` User-Agent. The client therefore sends a real UA (``extra_headers``);
+  without it EVERY call fails 403 (verified live).
+* **Non-idempotent launch.** ``POST /instance-operations/launch`` provisions a NEW (billed)
+  instance every time it succeeds, so it is NEVER retried — a blind retry on a timeout where
+  Lambda actually accepted the first request would double-provision. Idempotent calls
+  (instance-types, list, detail, terminate) keep their retries.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from flash._logging import get_logger
+from flash.providers._http import RestClient, is_not_found
+
+logger = get_logger(__name__)
+
+LAMBDA_BASE = "https://cloud.lambdalabs.com/api/v1"
+# A real User-Agent: Lambda's Cloudflare edge rejects the stdlib default with 403 (verified live).
+_USER_AGENT = "flash-lambda/1.0 (+https://freesolo.co)"
+
+
+class LambdaApiError(RuntimeError):
+    pass
+
+
+_CLIENT = RestClient(
+    env_var="LAMBDA_API_KEY",
+    error_cls=LambdaApiError,
+    base_url=LAMBDA_BASE,
+    missing_key_message="LAMBDA_API_KEY not configured on the control-plane host",
+    extra_headers={"User-Agent": _USER_AGENT},
+)
+
+
+def request_with_retries(
+    path: str,
+    method: str = "GET",
+    body: dict | None = None,
+    retries: int = 4,
+    base_delay: float = 2.0,
+) -> Any:
+    """REST call hardened against transient network/5xx blips (jittered backoff)."""
+    return _CLIENT.request_with_retries(
+        path, method=method, body=body, retries=retries, base_delay=base_delay
+    )
+
+
+def _data(out: Any) -> Any:
+    """Unwrap Lambda's ``{"data": ...}`` envelope (every 2xx response uses it)."""
+    if isinstance(out, dict) and "data" in out:
+        return out["data"]
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Instance types + capacity (cached: pricing, the allocator, and the launcher all read this)
+# ---------------------------------------------------------------------------
+_TYPES_TTL_S = 45.0
+_types_cache: dict[str, Any] = {"ts": 0.0, "data": None}
+
+
+def list_instance_types(force: bool = False) -> dict[str, dict]:
+    """Map of ``instance_type_name -> {instance_type, regions_with_capacity_available}``.
+
+    Cached for ``_TYPES_TTL_S`` so pricing + allocation + the launch path share one fetch within an
+    allocation pass. ``force`` bypasses the cache. Raises ``LambdaApiError`` on a hard failure;
+    callers that must degrade gracefully (pricing) catch it.
+    """
+    now = time.time()
+    if not force and _types_cache["data"] is not None and now - _types_cache["ts"] < _TYPES_TTL_S:
+        return _types_cache["data"]
+    out = _data(request_with_retries("/instance-types"))
+    if not isinstance(out, dict):
+        raise LambdaApiError(f"unexpected /instance-types response: {out!r}")
+    _types_cache.update(ts=now, data=out)
+    return out
+
+
+def regions_with_capacity(instance_type: str, force: bool = False) -> list[str]:
+    """Region names that currently have capacity for ``instance_type`` (cheapest source of truth
+    for whether a launch can succeed at all)."""
+    info = list_instance_types(force=force).get(instance_type) or {}
+    return [
+        r.get("name")
+        for r in info.get("regions_with_capacity_available", [])
+        if r.get("name")
+    ]
+
+
+def all_regions(force: bool = False) -> list[str]:
+    """Every Lambda region with at least one capacity-available instance type — the UNION of the
+    ``regions_with_capacity_available`` lists across all instance types (the API has no standalone
+    region list, so this is the only way to enumerate reachable regions). Used by the eager
+    weight-cache provision step to create the ``flash-weights`` filesystem in those regions.
+
+    This is therefore capacity-DEPENDENT: a region that currently advertises ZERO capacity for every
+    instance type won't appear (Lambda only surfaces regions through the per-type capacity list); the
+    launch-time ``ensure_filesystem`` backstop covers any such region the moment a run lands there.
+    Sorted for a stable provision order.
+    """
+    regions: set[str] = set()
+    for info in list_instance_types(force=force).values():
+        for r in (info or {}).get("regions_with_capacity_available", []):
+            if r.get("name"):
+                regions.add(r["name"])
+    return sorted(regions)
+
+
+def instance_type_price_usd_hr(instance_type: str) -> float | None:
+    """Live $/hr for a Lambda instance type (``price_cents_per_hour`` / 100), or None."""
+    info = (list_instance_types().get(instance_type) or {}).get("instance_type") or {}
+    cents = info.get("price_cents_per_hour")
+    return float(cents) / 100.0 if cents else None
+
+
+# ---------------------------------------------------------------------------
+# SSH keys (launch requires exactly one; the box is bootstrapped via user_data, not SSH)
+# ---------------------------------------------------------------------------
+def list_ssh_keys() -> list[dict]:
+    out = _data(request_with_retries("/ssh-keys"))
+    return out if isinstance(out, list) else []
+
+
+# ---------------------------------------------------------------------------
+# Instances
+# ---------------------------------------------------------------------------
+def launch_instance(
+    *,
+    region_name: str,
+    instance_type_name: str,
+    ssh_key_names: list[str],
+    name: str,
+    user_data: str,
+    file_system_names: list[str] | None = None,
+) -> str:
+    """Launch one instance -> its id. Raises ``LambdaApiError`` on rejection (no capacity, etc.).
+
+    NON-IDEMPOTENT (see module docstring): never retried. A transient failure surfaces to the
+    launcher, which walks to the next region/class.
+
+    ``file_system_names`` attaches persistent filesystems (the weight cache) AT LAUNCH — Lambda can
+    only attach at launch, and each must already exist in ``region_name`` (auto-mounted on the host
+    at ``/lambda/nfs/<name>``).
+    """
+    body = {
+        "region_name": region_name,
+        "instance_type_name": instance_type_name,
+        "ssh_key_names": list(ssh_key_names),
+        # ``name`` is bounded <=60 by ``_instance.run_label_prefix`` (NOT truncated here) so the
+        # stored name always equals the prefix ``sweep_orphans`` matches on.
+        "name": name,
+        "quantity": 1,
+        "user_data": user_data,
+    }
+    if file_system_names:
+        body["file_system_names"] = list(file_system_names)
+    out = _data(request_with_retries("/instance-operations/launch", method="POST", body=body, retries=0))
+    ids = out.get("instance_ids") if isinstance(out, dict) else None
+    if not ids:
+        raise LambdaApiError(f"launch({instance_type_name}@{region_name}) returned no instance id: {out}")
+    return str(ids[0])
+
+
+# ---------------------------------------------------------------------------
+# Persistent filesystems (the weight cache). Region-scoped, NFS, multi-attach; auto-mounted on the
+# host at /lambda/nfs/<name>. Created via the Cloud API, attached at launch via file_system_names.
+# ---------------------------------------------------------------------------
+# NB: Lambda's filesystem API paths are ASYMMETRIC and this is intentional/correct, not a typo —
+# verified LIVE against cloud.lambdalabs.com/api/v1 with a real create->ensure->delete probe (the FS
+# was created, reused idempotently, then confirmed deleted, no stranded resources). LIST is the
+# hyphenated GET /file-systems; CREATE/DELETE are the un-hyphenated POST /filesystems and
+# DELETE /filesystems/{id}. Lambda's own surface differs from its other (hyphenated) resources here,
+# so DO NOT "unify" these to /file-systems — that 404s the working create/delete endpoints and
+# silently disables the cache. (Reviewers keep flagging the inconsistency; it's the real API.)
+def list_filesystems() -> list[dict]:
+    """All filesystems on the account: ``[{id, name, mount_point, region:{name}, is_in_use}, ...]``."""
+    out = _data(request_with_retries("/file-systems"))  # LIST: hyphenated (verified live)
+    return out if isinstance(out, list) else []
+
+
+def create_filesystem(name: str, region_name: str) -> dict:
+    """Create filesystem ``name`` in ``region_name`` -> its object (incl. ``mount_point``)."""
+    out = _data(
+        request_with_retries(
+            # CREATE: un-hyphenated /filesystems (NOT /file-systems) — verified live; see note above.
+            "/filesystems", method="POST", body={"name": name, "region": region_name}, retries=2
+        )
+    )
+    return out if isinstance(out, dict) else {}
+
+
+def delete_filesystem(filesystem_id: str) -> bool:
+    """Delete a filesystem by id (best-effort). Returns True if the request didn't raise."""
+    try:
+        # DELETE: un-hyphenated /filesystems/{id} (NOT /file-systems/{id}) — verified live; see note.
+        request_with_retries(f"/filesystems/{filesystem_id}", method="DELETE", retries=2)
+        return True
+    except Exception as exc:
+        logger.warning("lambda delete_filesystem(%s) failed: %s", filesystem_id, exc)
+        return False
+
+
+def ensure_filesystem(name: str, region_name: str) -> str:
+    """Create-if-absent the cache filesystem ``name`` in ``region_name``; return its mount_point
+    (``/lambda/nfs/<name>``). Idempotent: reuses an existing same-name filesystem in that region."""
+    for fs in list_filesystems():
+        if fs.get("name") == name and (fs.get("region") or {}).get("name") == region_name:
+            return fs.get("mount_point") or f"/lambda/nfs/{name}"
+    created = create_filesystem(name, region_name)
+    return created.get("mount_point") or f"/lambda/nfs/{name}"
+
+
+def get_instance(instance_id: str) -> dict | None:
+    """Instance detail dict, or None once it no longer exists (terminated)."""
+    try:
+        out = request_with_retries(f"/instances/{instance_id}")
+    except LambdaApiError as e:
+        if is_not_found(e):
+            return None
+        raise
+    data = _data(out)
+    return data if isinstance(data, dict) else None
+
+
+def list_instances() -> list[dict]:
+    out = _data(request_with_retries("/instances"))
+    return out if isinstance(out, list) else []
+
+
+def terminate_instances(instance_ids: list[str]) -> list[str]:
+    """Terminate (and stop billing for) instances; return the ids that ACTUALLY terminated.
+
+    PER-ID ISOLATED (one POST per id), so a single stale/invalid/race-deleted id can't abort
+    teardown of the rest — this is the crash-backstop path (``sweep_orphans`` /
+    ``terminate_run_instances`` pass many ids at once, where stale ids are common). Lambda's
+    terminate endpoint validates the whole request, so a batch of N ids with one bad id 4xx's and
+    terminates NONE; isolating per id removes that money-leak. Best-effort: never raises."""
+    deleted: list[str] = []
+    for iid in [str(i) for i in instance_ids if i]:
+        try:
+            request_with_retries(
+                "/instance-operations/terminate",
+                method="POST",
+                body={"instance_ids": [iid]},
+                retries=2,
+            )
+            deleted.append(iid)
+        except Exception as exc:
+            logger.warning("lambda terminate(%s) failed: %s", iid, exc)
+    return deleted

flash/providers/lambdalabs/auth.py

@@ -0,0 +1,18 @@
+"""Lambda Cloud credential handling (operator-side), mirroring the RunPod auth module.
+
+The Lambda REST client authenticates via the ``LAMBDA_API_KEY`` environment variable, set by
+the **operator** on the control-plane host. Env-only by design, exactly like ``RUNPOD_API_KEY``:
+it is never written to config files. (Unlike Vast/RunPod, Lambda has no instance-scoped key, so
+the operator key is also NOT shipped to the box — teardown is control-plane-side only.)
+"""
+
+from __future__ import annotations
+
+from .._auth import load_provider_key
+
+_ENV_VAR = "LAMBDA_API_KEY"
+
+
+def load_api_key() -> str | None:
+    """API key from the environment (operator configuration)."""
+    return load_provider_key(_ENV_VAR)

flash/providers/lambdalabs/gpus.py

@@ -0,0 +1,29 @@
+"""Lambda Cloud's GPU classes (its rows of the shared GPU table).
+
+The class table is provider-agnostic and lives in ``providers/base.py``. This module carves out
+Lambda's rows (``gpu_classes()`` == every class with a ``lambda_name``) and owns the
+friendly-name -> Lambda instance-type translation.
+"""
+
+from __future__ import annotations
+
+from flash.providers.base import GpuClass, UnsupportedGpuError, get_gpu_info, providers_for
+
+__all__ = ["gpu_classes", "instance_type_for"]
+
+
+def gpu_classes() -> list[GpuClass]:
+    """The GPU classes Lambda can provision (those with a ``lambda_name``)."""
+    from flash.providers.base import GPU_INFO
+
+    return [g for g in GPU_INFO.values() if g.lambda_name]
+
+
+def instance_type_for(name: str) -> str:
+    """Lambda instance-type name (e.g. 'gpu_1x_a10') for a friendly GPU class name."""
+    info = get_gpu_info(name)
+    if not info.lambda_name:
+        raise UnsupportedGpuError(
+            f"{info.name} is not available on Lambda (providers: {', '.join(providers_for(name))})"
+        )
+    return info.lambda_name