fcloud-sdk 0.1.0__py3-none-any.whl

fcloud/_direct_bridge.py

@@ -0,0 +1,1645 @@
+"""Synchronous bridge from the sync Client API to the async DirectClient.
+
+fcloud.Client is synchronous; fcloud.direct.DirectClient is async
+(aiohttp WebSockets). This bridge runs a dedicated asyncio event loop
+in a background thread and marshals sync callers onto it via
+run_coroutine_threadsafe.
+
+Session lifecycle: dispatcher allocates a host and mints a token,
+then the bridge opens a data-plane WS directly to the host for
+execute/spawn/poll/file ops. The dispatcher is always consulted
+for allocation, token refresh, and teardown.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import json
+import signal
+import sys
+import tempfile
+import threading
+import time
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from fcloud import v2_connect, volume_wait
+from fcloud.errors import ClientTooOldError, FcloudError, FcloudRuntimeError
+from fcloud.fileset import UploadOptions, collect_upload_files
+from fcloud.types import ExecResult, Process, ProcessStatus, ProjectInfo, SessionInfo
+from fcloud.version import CLIENT_NAME, VERSION
+
+if TYPE_CHECKING:
+    from fcloud.direct import DirectClient
+
+_TERMINAL_STATUSES = {"closing", "closed", "failed"}
+
+
+def _session_info_from_allocation(
+    allocation: dict[str, Any], project_id: str, host_id: str = "",
+) -> SessionInfo:
+    """SessionInfo sourced from the dispatcher's allocation response —
+    server truth as of the allocation, never a locally-invented status.
+    Fields an older dispatcher doesn't send stay "" (explicitly unknown).
+    project_id is a locally-known fact (the server has no project concept).
+    """
+    return SessionInfo(
+        session_id=str(allocation.get("sid") or ""),
+        project_id=project_id,
+        host_id=host_id or str(allocation.get("hid") or ""),
+        status=str(allocation.get("status") or ""),
+        lifecycle=str(allocation.get("lifecycle") or ""),
+        activity=str(allocation.get("activity") or ""),
+        created_at=str(allocation.get("created_at") or ""),
+        last_active=str(allocation.get("updated_at") or ""),
+        allocated_at=str(allocation.get("allocated_at") or ""),
+        ephemeral=bool(allocation.get("ephemeral")),
+    )
+
+
+def _can_handle_interrupt_on_main_thread() -> bool:
+    """signal.signal only works on the main thread, and only if the
+    signal module exposes SIGINT (it doesn't on some embedded builds).
+    Library callers running execs from worker threads fall back to the
+    older cancel+reap path."""
+    if threading.current_thread() is not threading.main_thread():
+        return False
+    return getattr(signal, "SIGINT", None) is not None
+
+_CLEANUP_TIMEOUT_S = 5.0
+_DEFAULT_EXEC_OUTPUT_LIMIT_BYTES = 64 * 1024
+_OUTPUT_DRAIN_TIMEOUT_S = 2.0
+# After interrupting a foreground exec we kill the remote process and
+# briefly wait for it to actually exit, so the follow-up release sees
+# no running process and tears the ephemeral session down promptly.
+_INTERRUPT_KILL_TIMEOUT_S = 8.0
+_VALID_OUTPUT_RANGES = {"all", "head", "middle", "tail"}
+_TRANSIENT_POLL_RETRY_DELAY_S = 1.0
+# How long the foreground poll keeps retrying a failing host before it
+# declares the host LOST (vs a momentary blip). Past this, the bridge
+# returns status="host_lost" so the CLI can ride a spot migration instead
+# of either hanging to the run deadline or falsely reporting exit 0.
+_HOST_LOSS_GRACE_S = 6.0
+
+# Files at or below this size ride inline over the WS in batched
+# write_files frames. Larger files are uploaded straight to S3 via a
+# dispatcher-minted pre-signed URL and pulled by the host (sync_pull).
+# The pre-signed path is a single PUT, so it inherits S3's 5 GiB
+# per-object PUT ceiling (enforced by the dispatcher at presign time).
+_INLINE_FILE_LIMIT = 5 * 1024 * 1024
+# Keep each inline frame under ~8 MiB raw so its base64 fits the host's
+# 64 MiB WS read-limit with headroom.
+_INLINE_BATCH_BYTES = 8 * 1024 * 1024
+# (connect, read) timeouts for direct S3 PUT/GET. The long read budget
+# covers multi-hundred-MB transfers.
+_BLOB_HTTP_TIMEOUT = (30, 3600)
+# Pre-signed URLs expire (dispatcher TTL: 15 min) and the dispatcher caps
+# one presign request at 1000 hashes — so URLs are minted in small
+# per-chunk batches right before each chunk's PUTs. One up-front batch
+# for the whole tree 403'd on slow uploads.
+_PRESIGN_CHUNK = 100
+# Keepalive cadence during the (otherwise WS-silent) S3 PUT loop. Keeps
+# the host's transfer lease fresh and proves the WS is alive.
+_UPLOAD_KEEPALIVE_S = 60.0
+
+
+@dataclass
+class _CapturedOutput:
+    text: str
+    total_bytes: int
+    retained_bytes: int
+    truncated: bool
+    range_name: str
+
+
+class _OutputCapture:
+    """Spool command output so completed exec results can be sliced safely."""
+
+    def __init__(self) -> None:
+        self._file = tempfile.TemporaryFile()
+        self._total_bytes = 0
+
+    def write(self, text: str) -> None:
+        data = text.encode("utf-8", errors="replace")
+        self._total_bytes += len(data)
+        self._file.write(data)
+
+    def result(
+        self,
+        limit_bytes: int | None,
+        range_name: str,
+    ) -> _CapturedOutput:
+        if range_name not in _VALID_OUTPUT_RANGES:
+            range_name = "tail"
+        if limit_bytes is None or range_name == "all":
+            return self._read_range(0, self._total_bytes, False, "all")
+        if self._total_bytes <= limit_bytes:
+            return self._read_range(0, self._total_bytes, False, "all")
+        start = self._range_start(limit_bytes, range_name)
+        return self._read_range(start, limit_bytes, True, range_name)
+
+    def close(self) -> None:
+        self._file.close()
+
+    def _range_start(self, limit_bytes: int, range_name: str) -> int:
+        if range_name == "head":
+            return 0
+        if range_name == "middle":
+            return max(0, (self._total_bytes - limit_bytes) // 2)
+        return max(0, self._total_bytes - limit_bytes)
+
+    def _read_range(
+        self,
+        start: int,
+        size: int,
+        truncated: bool,
+        range_name: str,
+    ) -> _CapturedOutput:
+        self._file.flush()
+        self._file.seek(start)
+        data = self._file.read(size)
+        return _CapturedOutput(
+            text=data.decode("utf-8", errors="replace"),
+            total_bytes=self._total_bytes,
+            retained_bytes=len(data),
+            truncated=truncated,
+            range_name=range_name,
+        )
+
+
+def _persistent_session_status_printer(frame: dict[str, Any]) -> None:
+    status = str(frame.get("status") or "")
+    event = frame.get("event", "?")
+    detail = str(frame.get("detail") or "")
+    if event == "environment_reset":
+        # The host reset everything outside /workspace at resume (image
+        # re-baseline or an incomplete close sync) — without this banner
+        # the user's first symptom is broken imports with a clean
+        # `pip list`.
+        print(
+            f"  ⟵ environment reset — {detail}",
+            file=sys.stderr, flush=True,
+        )
+        return
+    if event == "workload_oom":
+        # The kernel OOM-killed the sandbox, taking the run with it. The
+        # session itself was rebuilt and still holds the user's files, so the
+        # useful thing to say is "retry", not "it broke".
+        print(f"  ⟵ out of memory — {detail}", file=sys.stderr, flush=True)
+        return
+    if event == "memory_pressure":
+        # Said BEFORE the kill, while the user can still act on it.
+        print(f"  ⟵ memory pressure — {detail}", file=sys.stderr, flush=True)
+        return
+    if "host_interrupted" in detail or event == "host_interrupted":
+        sid = str(frame.get("sid") or "")
+        hint = f" sid={sid}" if sid else ""
+        print(
+            f"  \u27f5 host interrupted \u2014 files preserved. "
+            f"It comes back online automatically next time you use it.{hint}",
+            file=sys.stderr, flush=True,
+        )
+        return
+    if status not in _TERMINAL_STATUSES:
+        return
+    print(
+        f"  \u27f5 session_status: {status} ({event}) {detail}".rstrip(),
+        file=sys.stderr,
+        flush=True,
+    )
+
+
+def _workspace_relative_path(path: str) -> str:
+    """Convert CLI /workspace paths to direct host workspace-relative paths."""
+    normalized = path.replace("\\", "/")
+    if normalized == "/workspace":
+        return ""
+    if normalized.startswith("/workspace/"):
+        return normalized.removeprefix("/workspace/")
+    return normalized
+
+
+def _hash_file(path) -> str:
+    """SHA-256 hex of a file, streamed so memory stays flat for large
+    files. Matches the host/dispatcher content-addressed blob key."""
+    import hashlib
+
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        while chunk := fh.read(1024 * 1024):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _is_connection_error(exc: BaseException) -> bool:
+    """True for errors meaning the WS/TCP connection died (safe to
+    reconnect and retry an idempotent request), as opposed to the host
+    rejecting the request. "closing transport" is aiohttp's message when
+    a write races a dropped connection."""
+    if isinstance(exc, ConnectionError):
+        return True
+    try:
+        import aiohttp
+    except ImportError:
+        pass
+    else:
+        if isinstance(exc, aiohttp.ClientConnectionError):
+            return True
+    return "closing transport" in str(exc).lower()
+
+
+def _s3_route_note(large: list[tuple[str, Path, int]]) -> str:
+    """Name the files that took the presigned-S3 route, for an error that
+    would otherwise blame the whole command."""
+    if not large:
+        return "no files (the S3 path was not used)"
+    shown = ", ".join(
+        f"{dest} ({sz / 1e6:.1f} MB)" for dest, _p, sz in large[:5]
+    )
+    if len(large) > 5:
+        shown += f", +{len(large) - 5} more"
+    return (
+        f"{len(large)} file(s) over "
+        f"{_INLINE_FILE_LIMIT // (1 << 20)} MiB: {shown}"
+    )
+
+
+def _raise_if_client_too_old(status_code: int, data: object) -> None:
+    """Turn the dispatcher's version-floor rejection into the typed error.
+
+    The blob presign calls below talk to the dispatcher directly instead of
+    going through Client._get/_post, so they need the same translation —
+    and this is the exact path where a too-old client used to fail as an S3
+    ``SignatureDoesNotMatch`` instead of as "upgrade your client"."""
+    if not isinstance(data, dict):
+        return
+    if data.get("code") != "client_too_old" and status_code != 426:
+        return
+    raise ClientTooOldError(
+        str(data.get("error") or "client too old for this deployment"),
+        status_code, str(data.get("min_client_version") or ""),
+    )
+
+
+def _dispatcher_headers(api_key: str) -> dict[str, str]:
+    """JSON + auth + the client-version handshake (see fcloud.version)."""
+    headers = {
+        "Content-Type": "application/json",
+        "X-Fcloud-Client": CLIENT_NAME,
+        "X-Fcloud-Client-Version": VERSION,
+        "User-Agent": f"{CLIENT_NAME}/{VERSION}",
+    }
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    return headers
+
+
+def _blob_put(url: str, path, headers: dict[str, str] | None = None) -> None:
+    """Stream a file to a pre-signed S3 PUT URL.
+
+    ``headers`` are the ones the dispatcher says the signature covers —
+    today the ``x-amz-checksum-sha256`` binding that ties the object to its
+    content hash. Sending anything else would break the signature, so only
+    the dispatcher-supplied set is forwarded, exactly as given.
+    """
+    import requests as _requests
+
+    with open(path, "rb") as fh:
+        resp = _requests.put(
+            url, data=fh, headers=headers or None, timeout=_BLOB_HTTP_TIMEOUT,
+        )
+    if resp.status_code >= 300:
+        raise FcloudRuntimeError(
+            f"S3 upload failed ({resp.status_code}): {resp.text[:200]}"
+        )
+
+
+@dataclass
+class _BridgeConfig:
+    dispatcher_url: str
+    api_key: str = ""
+
+
+def _config_from_env() -> _BridgeConfig:
+    """Build bridge config. Always routes through the dispatcher."""
+    from fcloud.config import resolve_url
+
+    return _BridgeConfig(dispatcher_url=resolve_url(None))
+
+
+class _DirectBridge:
+    """Owns a background asyncio loop and one DirectClient per live
+    session. Every public method is sync — callers don't see the
+    loop. Thread-safe for simultaneous calls from different threads.
+    """
+
+    def __init__(self, cfg: _BridgeConfig) -> None:
+        self._cfg = cfg
+        self._loop = asyncio.new_event_loop()
+        self._thread = threading.Thread(
+            target=self._loop.run_forever,
+            name="fcloud-direct-bridge",
+            daemon=True,
+        )
+        self._thread.start()
+        self._lock = threading.Lock()
+        # sid → DirectClient. Each entered session owns one
+        # WebSocket to the host.
+        self._clients: dict[str, DirectClient] = {}
+        # project_id → image_spec (JSON). create_project stashes
+        # here; create_session looks up to seed setup_session.
+        self._project_specs: dict[str, str] = {}
+        self._session_projects: dict[str, str] = {}
+
+    # ---- internal plumbing ----------------------------------------
+
+    def _await(self, coro, *, timeout: float | None = None):
+        """Run `coro` on the bridge's loop and block until done.
+        Exceptions are re-raised on the calling thread.
+
+        A KeyboardInterrupt (Ctrl-C) or a timeout cancels the coroutine
+        on the loop before propagating, so an interrupted caller does
+        not leave work running on the bridge thread (which would, e.g.,
+        keep polling a remote process and hold the session open)."""
+        fut = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        try:
+            return fut.result(timeout=timeout)
+        except (concurrent.futures.TimeoutError, KeyboardInterrupt):
+            fut.cancel()
+            try:
+                asyncio.run_coroutine_threadsafe(
+                    asyncio.sleep(0),
+                    self._loop,
+                ).result(timeout=0.1)
+                fut.result(timeout=0.1)
+            except Exception:
+                pass
+            raise
+
+    def _cleanup_client(self, client: DirectClient) -> None:
+        """Best-effort local socket cleanup; dispatcher owns teardown."""
+        try:
+            self._await(client.__aexit__(None, None, None), timeout=_CLEANUP_TIMEOUT_S)
+        except Exception:
+            pass
+
+    # ---- project surface ------------------------------------------
+
+    def create_project(
+        self,
+        name: str,
+        *,
+        image: Any | None = None,
+        image_id: str = "",
+        datastore_id: str = "",
+    ) -> ProjectInfo:
+        """Synthesize a ProjectInfo locally. The image spec (if any)
+        is stashed for later setup_session calls."""
+        project_id = f"direct-proj-{uuid.uuid4().hex[:8]}"
+        spec_json = ""
+        if image is not None:
+            spec_json = json.dumps(image.to_dict())
+            image_id = image_id or image.spec_hash()
+
+        info = ProjectInfo(
+            project_id=project_id,
+            name=name,
+            image_id=image_id,
+            datastore_id=datastore_id,
+        )
+        if spec_json:
+            with self._lock:
+                self._project_specs[project_id] = spec_json
+        return info
+
+    # ---- session surface ------------------------------------------
+
+    def _resolve_spec(self, project_id: str, sku: str) -> str:
+        """Return the project's image spec JSON, or the SKU default."""
+        with self._lock:
+            spec_json = self._project_specs.get(project_id, "")
+        if not spec_json:
+            from fcloud.image import Image
+
+            spec_json = json.dumps(Image.default_for_sku(sku).to_dict())
+        return spec_json
+
+    def create_cold_session(
+        self,
+        project_id: str,
+        *,
+        sku: str = "",
+        volume_mounts: list[dict[str, str]] | None = None,
+        min_disk_gb: int = 0,
+        keep_warm_seconds: int = 0,
+        provider: str = "",
+        region: str = "",
+    ) -> SessionInfo:
+        """Register a cold session (filesystem only, no host) and return
+        immediately. The session is brought online transparently on first
+        use via the resume-on-attach path. This is what `fcloud create`
+        uses — a session is a filesystem, not a held GPU.
+
+        Declared volumes are persisted on the cold record so they're composed
+        into the rootfs at setup when the session is first used."""
+        spec_json = self._resolve_spec(project_id, sku)
+        allocation = v2_connect.allocate_session(
+            self._cfg.dispatcher_url,
+            self._cfg.api_key,
+            image_spec=spec_json,
+            sku=sku,
+            cold=True,
+            volume_mounts=volume_mounts,
+            min_disk_gb=min_disk_gb,
+            keep_warm_seconds=keep_warm_seconds,
+            provider=provider,
+            region=region,
+        )
+        sid = allocation["sid"]
+        with self._lock:
+            self._project_specs.setdefault(sid, spec_json)
+            self._session_projects[sid] = project_id
+        return _session_info_from_allocation(allocation, project_id)
+
+    def _attach_volumes(
+        self, sid: str, volume_mounts: list[dict[str, str]] | None,
+        client: DirectClient | None = None,
+    ) -> None:
+        """Attach requested volumes to the now-live session and, when a
+        connected ``client`` is given, wait until their files have landed.
+
+        Idempotent server-side: re-attaching a volume already mounted at
+        the same path is a no-op, so callers may re-pass ``--volume`` (or
+        a fcloud.json ``volumes`` default) on every invocation safely. The
+        attach itself returns as soon as the host registers the mount; the
+        pull runs in the background, so without the wait a command issued
+        right after could run against a half-populated directory.
+        """
+        mounts = [m for m in (volume_mounts or []) if m.get("name")]
+        if not mounts:
+            return
+        v2_connect.attach_volumes(
+            self._cfg.dispatcher_url,
+            self._cfg.api_key,
+            sid,
+            mounts,
+        )
+        if client is not None:
+            self._await(volume_wait.wait_volumes_hydrated(
+                client, [str(m["name"]) for m in mounts],
+            ))
+
+    def create_session(
+        self,
+        project_id: str,
+        *,
+        sku: str = "",
+        volume_mounts: list[dict[str, str]] | None = None,
+        keep_warm_seconds: int = 0,
+        min_disk_gb: int = 0,
+        provider: str = "",
+        region: str = "",
+        ephemeral: bool = False,
+    ) -> SessionInfo:
+        """Allocate a session on this host. Returns a SessionInfo
+        with status=active once setup_session resolves. If the
+        project was created via create_project, its image_spec is
+        used; otherwise defaults to debian_slim.
+
+        ephemeral=True requests a run-to-completion job session: the
+        workspace is never persisted, the session cannot be resumed or
+        mounted after close, and durable outputs must go to volumes (or
+        be downloaded while the session is live)."""
+        spec_json = self._resolve_spec(project_id, sku)
+        # Captured as soon as the dispatcher queues the session, so an
+        # interrupt during provisioning (Ctrl-C while we poll/connect)
+        # can cancel it instead of leaving it stuck in "preparing".
+        pending_sid = ""
+        # State-aware narration: the fleet's provisioning feed distinguishes
+        # a real cloud launch ("provisioning/booting ...") from a queue that
+        # is just waiting for a busy host to free up (e.g. mid-teardown).
+        reporter = v2_connect.ProgressReporter(sku=sku)
+
+        def _on_queued(queued_sid: str) -> None:
+            nonlocal pending_sid
+            pending_sid = queued_sid
+            reporter.on_queued(queued_sid)
+
+        try:
+            allocation = v2_connect.allocate_session(
+                self._cfg.dispatcher_url,
+                self._cfg.api_key,
+                image_spec=spec_json,
+                sku=sku,
+                keep_warm_seconds=keep_warm_seconds,
+                volume_mounts=volume_mounts,
+                min_disk_gb=min_disk_gb,
+                provider=provider,
+                region=region,
+                # Names also drive the dispatcher's root-volume sizing.
+                volume_names=[
+                    m["name"] for m in (volume_mounts or []) if m.get("name")
+                ],
+                ephemeral=ephemeral,
+                on_queued=_on_queued,
+                on_wait=reporter.on_wait,
+                on_provisioning=reporter.on_provisioning,
+            )
+            pending_sid = allocation["sid"]
+            client = self._connect_ready_client(allocation, spec_json, sku)
+            # Volumes are composed into the rootfs at setup (sent with the
+            # allocate request above) — no post-live attach needed.
+        except BaseException:
+            # Allocation failed or was interrupted (KeyboardInterrupt
+            # included). Hand the half-provisioned session back to the
+            # dispatcher so it doesn't linger as "preparing".
+            self._abandon_pending_session(pending_sid)
+            raise
+        sid = allocation["sid"]
+        with self._lock:
+            self._clients[sid] = client
+            self._session_projects[sid] = project_id
+        # SessionInfo has no `sku` field; sku only matters at
+        # allocation time and we've already picked a host.
+        del sku
+        # Status/lifecycle come from the dispatcher's allocation response —
+        # never invented locally. host_id is the one local fact worth
+        # preferring: the WS handshake just completed against it.
+        return _session_info_from_allocation(
+            allocation, project_id, host_id=client._session.hid,
+        )
+
+    def _connect_ready_client(
+        self, allocation: dict[str, Any], spec_json: str, sku: str,
+    ) -> DirectClient:
+        """Open the data-plane WS and block until the host reports ready.
+        Tears the local socket down if readiness fails so we never leak a
+        half-open client; the dispatcher session is cleaned up by the
+        caller's allocation guard."""
+        client = self._await(
+            v2_connect.connect_with_retry(
+                allocation,
+                dispatcher_url=self._cfg.dispatcher_url,
+                api_key=self._cfg.api_key,
+                sku=sku,
+                image_spec=spec_json,
+            )
+        )
+        try:
+            self._await(client.wait_for_ready())
+        except BaseException:
+            self._cleanup_client(client)
+            raise
+        return client
+
+    def _abandon_pending_session(self, sid: str) -> None:
+        """Tell the dispatcher to drop a session we never finished
+        bringing up. Best-effort: a failure here must not mask the
+        original error (often a KeyboardInterrupt) we're unwinding."""
+        if not sid:
+            return
+        try:
+            v2_connect.close_session(
+                self._cfg.dispatcher_url,
+                self._cfg.api_key,
+                sid,
+            )
+        except Exception:
+            pass
+
+    # NOTE deliberately no get_session here. _clients/_session_projects are
+    # connection bookkeeping — which WS to use for ops, which project a sid
+    # was created under. They are NOT session state: an entry lingers after
+    # a host dies and is absent in every process that didn't attach. A
+    # bridge get_session fabricated status from this map and starved a
+    # fresh-process provision poller for 10 minutes per session, which once
+    # caused an outage. Server-state reads go through Client.describe_session.
+
+    def attach_session(
+        self,
+        session_id: str,
+        *,
+        volume_mounts: list[dict[str, str]] | None = None,
+        image_spec: str = "",
+        sku: str = "",
+    ) -> SessionInfo:
+        """Connect to an existing session (fresh token, new WS).
+
+        Session warmth is invisible to the caller. If the session is
+        already active this reconnects directly; if it was closed
+        (reaped while idle) it is transparently resumed — rebuilt on a
+        fresh host with its filesystem restored. Either way the method
+        gets a fresh token, opens a data-plane WS, and waits for ready.
+        Use ``detach_session`` to drop the WS without closing the
+        session.
+
+        A QUEUED/cold session blocks here while the host provisions; the
+        ``ProgressReporter`` narrates each stage on stderr (the same UX the
+        create path already has) so ``fcloud exec --on <sid>`` no longer looks
+        hung. Ctrl-C still cancels: the callbacks only print, so an interrupt
+        propagates out of ``reconnect_session``/``connect_with_retry`` exactly
+        as before.
+
+        ``image_spec`` (when non-empty) is forwarded on the resume path
+        so an edited fcloud.json takes effect on the next use of an
+        existing session. Has no effect when the session is already
+        active — the running image cannot be swapped under a live host.
+        """
+        reporter = v2_connect.ProgressReporter()
+        allocation = v2_connect.reconnect_session(
+            self._cfg.dispatcher_url,
+            self._cfg.api_key,
+            session_id,
+            sku=sku,
+            image_spec=image_spec,
+            **reporter.callbacks(),
+        )
+        client = self._await(
+            v2_connect.connect_with_retry(
+                allocation,
+                dispatcher_url=self._cfg.dispatcher_url,
+                api_key=self._cfg.api_key,
+                on_status=_persistent_session_status_printer,
+            )
+        )
+        try:
+            self._await(client.wait_for_ready())
+            reporter.ready()
+            # Attach the requested volumes once the session is online,
+            # whether it was already active or transparently resumed.
+            self._attach_volumes(session_id, volume_mounts, client)
+        except Exception:
+            self._cleanup_client(client)
+            raise
+        with self._lock:
+            self._clients[session_id] = client
+            project_id = self._session_projects.get(session_id, "")
+        return _session_info_from_allocation(
+            allocation, project_id, host_id=str(allocation.get("hid") or ""),
+        )
+
+    def session_host_id(self, session_id: str) -> str:
+        """Return the host id of a currently-attached session, or "".
+
+        Lets reconnect paths compare the host before and after a drop, so
+        a reconnect to the same host is not reported as a migration.
+        """
+        with self._lock:
+            client = self._clients.get(session_id)
+        if client is None or client._session is None:
+            return ""
+        return client._session.hid
+
+    def detach_session(self, session_id: str) -> None:
+        """Drop the data-plane WS without closing the session.
+
+        The session stays alive on the host (subject to its idle
+        timeout). Reconnect later with ``attach_session``.
+        """
+        with self._lock:
+            client = self._clients.pop(session_id, None)
+        if client is None:
+            return
+        self._cleanup_client(client)
+
+    def close_session(self, session_id: str) -> dict[str, Any]:
+        with self._lock:
+            client = self._clients.pop(session_id, None)
+        if client is not None:
+            self._cleanup_client(client)
+            with self._lock:
+                self._session_projects.pop(session_id, None)
+        return v2_connect.close_session(
+            self._cfg.dispatcher_url,
+            self._cfg.api_key,
+            session_id,
+        )
+
+    def release_session(self, session_id: str) -> dict[str, Any]:
+        """Soft-release: ask the host to tear the session down only if
+        nothing else is using it (no other connection, no in-flight op,
+        no running process), then drop the local WS. Unlike
+        ``close_session`` this never force-closes a session that other
+        terminals or background processes still need — it is the exit
+        path for ephemeral ``exec``/``run``/``shell`` commands.
+        """
+        with self._lock:
+            client = self._clients.get(session_id)
+        result: dict[str, Any] = {"released": False, "torn_down": False}
+        if client is not None:
+            try:
+                result = self._await(
+                    client._request(                     # noqa: SLF001
+                        {
+                            "type": "release_session",
+                            "session_id": session_id,
+                        },
+                        timeout=_CLEANUP_TIMEOUT_S,
+                    ),
+                    timeout=_CLEANUP_TIMEOUT_S + 1.0,
+                )
+            except Exception:
+                pass
+        self.detach_session(session_id)
+        return result
+
+    # ---- exec surface ---------------------------------------------
+
+    def session_execute(
+        self,
+        session_id: str,
+        command: list[str],
+        *,
+        timeout_ms: int = 30_000,
+        workdir: str = "",
+        env: dict[str, str] | None = None,
+        output_limit_bytes: int | None = _DEFAULT_EXEC_OUTPUT_LIMIT_BYTES,
+        output_range: str = "tail",
+        on_output: Callable[[str], None] | None = None,
+        on_start: Callable[[str], None] | None = None,
+        pty: dict[str, int] | None = None,
+    ) -> ExecResult:
+        client = self._require_client(session_id)
+        resp = self._await(
+            client.spawn(
+                command,
+                work_dir=workdir,
+                env=env or {},
+                pty=pty,
+            )
+        )
+        pid = resp.get("process_id", "")
+        if not pid:
+            raise FcloudRuntimeError(f"spawn failed: {resp}")
+        # The process now exists with a durable id, before we block on its
+        # output. Surface the id here so a caller can persist it (e.g. to a
+        # --emit-pid file) even if the blocking wait's stdio is later eaten
+        # by a `| tail` or a backgrounded shell.
+        if on_start is not None:
+            on_start(pid)
+        proc = Process(
+            process_id=pid,
+            session_id=session_id,
+            status=ProcessStatus(resp.get("status") or "running"),
+        )
+        return self._run_capture_with_interrupt_kill(
+            client,
+            proc,
+            timeout_ms,
+            output_limit_bytes,
+            output_range,
+            on_output,
+        )
+
+    def _run_capture_with_interrupt_kill(
+        self,
+        client: DirectClient,
+        proc: Process,
+        timeout_ms: int,
+        output_limit_bytes: int | None,
+        output_range: str,
+        on_output: Callable[[str], None] | None,
+    ) -> ExecResult:
+        """Run the streaming-capture coroutine, translating a Ctrl-C
+        into a remote kill that lets streaming finish naturally.
+
+        Cancelling the capture coroutine on SIGINT (the previous
+        behavior) tears down the WS subscribe before the remote process
+        actually exits, so any output produced during the kill window
+        lands in the durable log but never reaches the user's terminal.
+        Instead we install a SIGINT handler that schedules the remote
+        kill while the capture keeps running; output keeps printing
+        until the remote sees the signal and the stream ends. A
+        ``_INTERRUPT_KILL_TIMEOUT_S`` watchdog bounds the wait so a
+        remote that ignores the kill cannot wedge the terminal, and a
+        second Ctrl-C force-quits immediately.
+        """
+        fut = asyncio.run_coroutine_threadsafe(
+            self._capture_spawned_exec(
+                client, proc, timeout_ms,
+                output_limit_bytes, output_range, on_output,
+            ),
+            self._loop,
+        )
+        if not _can_handle_interrupt_on_main_thread():
+            # Library use from non-main threads (no signal handling
+            # there): fall back to the cancel+reap path so a
+            # KeyboardInterrupt from the calling thread still tears
+            # the session down promptly.
+            try:
+                return fut.result()
+            except KeyboardInterrupt:
+                fut.cancel()
+                self._kill_after_interrupt(client, proc)
+                raise
+
+        interrupted = threading.Event()
+        watchdog: list[threading.Timer | None] = [None]
+        prev = signal.getsignal(signal.SIGINT)
+
+        def handler(_signum, _frame) -> None:
+            if interrupted.is_set():
+                # Second Ctrl-C: stop streaming immediately. Restoring
+                # the previous handler lets a third Ctrl-C reach the
+                # default behavior (e.g. pytest's). fut.cancel() makes
+                # fut.result() return CancelledError quickly.
+                signal.signal(signal.SIGINT, prev)
+                if watchdog[0] is not None:
+                    watchdog[0].cancel()
+                fut.cancel()
+                raise KeyboardInterrupt
+            interrupted.set()
+            sys.stderr.write(
+                "\nInterrupting — waiting for remote process to exit "
+                "(press Ctrl-C again to force-quit).\n",
+            )
+            sys.stderr.flush()
+            try:
+                asyncio.run_coroutine_threadsafe(
+                    client.kill(proc.process_id),
+                    self._loop,
+                )
+            except Exception:
+                pass
+            timer = threading.Timer(
+                _INTERRUPT_KILL_TIMEOUT_S, fut.cancel,
+            )
+            timer.daemon = True
+            timer.start()
+            watchdog[0] = timer
+
+        signal.signal(signal.SIGINT, handler)
+        try:
+            try:
+                result = fut.result()
+            except concurrent.futures.CancelledError:
+                if interrupted.is_set():
+                    raise KeyboardInterrupt from None
+                raise
+        finally:
+            signal.signal(signal.SIGINT, prev)
+            if watchdog[0] is not None:
+                watchdog[0].cancel()
+
+        if interrupted.is_set():
+            raise KeyboardInterrupt
+        return result
+
+    def _kill_after_interrupt(self, client: DirectClient, proc: Process) -> None:
+        """Best-effort kill + reap of a process whose foreground caller
+        was interrupted. Never raises — the caller is already unwinding a
+        KeyboardInterrupt and just needs the remote work stopped. Used
+        only on the non-main-thread fallback path; the main-thread path
+        kills via the SIGINT handler and lets streaming complete."""
+        try:
+            self._await(client.kill(proc.process_id), timeout=_CLEANUP_TIMEOUT_S)
+        except BaseException:
+            return
+        deadline = time.time() + _INTERRUPT_KILL_TIMEOUT_S
+        while time.time() < deadline:
+            try:
+                data = self._await(
+                    client.poll(proc.process_id, lines=0),
+                    timeout=_CLEANUP_TIMEOUT_S,
+                )
+            except BaseException:
+                return
+            if ProcessStatus(data.get("status") or "running") != ProcessStatus.RUNNING:
+                return
+            time.sleep(0.1)
+
+    async def _capture_spawned_exec(
+        self,
+        client: DirectClient,
+        proc: Process,
+        timeout_ms: int,
+        output_limit_bytes: int | None,
+        output_range: str,
+        on_output: Callable[[str], None] | None = None,
+    ) -> ExecResult:
+        start = time.time()
+        deadline = start + max(0.0, timeout_ms / 1000)
+        capture = _OutputCapture()
+        capture_err = _OutputCapture()
+        collector = asyncio.create_task(
+            self._collect_process_output(
+                client, proc.process_id, capture, capture_err, on_output,
+            ),
+        )
+        try:
+            return await self._poll_captured_exec(
+                client,
+                proc,
+                deadline,
+                start,
+                collector,
+                capture,
+                capture_err,
+                output_limit_bytes,
+                output_range,
+            )
+        finally:
+            # On a normal return the poll loop has already drained the
+            # collector; on cancellation (the caller was interrupted) it
+            # has not, so stop it here before closing the spool it writes
+            # to. _cancel_output_collection is idempotent.
+            await self._cancel_output_collection(collector)
+            capture.close()
+            capture_err.close()
+
+    async def _collect_process_output(
+        self,
+        client: DirectClient,
+        process_id: str,
+        capture: _OutputCapture,
+        capture_err: _OutputCapture,
+        on_output: Callable[[str], None] | None = None,
+    ) -> None:
+        """Spool process output, and mirror each chunk to ``on_output``
+        live as it arrives (used to stream to the console in real time).
+
+        Frames are routed by their ``stream`` label so the exec reply can
+        report stdout and stderr separately; a PTY genuinely merges the
+        streams, so pty (and unlabeled) frames count as stdout.
+
+        ``on_output`` runs on the bridge's event-loop thread while the
+        calling thread is parked in ``_await``, so it is the sole writer
+        to the console — no interleaving with the caller. A failing sink
+        must not kill output collection, so its errors are swallowed.
+        """
+        async for frame in client.subscribe(process_id):
+            if frame.get("type") == "stream_end":
+                return
+            text = str(frame.get("data") or "")
+            if not text:
+                continue
+            if frame.get("stream") == "stderr":
+                capture_err.write(text)
+            else:
+                capture.write(text)
+            if on_output is not None:
+                try:
+                    on_output(text)
+                except Exception:
+                    pass
+
+    async def _poll_captured_exec(
+        self,
+        client: DirectClient,
+        proc: Process,
+        deadline: float,
+        start: float,
+        collector: asyncio.Task,
+        capture: _OutputCapture,
+        capture_err: _OutputCapture,
+        output_limit_bytes: int | None,
+        output_range: str,
+    ) -> ExecResult:
+        poll_delays = [0.01, 0.05, 0.1, 0.25, 2.0]
+        polls = 0
+        last_data: dict[str, Any] = {"status": "running"}
+        first_fail: float | None = None
+        while True:
+            try:
+                data = await client.poll(proc.process_id, lines=0)
+            except (TimeoutError, ConnectionError, OSError, asyncio.TimeoutError):
+                # If the host announced `host_interrupted` over the WS
+                # right before this poll failed, the process was SIGTERM'd on
+                # an EVACUATE (non-checkpointable) reclaim — it is gone for
+                # good, no restore is coming. Surface that terminal status.
+                if getattr(client, "_host_interrupted", False):
+                    await self._cancel_output_collection(collector)
+                    captured = capture.result(output_limit_bytes, output_range)
+                    captured_err = capture_err.result(
+                        output_limit_bytes, output_range,
+                    )
+                    result = self._captured_exec_result(
+                        proc, last_data, start, "host_terminated",
+                        captured, captured_err,
+                    )
+                    result.returncode = None
+                    result.message = (
+                        "Host was interrupted (spot reclaim or eviction); "
+                        f"the process did not survive. Reconnect with fcloud logs "
+                        f"{proc.session_id} {proc.process_id} --output all "
+                        "for any captured output."
+                    )
+                    return result
+                # No host_interrupted notice: the host vanished mid-stream — a
+                # spot MIGRATE (the session is being checkpointed/restored on a
+                # new host) or a hard drop. Retry briefly for a momentary blip,
+                # then return "host_lost" so the caller can ride the migration
+                # — rather than hanging to the run deadline or, worse, returning
+                # a false "still running" (exit 0) that masks a lost host.
+                now = time.time()
+                if first_fail is None:
+                    first_fail = now
+                if now - first_fail >= _HOST_LOSS_GRACE_S or now >= deadline:
+                    await self._cancel_output_collection(collector)
+                    captured = capture.result(output_limit_bytes, output_range)
+                    captured_err = capture_err.result(
+                        output_limit_bytes, output_range,
+                    )
+                    result = self._captured_exec_result(
+                        proc, last_data, start, "host_lost",
+                        captured, captured_err,
+                    )
+                    result.returncode = None
+                    result.message = (
+                        "Host connection lost — checking whether the session is "
+                        "migrating to a new host."
+                    )
+                    return result
+                await asyncio.sleep(
+                    min(_TRANSIENT_POLL_RETRY_DELAY_S, max(0.0, deadline - time.time())),
+                )
+                continue
+            first_fail = None
+            last_data = data
+            # NOTE: a reply with no `status` is still read as RUNNING here, and
+            # that is deliberate. It looks like a bug — the host answers a
+            # status-less error frame for a process whose sandbox was rebuilt
+            # under it, and this loop will poll that corpse until --wait
+            # expires. But "no status" is NOT specific to a dead process: the
+            # client plane answers every op with a status-less
+            # `{"ok": false, "error": ...}` whenever the session is not
+            # PhaseReady, which is exactly what a session being CHECKPOINTED
+            # OR RESTORED looks like. Treating that as "your process died"
+            # would abandon a migration the caller is supposed to ride, and
+            # ending a live training run early is far worse than waiting.
+            #
+            # The host resolves this authoritatively instead: once the rebuilt
+            # container no longer owns the process, `poll` answers from the
+            # durable index with a real terminal status, and the branch below
+            # ends the wait in ~2s. A client-side verdict would need the host to say
+            # explicitly "this process is gone and is not coming back", which
+            # the protocol has no way to express today.
+            status = ProcessStatus(data.get("status") or "running")
+            if status != ProcessStatus.RUNNING:
+                await self._finish_output_collection(collector)
+                captured = capture.result(output_limit_bytes, output_range)
+                captured_err = capture_err.result(
+                    output_limit_bytes, output_range,
+                )
+                durable = await self._durable_exec_result(
+                    client,
+                    proc,
+                    data,
+                    start,
+                    output_limit_bytes,
+                    output_range,
+                )
+                if durable is not None:
+                    return durable
+                return self._captured_exec_result(
+                    proc,
+                    data,
+                    start,
+                    "exited",
+                    captured,
+                    captured_err,
+                )
+            if time.time() >= deadline:
+                await self._cancel_output_collection(collector)
+                captured = capture.result(output_limit_bytes, output_range)
+                captured_err = capture_err.result(
+                    output_limit_bytes, output_range,
+                )
+                result = self._captured_exec_result(
+                    proc,
+                    data,
+                    start,
+                    "running",
+                    captured,
+                    captured_err,
+                )
+                result.returncode = None
+                result.message = (
+                    "Command is still running; use fcloud logs "
+                    f"{proc.session_id} {proc.process_id} --follow"
+                )
+                return result
+            delay = poll_delays[min(polls, len(poll_delays) - 1)]
+            polls += 1
+            await asyncio.sleep(min(delay, max(0.0, deadline - time.time())))
+
+    async def _finish_output_collection(self, collector: asyncio.Task) -> None:
+        try:
+            await asyncio.wait_for(collector, timeout=_OUTPUT_DRAIN_TIMEOUT_S)
+        except asyncio.TimeoutError:
+            await self._cancel_output_collection(collector)
+        except Exception:
+            await asyncio.gather(collector, return_exceptions=True)
+
+    async def _cancel_output_collection(self, collector: asyncio.Task) -> None:
+        if collector.done():
+            await asyncio.gather(collector, return_exceptions=True)
+            return
+        collector.cancel()
+        await asyncio.gather(collector, return_exceptions=True)
+
+    async def _durable_exec_result(
+        self,
+        client: DirectClient,
+        proc: Process,
+        data: dict,
+        start: float,
+        output_limit_bytes: int | None,
+        output_range: str,
+    ) -> ExecResult | None:
+        try:
+            log = await client.read_log(
+                proc.process_id,
+                stream="stdout",
+                output_range=output_range,
+                output_bytes=output_limit_bytes,
+            )
+        except Exception:
+            return None
+        # stderr is best-effort: a missing stderr log must not discard the
+        # durable stdout read (the captured fallback would merge the streams).
+        stderr = ""
+        try:
+            err_log = await client.read_log(
+                proc.process_id,
+                stream="stderr",
+                output_range=output_range,
+                output_bytes=output_limit_bytes,
+            )
+            stderr = str(err_log.get("output", ""))
+        except Exception:
+            pass
+        return ExecResult(
+            stdout=str(log.get("output", "")),
+            stderr=stderr,
+            returncode=int(data.get("exit_code", -1)),
+            duration_ms=int((time.time() - start) * 1000),
+            status="exited",
+            process_id=proc.process_id,
+            session_id=proc.session_id,
+            stdout_truncated=bool(log.get("output_truncated", False)),
+            stdout_total_bytes=int(log.get("output_total_bytes", 0)),
+            stdout_retained_bytes=int(log.get("output_retained_bytes", 0)),
+            stdout_range=str(log.get("output_range", output_range)),
+        )
+
+    def _captured_exec_result(
+        self,
+        proc: Process,
+        data: dict,
+        start: float,
+        status: str,
+        captured: _CapturedOutput,
+        captured_err: _CapturedOutput,
+    ) -> ExecResult:
+        return ExecResult(
+            stdout=captured.text,
+            stderr=captured_err.text,
+            returncode=int(data.get("exit_code", -1)),
+            duration_ms=int((time.time() - start) * 1000),
+            status=status,
+            process_id=proc.process_id,
+            session_id=proc.session_id,
+            stdout_truncated=captured.truncated,
+            stdout_total_bytes=captured.total_bytes,
+            stdout_retained_bytes=captured.retained_bytes,
+            stdout_range=captured.range_name,
+        )
+
+    def session_spawn(
+        self,
+        session_id: str,
+        command: list[str],
+        *,
+        workdir: str = "",
+        env: dict[str, str] | None = None,
+        pty: dict[str, int] | None = None,
+    ) -> Process:
+        client = self._require_client(session_id)
+        resp = self._await(
+            client.spawn(
+                command,
+                work_dir=workdir,
+                env=env or {},
+                pty=pty,
+            )
+        )
+        pid = resp.get("process_id", "")
+        if not pid:
+            raise FcloudRuntimeError(f"spawn failed: {resp}")
+        return Process(
+            process_id=pid,
+            session_id=session_id,
+            status=ProcessStatus(resp.get("status") or "running"),
+        )
+
+    def session_poll(
+        self,
+        session_id: str,
+        process_id: str,
+        lines: int = 20,
+    ) -> dict:
+        client = self._require_client(session_id)
+        return self._await(client.poll(process_id, lines=lines))
+
+    def session_logs(
+        self,
+        session_id: str,
+        process_id: str,
+        *,
+        stream: str = "combined",
+        output_limit_bytes: int | None = None,
+        output_range: str = "all",
+    ) -> dict:
+        client = self._require_client(session_id)
+        return self._await(
+            client.read_log(
+                process_id,
+                stream=stream,
+                output_range=output_range,
+                output_bytes=output_limit_bytes,
+            )
+        )
+
+    def session_kill(
+        self,
+        session_id: str,
+        process_id: str,
+    ) -> dict:
+        client = self._require_client(session_id)
+        return self._await(client.kill(process_id))
+
+    def session_sync(self, session_id: str) -> dict:
+        client = self._require_client(session_id)
+        return self._await(client.sync_session())
+
+    def watch_process(self, session_id: str, process_id: str):
+        client = self._require_client(session_id)
+        stream = client.subscribe(process_id)
+        try:
+            while True:
+                try:
+                    yield self._await(stream.__anext__())
+                except StopAsyncIteration:
+                    return
+        finally:
+            self._await(stream.aclose())
+
+    # ---- file I/O -------------------------------------------------
+
+    def upload_files(
+        self,
+        session_id: str,
+        local_path: str,
+        remote_path: str = "",
+        options: UploadOptions | None = None,
+    ) -> dict:
+        """Push files from local_path into the session's workspace.
+
+        Which files ship is decided by :mod:`fcloud.fileset` (hidden entries,
+        dependency dirs and symlinks are skipped unless ``options`` says
+        otherwise). Small files ride inline over the WS in batched
+        write_files frames. Large files (> _INLINE_FILE_LIMIT) are uploaded
+        directly to S3 via a dispatcher-minted pre-signed URL, then pulled by
+        the host (sync_pull) — no bytes through the dispatcher, up to 5 GiB
+        per file (the single-PUT ceiling; the dispatcher rejects larger
+        files at presign time). If the dispatcher has no S3 configured,
+        large files fall back to the inline path.
+        """
+        collected = collect_upload_files(
+            local_path, remote_path, options, dest_for=_workspace_relative_path,
+        )
+        client = self._require_client(session_id)
+        files = collected.files
+
+        small = [(dest, p) for dest, p, sz in files if sz <= _INLINE_FILE_LIMIT]
+        large = [(dest, p, sz) for dest, p, sz in files if sz > _INLINE_FILE_LIMIT]
+
+        written = self._upload_inline(client, small)
+        # Which files take the S3 route is invisible from the outside, and
+        # that route has its own failure modes (presign, signing, quota).
+        # A 6.3 MB results JSON inside an otherwise tiny code upload once
+        # routed the whole batch through a broken presigned path and the
+        # failure was attributable to the command, not to a file. Name
+        # them, and name them on failure too.
+        try:
+            written += self._upload_via_s3(client, session_id, large)
+        except FcloudError as exc:
+            raise type(exc)(
+                f"{exc} — failed on the S3 upload path, taken by "
+                f"{_s3_route_note(large)}"
+            ) from exc
+        if written != len(files):
+            raise FcloudRuntimeError(
+                f"upload incomplete: host wrote {written}/{len(files)} files"
+            )
+        return {
+            "status": "ok",
+            "written": written,
+            "skipped": [],
+            "bytes_total": collected.bytes_total,
+            "files_count": len(files),
+            "skipped_hidden": collected.skipped_hidden,
+            "skipped_symlinks": collected.skipped_symlinks,
+            "skipped_other": collected.skipped_other,
+            "skip_note": collected.skip_note(),
+            "inline_files": len(small),
+            "s3_files": [{"path": dest, "size": sz} for dest, _p, sz in large],
+            "s3_threshold_bytes": _INLINE_FILE_LIMIT,
+        }
+
+    def _upload_inline(
+        self, client: DirectClient, items: list[tuple[str, Path]],
+    ) -> int:
+        """Send small files inline over the WS, batched to keep each
+        frame under the read-limit. Returns the number written."""
+        written = 0
+        batch: list[tuple[str, bytes]] = []
+        batch_size = 0
+
+        def flush() -> None:
+            nonlocal written, batch, batch_size
+            if not batch:
+                return
+            r = self._await(client.write_files(batch))
+            skipped = r.get("skipped", [])
+            if isinstance(skipped, list) and skipped:
+                raise FcloudRuntimeError(
+                    "upload failed: " + "; ".join(str(s) for s in skipped)
+                )
+            written += int(r.get("written", 0))
+            batch = []
+            batch_size = 0
+
+        for dest, p in items:
+            data = p.read_bytes()
+            if batch_size + len(data) > _INLINE_BATCH_BYTES and batch:
+                flush()
+            batch.append((dest, data))
+            batch_size += len(data)
+        flush()
+        return written
+
+    def _upload_via_s3(
+        self,
+        client: DirectClient,
+        session_id: str,
+        items: list[tuple[str, Path, int]],
+    ) -> int:
+        """Upload large files to S3 via pre-signed PUT URLs, then have
+        the host pull them by hash. Falls back to the inline path when
+        the dispatcher has no S3 configured. Returns the number the
+        host confirms it wrote.
+
+        The PUT loop is bracketed by a host-side transfer lease
+        (begin/end_transfer, best-effort — the host TTL covers a dead
+        client) and shadowed by a WS keepalive task, so the empty reaper
+        does not close the session during the WS-silent transfer and the
+        final sync_pull does not land on a corpse."""
+        if not items:
+            return 0
+
+        hashed = [(dest, p, sz, _hash_file(p)) for dest, p, sz in items]
+        self._best_effort(client.begin_transfer())
+        keepalive = asyncio.run_coroutine_threadsafe(
+            self._upload_keepalive(client), self._loop,
+        )
+        try:
+            bucket = self._put_blobs_chunked(hashed)
+            if bucket is None:
+                # S3 unconfigured — degrade to inline (host caps apply).
+                return self._upload_inline(
+                    client, [(d, p) for d, p, _, _ in hashed],
+                )
+            manifest = [
+                {"path": dest, "hash": h, "size": sz}
+                for dest, _p, sz, h in hashed
+            ]
+            return self._sync_pull_checked(session_id, bucket, manifest)
+        finally:
+            keepalive.cancel()
+            # The sync_pull retry may have swapped the client; release
+            # the lease on whichever WS is live now.
+            with self._lock:
+                live = self._clients.get(session_id) or client
+            self._best_effort(live.end_transfer())
+
+    def _put_blobs_chunked(
+        self, hashed: list[tuple[str, Path, int, str]],
+    ) -> str | None:
+        """PUT blobs to S3, minting pre-signed URLs one chunk at a time
+        so every signature is fresh when used (see _PRESIGN_CHUNK).
+        Returns the bucket, or None when S3 is unconfigured."""
+        bucket: str | None = None
+        for start in range(0, len(hashed), _PRESIGN_CHUNK):
+            chunk = hashed[start:start + _PRESIGN_CHUNK]
+            urls = self._request_upload_urls(
+                [{"hash": h, "size": sz} for _, _, sz, h in chunk]
+            )
+            if urls is None:
+                return None
+            bucket, uploads, exists, put_headers = urls
+            for _dest, p, _sz, h in chunk:
+                if h in exists:
+                    continue  # deduped: blob already in S3
+                put_url = uploads.get(h)
+                if not put_url:
+                    raise FcloudRuntimeError(f"no upload URL minted for {p}")
+                _blob_put(put_url, p, put_headers.get(h))
+        return bucket
+
+    def _sync_pull_checked(
+        self, session_id: str, bucket: str, manifest: list[dict],
+    ) -> int:
+        """Send sync_pull and validate the host's ack. Reconnects and
+        retries once on a connection error ("Cannot write to closing
+        transport", peer reset): the S3 objects are durable and
+        content-addressed, so a second sync_pull is idempotent.
+        Raises unless the host confirms every manifest entry."""
+        client = self._require_client(session_id)
+        try:
+            resp = self._await(client.sync_pull(bucket, manifest))
+        except Exception as exc:
+            if not _is_connection_error(exc):
+                raise
+            client = self._reattach_client(session_id)
+            resp = self._await(client.sync_pull(bucket, manifest))
+        # The host reports a failed request as {"stderr": msg, "returncode":
+        # -1} (hostservice errorResponse), not under an "error" key — reading
+        # only "error" turned every real host failure into a misleading
+        # "0/N landed". Surface the actual message.
+        host_err = resp.get("error") or (
+            resp.get("stderr") if resp.get("returncode") == -1 else None
+        )
+        if host_err:
+            raise FcloudRuntimeError(f"host pull failed: {host_err}")
+        added = int(resp.get("files_added", 0))
+        requested = int(resp.get("files_requested", len(manifest)))
+        if resp.get("status") == "partial" or added < requested:
+            raise FcloudRuntimeError(
+                f"host pull incomplete: {added}/{requested} "
+                f"large files landed"
+            )
+        return added
+
+    def _reattach_client(self, session_id: str) -> DirectClient:
+        """Re-open the data-plane WS for a session whose connection died
+        mid-upload: fresh dispatcher token, connect_with_retry, wait for
+        ready, then swap the client table entry."""
+        with self._lock:
+            old = self._clients.pop(session_id, None)
+        if old is not None:
+            self._cleanup_client(old)
+        allocation = v2_connect.reconnect_session(
+            self._cfg.dispatcher_url, self._cfg.api_key, session_id,
+        )
+        client = self._await(
+            v2_connect.connect_with_retry(
+                allocation,
+                dispatcher_url=self._cfg.dispatcher_url,
+                api_key=self._cfg.api_key,
+            )
+        )
+        try:
+            self._await(client.wait_for_ready())
+        except BaseException:
+            self._cleanup_client(client)
+            raise
+        with self._lock:
+            self._clients[session_id] = client
+        return client
+
+    def _best_effort(self, coro) -> None:
+        """Run a fire-and-forget frame (transfer-lease bookkeeping) and
+        swallow failures: an older host that doesn't know the frame, or
+        a dying WS, must not fail the upload itself — the host-side
+        lease TTL covers a client that never sends end_transfer."""
+        try:
+            self._await(coro, timeout=30.0)
+        except Exception:  # noqa: BLE001 — see docstring
+            pass
+
+    async def _upload_keepalive(self, client: DirectClient) -> None:
+        """Periodic cheap frame over the session WS while the PUT loop
+        runs on the calling thread. Without it the WS carries zero
+        traffic for the whole transfer, the host lease expires under a
+        slow single-file PUT, and the failure only surfaces at the final
+        sync_pull send. Errors are swallowed — reconnection is
+        owned by the sync_pull retry."""
+        while True:
+            await asyncio.sleep(_UPLOAD_KEEPALIVE_S)
+            try:
+                await client.sync_check()
+            except Exception:  # noqa: BLE001 — keepalive is best-effort
+                pass
+
+    def _request_upload_urls(
+        self, files: list[dict],
+    ) -> tuple[str, dict, set, dict] | None:
+        """Ask the dispatcher for pre-signed PUT URLs. Returns
+        (bucket, {hash: url}, {existing hashes}, {hash: signed headers}),
+        or None if the dispatcher reports S3 is unconfigured. The headers
+        (``put_headers`` in the response, absent from older dispatchers)
+        must accompany the PUT verbatim — see _blob_put."""
+        import requests as _requests
+
+        headers = _dispatcher_headers(self._cfg.api_key)
+        resp = _requests.post(
+            self._cfg.dispatcher_url + "/v1/blobs/upload-url",
+            json={"files": files},
+            headers=headers,
+            timeout=30,
+        )
+        if resp.status_code == 503:
+            return None
+        data = resp.json()
+        _raise_if_client_too_old(resp.status_code, data)
+        if resp.status_code >= 400 or "error" in data:
+            raise FcloudRuntimeError(
+                data.get("error") or f"presign failed: status {resp.status_code}"
+            )
+        return (
+            data.get("bucket", ""),
+            data.get("uploads", {}) or {},
+            set(data.get("exists", []) or []),
+            data.get("put_headers", {}) or {},
+        )
+
+    def download_file(
+        self,
+        session_id: str,
+        remote_path: str,
+    ) -> bytes:
+        """Read a single file from the session's workspace and return
+        its bytes. Small files come back inline over the WS; large files
+        are fetched from S3 via a pre-signed GET (the host returns a blob
+        reference). Raises FileNotFoundError if not present."""
+        import base64 as _b64
+
+        client = self._require_client(session_id)
+        resp = self._await(
+            client.read_file_meta(remote_path, _INLINE_FILE_LIMIT)
+        )
+        if resp.get("status") != "ok":
+            err = resp.get("stderr") or resp.get("error") or str(resp)
+            raise FileNotFoundError(f"download {remote_path}: {err}")
+        if resp.get("mode") == "blob":
+            return self._download_blob(resp["hash"])
+        return _b64.b64decode(resp.get("content_b64", ""))
+
+    def _download_blob(self, file_hash: str) -> bytes:
+        """Fetch a content-addressed blob from S3 via a dispatcher-minted
+        pre-signed GET URL."""
+        import requests as _requests
+
+        headers = _dispatcher_headers(self._cfg.api_key)
+        resp = _requests.post(
+            self._cfg.dispatcher_url + "/v1/blobs/download-url",
+            json={"hashes": [file_hash]},
+            headers=headers,
+            timeout=30,
+        )
+        data = resp.json()
+        _raise_if_client_too_old(resp.status_code, data)
+        if resp.status_code >= 400 or "error" in data:
+            raise FcloudRuntimeError(
+                data.get("error")
+                or f"download presign failed: status {resp.status_code}"
+            )
+        url = (data.get("urls") or {}).get(file_hash)
+        if not url:
+            raise FcloudRuntimeError(f"no download URL minted for {file_hash}")
+        got = _requests.get(url, timeout=_BLOB_HTTP_TIMEOUT)
+        if got.status_code >= 300:
+            raise FcloudRuntimeError(
+                f"S3 download failed ({got.status_code}): {got.text[:200]}"
+            )
+        return got.content
+
+    def _require_client(self, session_id: str) -> DirectClient:
+        with self._lock:
+            client = self._clients.get(session_id)
+        if client is None:
+            raise FcloudRuntimeError(
+                f"session {session_id} is not active on this direct "
+                f"client (was it closed? only sessions created via "
+                f"this process have WS handles here)"
+            )
+        return client
+
+    # ---- shutdown -------------------------------------------------
+
+    def shutdown(self) -> None:
+        """Close every live WS and stop the event loop. Safe to
+        call multiple times."""
+        with self._lock:
+            clients = list(self._clients.items())
+            self._clients.clear()
+        for _, c in clients:
+            self._cleanup_client(c)
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        self._thread.join(timeout=_CLEANUP_TIMEOUT_S)