fcloud-sdk 0.1.0__py3-none-any.whl

fcloud/cli/console.py

@@ -0,0 +1,227 @@
+"""Foreground-command plumbing shared by exec/run/wait.
+
+Live console streaming, the long-wait stderr heartbeat, the ExecResult
+exit-code contract and the --emit-pid writer.
+"""
+
+import json as _json
+import os
+import sys
+
+from fcloud.cli import output
+
+
+def _live_console_sink():
+    """Return (on_output, pty) for live console streaming, or (None, None).
+
+    In human (non-JSON) output mode we stream the command's output to the
+    console as it arrives and run it under a pseudoterminal so the remote
+    process flushes in real time — just like running it locally in a
+    terminal. Under --json the output is captured and returned as one
+    structured result, so we neither stream nor allocate a PTY (a PTY
+    would merge stdout/stderr and inject \\r\\n into machine-read output).
+    """
+    if output._JSON_OUTPUT:
+        return None, None
+
+    def on_output(text: str) -> None:
+        sys.stdout.write(text)
+        sys.stdout.flush()
+
+    return on_output, _console_pty()
+
+
+def _console_pty() -> dict[str, int]:
+    """PTY dimensions matching the local terminal (fallback 80x24)."""
+    import shutil
+
+    try:
+        sz = shutil.get_terminal_size()
+        return {"cols": max(1, int(sz.columns)), "rows": max(1, int(sz.lines))}
+    except Exception:
+        return {"cols": 80, "rows": 24}
+
+
+class _ExecKeepalive:
+    """Periodic stderr heartbeat during a long foreground fcloud exec wait.
+
+    Tooling like the Claude Code Bash harness and many CI runners escalate
+    on no-output timers shorter than `fcloud exec --wait 8m`. Emitting a
+    short line every `interval_s` keeps those watchdogs happy without
+    polluting stdout (which carries the command's actual output / JSON).
+    """
+
+    def __init__(self, interval_s: float = 30.0) -> None:
+        import threading
+        self._interval = max(1.0, float(interval_s))
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._start_t = 0.0
+
+    def __enter__(self) -> "_ExecKeepalive":
+        import threading
+        import time
+        self._start_t = time.time()
+        self._thread = threading.Thread(
+            target=self._loop, name="fcloud-exec-keepalive", daemon=True,
+        )
+        self._thread.start()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=1.0)
+
+    def _loop(self) -> None:
+        import time
+        while not self._stop.wait(self._interval):
+            elapsed = int(time.time() - self._start_t)
+            sys.stderr.write(
+                f"  ⟳ still waiting on remote command ({elapsed}s)...\n",
+            )
+            sys.stderr.flush()
+
+
+def _maybe_keepalive(enabled: bool):
+    """Context manager that activates the stderr heartbeat only when
+    long-wait runs would otherwise trip no-output watchdogs."""
+    import contextlib
+    if not enabled:
+        return contextlib.nullcontext()
+    return _ExecKeepalive()
+
+
+def _exec_result(result, *, streamed: bool = False,
+                 still_running_exit: int = 75) -> None:
+    """Print an ExecResult and exit with its return code.
+
+    When ``streamed`` is True the output was already written to the
+    console live (see ``_live_console_sink``), so we skip re-printing it
+    here and only surface status/return code.
+    """
+    if output._JSON_OUTPUT:
+        output._out({
+            "stdout": result.stdout,
+            "stderr": result.stderr,
+            "returncode": result.returncode,
+            "duration_ms": result.duration_ms,
+            "status": result.status,
+            "process_id": result.process_id,
+            "session_id": result.session_id,
+            "message": result.message,
+            "stdout_truncated": result.stdout_truncated,
+            "stdout_total_bytes": result.stdout_total_bytes,
+            "stdout_retained_bytes": result.stdout_retained_bytes,
+            "stdout_range": result.stdout_range,
+            "output_dropped_bytes": result.output_dropped_bytes,
+        })
+    else:
+        if not streamed:
+            if result.stdout:
+                sys.stdout.write(result.stdout)
+            if result.stderr:
+                sys.stderr.write(result.stderr)
+            if result.stdout_truncated:
+                print(_stdout_truncation_warning(result), file=sys.stderr)
+        if result.output_dropped_bytes:
+            print(
+                "Warning: command output is incomplete: "
+                f"{result.output_dropped_bytes} byte(s) were dropped in transport.",
+                file=sys.stderr,
+            )
+        if result.status in ("running", "migrating"):
+            msg = result.message or (
+                "Command is still running. "
+                f"Follow logs: fcloud logs {result.session_id} "
+                f"{result.process_id} --follow"
+            )
+            print(msg, file=sys.stderr)
+        elif result.status in ("host_terminated", "host_lost"):
+            msg = result.message or (
+                "Host was interrupted; the process did not survive. "
+                f"Fetch any captured output: fcloud logs {result.session_id} "
+                f"{result.process_id} --output all"
+            )
+            print(msg, file=sys.stderr)
+    if result.returncode is None:
+        # Distinguish "still running, no return code yet" from "host died,
+        # the process was killed" (exit 143 = SIGTERM-style, so shell
+        # scripts and `set -e` callers don't silently treat eviction as
+        # success). host_lost reaching here means a migration ride wasn't
+        # wired for this path — treat it as a lost host (143), never a
+        # false success.
+        if result.status in ("host_terminated", "host_lost"):
+            sys.exit(143)
+        # "migrating": the ride ran out of --wait budget mid-restore. The
+        # process is neither confirmed alive nor dead, so exit retryable
+        # (75, EX_TEMPFAIL — the same convention as `fcloud wait` timeouts).
+        if result.status == "migrating":
+            sys.exit(75)
+        if result.status == "running":
+            # An expired wait exits 75 like `fcloud wait` does, so automation
+            # can't mistake an unfinished command for success (a fan-out
+            # once failed hundreds of healthy tasks on exactly that). Callers
+            # that explicitly did NOT wait (exec --wait 0 fire-and-forget)
+            # pass still_running_exit=0: "started" is their success.
+            sys.exit(still_running_exit)
+        sys.exit(0)
+    if result.returncode < 0:
+        # A negative code means the runtime lost the process to a signal —
+        # in practice the OOM killer. Passing it to sys.exit() wrapped it to
+        # 255, an exit code that means nothing to anyone: a training script
+        # dying at step 8 with "255" and no other clue is what that looks
+        # like. 137 (128+SIGKILL) is what a shell reports for a killed
+        # process, so `set -e` scripts and agents can branch on it. The
+        # reason, when the host knows it, arrives separately as the
+        # workload_oom session notice.
+        print(
+            "The command was killed before it could report an exit code "
+            "(exit 137). If this was memory, `fcloud logs` names the cause.",
+            file=sys.stderr,
+        )
+        sys.exit(137)
+    sys.exit(result.returncode)
+
+
+def _pid_emitter(path: str, session_id: str):
+    """Build an on_start callback that records the process id to ``path`` the
+    instant the command starts — before the blocking wait.
+
+    This is the pipe- and background-proof channel the detach message can't
+    be: `fcloud exec --wait 60m ... 2>&1 | tail` buffers all stdio until EOF and
+    a backgrounded shell swallows it, so the proc-id needed for `fcloud wait` /
+    `fcloud logs --follow` never surfaces. The file is written independently of
+    stdout/stderr, so it's readable immediately even mid-wait. Written
+    atomically (temp + rename) so a reader never sees a half-written id.
+    """
+    def _emit(pid: str) -> None:
+        payload = _json.dumps({"session_id": session_id, "process_id": pid})
+        tmp = f"{path}.tmp"
+        try:
+            # Create missing parent dirs so a mkdir failure lands in the same
+            # warn-and-continue branch as a write failure.
+            os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+            with open(tmp, "w") as fh:
+                fh.write(payload + "\n")
+            os.replace(tmp, path)
+        except OSError as exc:
+            # A bad --emit-pid path shouldn't sink the actual command; warn
+            # and carry on (the proc still runs and is waitable by other means).
+            sys.stderr.write(f"fcloud: could not write --emit-pid {path}: {exc}\n")
+    return _emit
+
+
+def _stdout_truncation_warning(result) -> str:
+    logs_cmd = (
+        f" If the session is still active, fetch full log: fcloud logs {result.session_id} "
+        f"{result.process_id} --output all."
+        if result.session_id and result.process_id else ""
+    )
+    return (
+        "Warning: stdout truncated to "
+        f"{result.stdout_range} {result.stdout_retained_bytes} of "
+        f"{result.stdout_total_bytes} bytes. Rerun with --output all, "
+        "or choose --output head|middle|tail --output-bytes N."
+        f"{logs_cmd}"
+    )

fcloud/cli/context.py

@@ -0,0 +1,161 @@
+"""Local context: fcloud.json project defaults and API credentials.
+
+Resolves what a command needs from the working directory (project
+config, volumes, disk floor) and from the saved token/URL.
+"""
+
+import sys
+
+from fcloud.cli.common import _parse_volume_mount
+from fcloud.client import Client
+from fcloud.config import image_from_config, load_project_config
+
+
+def _resolve_project(client: Client, sku: str = ""):
+    """Build a project from fcloud.json if present, else use defaults.
+
+    `sku` is the CLI-provided override (e.g. `--sku cpu_8x_graviton`).
+    When present, it takes precedence over `cfg["sku"]` for image
+    construction — without this, an fcloud.json declaring a GPU sku
+    would force the pytorch wheel index even when the user explicitly
+    overrides to a CPU sku at the CLI.
+    """
+    cfg = load_project_config()
+    default_sku = cfg.get("sku", "")
+    image = image_from_config(cfg, sku=sku or default_sku)
+    if image:
+        project = client.project("_cli", image=image)
+    else:
+        project = client.project("_cli")
+    return project, default_sku
+
+
+def _resolve_min_disk_gb(cli_value: int) -> int:
+    """CLI --min-disk-gb wins; else fall back to fcloud.json `min_disk_gb`.
+
+    Mirrors how --sku overrides cfg["sku"], so a project can declare a disk
+    floor once in fcloud.json and skip the flag on every command.
+    """
+    if cli_value > 0:
+        return cli_value
+    try:
+        return int(load_project_config().get("min_disk_gb", 0) or 0)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _normalize_config_volume(entry: object) -> dict[str, str] | None:
+    """Normalize one fcloud.json `volumes` entry to {name[, mount_path]}.
+
+    Accepts a string ("name" or "name:/mount") or an object with `name`
+    and optional `mount`/`mount_path`.
+    """
+    if isinstance(entry, str):
+        try:
+            return _parse_volume_mount(entry)
+        except ValueError:
+            return None
+    if isinstance(entry, dict):
+        name = str(entry.get("name", "")).strip()
+        if not name:
+            return None
+        out = {"name": name}
+        mount = entry.get("mount_path") or entry.get("mount") or ""
+        if mount:
+            out["mount_path"] = str(mount).strip()
+        return out
+    return None
+
+
+def _resolve_volume_mounts(
+    cli_mounts: list[dict[str, str]],
+) -> list[dict[str, str]]:
+    """Merge fcloud.json `volumes` defaults with CLI --volume flags.
+
+    Effective set = config volumes ∪ CLI flags, keyed by name; a CLI
+    --volume for a name already in fcloud.json re-maps its mount path for
+    that invocation. Mirrors how --sku overrides cfg["sku"], so a project
+    can declare its volumes once and skip --volume on every command.
+    """
+    cfg = load_project_config()
+    order: list[str] = []
+    by_name: dict[str, dict[str, str]] = {}
+    for entry in cfg.get("volumes", []) or []:
+        mount = _normalize_config_volume(entry)
+        if mount and mount["name"] not in by_name:
+            order.append(mount["name"])
+            by_name[mount["name"]] = mount
+    cli_seen: dict[str, dict[str, str]] = {}
+    for mount in cli_mounts:
+        name = mount["name"]
+        # A CLI flag overriding a fcloud.json default is the re-map feature;
+        # two CLI flags naming the same volume differently is a conflict —
+        # a volume mounts once per session, and silently keeping only the
+        # last flag hid that (BUG-14, 2026-08-20 report).
+        prior = cli_seen.get(name)
+        if prior is not None and prior != mount:
+            print(
+                f"Error: duplicate_volume — volume {name!r} can only be "
+                f"mounted once per session (asked for "
+                f"{prior.get('mount_path') or '/workspace/' + name} and "
+                f"{mount.get('mount_path') or '/workspace/' + name}).",
+                file=sys.stderr)
+            sys.exit(2)
+        cli_seen[name] = mount
+        if name not in by_name:
+            order.append(name)
+        by_name[name] = mount  # CLI re-maps mount path
+    return [by_name[name] for name in order]
+
+
+def _current_image_spec_json(sku: str) -> str:
+    """Serialize the local fcloud.json image spec for resume re-baseline.
+
+    Returns "" when there's no project config, so callers fall through
+    to the dispatcher's stored spec (existing behavior). When set, the
+    dispatcher uses this on resume of a cold session — the edited
+    fcloud.json actually takes effect on next use.
+    """
+    cfg = load_project_config()
+    if not cfg:
+        return ""
+    image = image_from_config(cfg, sku=sku or cfg.get("sku", ""))
+    if image is None:
+        return ""
+    import json as _json
+    return _json.dumps(image.to_dict())
+
+
+def _api_base() -> tuple[str, dict]:
+    from fcloud.config import resolve_api_key, resolve_url
+
+    url = resolve_url(None)
+    key = resolve_api_key(None)
+    if not key:
+        print("Error: no API key. Run: fcloud set_token <key>", file=sys.stderr)
+        sys.exit(2)
+    return url, {"Authorization": f"Bearer {key}"}
+
+
+def _api_key() -> str:
+    """The resolved API key, for SDK helpers that take it directly."""
+    from fcloud.config import resolve_api_key
+
+    key = resolve_api_key(None)
+    if not key:
+        print("Error: no API key. Run: fcloud set_token <key>", file=sys.stderr)
+        sys.exit(2)
+    return key
+
+
+def _fs_base(session_id: str) -> tuple[str, dict]:
+    """Resolve (browse-endpoint-base, auth-headers) for a session's
+    read-only file tree on the dispatcher."""
+    from fcloud.config import resolve_api_key, resolve_url
+
+    url = resolve_url(None)
+    key = resolve_api_key(None)
+    if not key:
+        print("Error: no API key. Run: fcloud set_token <key>", file=sys.stderr)
+        sys.exit(2)
+    return f"{url}/v1/fs/{session_id}/", {"Authorization": f"Bearer {key}"}

fcloud/cli/exec_cmd.py

@@ -0,0 +1,201 @@
+"""`fcloud exec`: run one command on a GPU host in the foreground.
+
+(Named exec_cmd because `exec` shadows the builtin.)
+"""
+
+import sys
+import time
+
+from fcloud.cli.attach import _attach_with_retry, _exit_session_unreachable
+from fcloud.cli.common import (
+    _consume_positional_session,
+    _extract_flag,
+    _extract_min_disk,
+    _extract_on,
+    _extract_output_policy,
+    _extract_sku,
+    _extract_volumes,
+    _extract_wait,
+    _shape_exec_command,
+    _split_trailing_args,
+    _warn_sku_ignored_on_attach,
+)
+from fcloud.cli.console import (
+    _exec_result,
+    _live_console_sink,
+    _maybe_keepalive,
+    _pid_emitter,
+)
+from fcloud.cli.context import (
+    _current_image_spec_json,
+    _resolve_min_disk_gb,
+    _resolve_project,
+    _resolve_volume_mounts,
+)
+from fcloud.cli.migration import _ride_migration
+from fcloud.cli.registry import cmd
+from fcloud.client import Client
+
+_EXEC_LONG_HELP = """\
+Where it runs:
+  --sku SKU     make a throwaway session on that hardware, run, release it.
+                Its filesystem is still saved and downloadable afterwards.
+  --on SID      attach to an existing session (see `fcloud create`), run, detach.
+                The session stays alive; a cold one comes online first.
+  (neither)     a throwaway session on the fcloud.json sku, or the default.
+
+Commands run from /workspace — the same place `fcloud upload` puts your files —
+so an uploaded train.py runs as `python train.py`. Don't hardcode /root paths.
+For shell syntax (pipes, &&, redirects) pass one quoted expression or use
+`sh -c`; unquoted metacharacters are eaten by your LOCAL shell first.
+
+--wait bounds the foreground wait (default 30s). Past the deadline the process
+keeps running and exec exits 75 — pick it up with `fcloud logs <sid> <pid>` and
+`fcloud wait`, or start it detached with `fcloud spawn` in the first place.
+
+--volume NAME[:MOUNT] mounts a volume for this command (repeatable) — see
+`fcloud help volume`.
+
+Examples:
+  fcloud exec --sku gpu_1x_l4 nvidia-smi -L
+  fcloud exec --on s-abc123 --wait 10m python train.py
+  fcloud exec --sku gpu_1x_l4 --volume checkpoints python train.py
+  fcloud exec --on s-abc123 -- sh -c 'cd data && ls | head'"""
+
+
+@cmd("exec",
+     "fcloud exec [--sku SKU] [--on SID|SID] [--volume NAME[:MOUNT]] "
+     "[--wait 30s] [--output MODE] <cmd...>",
+     "Run a command on a GPU host",
+     long_help=_EXEC_LONG_HELP)
+def cmd_exec(args: list[str]) -> None:
+    (sku, on_sid, wait_s, output_range, output_limit,
+     volume_mounts, min_disk_gb, emit_pid, cmd_args) = _parse_exec_args(args)
+    if not cmd_args:
+        print("Usage: fcloud exec [--sku SKU] [--on SESSION|SESSION] "
+              "[--volume NAME[:MOUNT]] [--wait 30s] "
+              "[--emit-pid FILE] "
+              "[--output tail|head|middle|all] "
+              "[--output-bytes N] <command...>")
+        print("\nExamples:")
+        print("  fcloud exec nvidia-smi -L")
+        print("  fcloud exec --sku gpu_1x_l4 python train.py")
+        print("  fcloud exec --on s-abc123 --wait 10m python train.py")
+        print("  fcloud exec --volume checkpoints:/workspace/ckpt python train.py")
+        print("  fcloud exec --json --output all python cuda_probe.py")
+        print("  fcloud exec --output head --output-bytes 128k python noisy.py")
+        print("  fcloud exec s-abc123 --wait 10m -- python train.py")
+        print("  fcloud exec --on s-abc123 --emit-pid pid.txt --wait 10m python train.py")
+        print("  fcloud exec --on s-abc123 python train.py")
+        sys.exit(1)
+    cmd_args = _shape_exec_command(cmd_args)
+    volume_mounts = _resolve_volume_mounts(volume_mounts)
+    on_output, pty = _live_console_sink()
+    # Long --wait blocks the foreground; no-output watchdogs (CI runners,
+    # Claude Code Bash) escalate without periodic stderr activity. Only
+    # arm the heartbeat for waits long enough to risk a watchdog kill, and
+    # only when stdout isn't already streaming (the stream itself is the
+    # heartbeat in human mode).
+    use_keepalive = wait_s >= 60 and on_output is None
+    client = Client()
+    if on_sid:
+        _warn_sku_ignored_on_attach(client, on_sid, sku)
+        # Forward the local fcloud.json spec on attach so a cold session's
+        # next resume re-baselines against the current config. Without
+        # this, sessions stay pinned to the spec they were first created
+        # with and edits to fcloud.json silently never apply.
+        attach_spec = _current_image_spec_json(sku)
+        on_start = _pid_emitter(emit_pid, on_sid) if emit_pid else None
+        try:
+            # Attach can blip in the setup/binding window right after a host
+            # first reports ready — the session isn't bound to its host yet, so
+            # the very first exec fails transiently and self-heals in seconds.
+            # Retry the (idempotent) attach before giving up; run() is issued
+            # once, outside the retry, so a command never double-executes.
+            session = _attach_with_retry(
+                client, on_sid, volume_mounts, attach_spec, sku)
+            try:
+                # The --wait budget covers the whole foreground command,
+                # including any migration ride after the run itself returns.
+                deadline = time.monotonic() + wait_s if wait_s > 0 else None
+                with _maybe_keepalive(use_keepalive):
+                    result = session.run(
+                        cmd_args,
+                        timeout_ms=wait_s * 1000,
+                        output_limit_bytes=output_limit,
+                        output_range=output_range,
+                        on_output=on_output,
+                        on_start=on_start,
+                        pty=pty,
+                    )
+                result = _ride_migration(client, result, on_output, deadline)
+                # --wait 0 is explicit fire-and-forget: "launched OK" is
+                # success (0). Any positive wait that expires exits 75.
+                _exec_result(result, streamed=on_output is not None,
+                             still_running_exit=0 if wait_s == 0 else 75)
+            finally:
+                client.detach_session(on_sid)
+        except Exception as exc:
+            # Attach/run raised and couldn't recover. This fires for an initial
+            # attach that never landed, or a run whose host vanished before
+            # _ride_migration could re-attach. Ask the dispatcher what actually
+            # happened and report the true cause — we only claim a migration
+            # when one is real, never as a blanket label for every blip.
+            _exit_session_unreachable(client, on_sid, exc)
+    else:
+        project, default_sku = _resolve_project(client, sku=sku)
+        sku = sku or default_sku
+        session = project.session(
+            sku=sku, volume_mounts=volume_mounts,
+            min_disk_gb=_resolve_min_disk_gb(min_disk_gb),
+        )
+        print(f"Session: {session.session_id}", file=sys.stderr)
+        on_start = _pid_emitter(emit_pid, session.session_id) if emit_pid else None
+        result = None
+        try:
+            # The --wait budget covers the whole foreground command,
+            # including any migration ride after the run itself returns.
+            deadline = time.monotonic() + wait_s if wait_s > 0 else None
+            with _maybe_keepalive(use_keepalive):
+                result = session.run(
+                    cmd_args,
+                    timeout_ms=wait_s * 1000,
+                    output_limit_bytes=output_limit,
+                    output_range=output_range,
+                    on_output=on_output,
+                    on_start=on_start,
+                    pty=pty,
+                )
+            result = _ride_migration(client, result, on_output, deadline)
+        finally:
+            # Soft-release once the command is done: the host tears the
+            # ephemeral session down only if nothing else is using it,
+            # freeing the host promptly for the common one-shot case. If
+            # the command is still running — or still riding a restore
+            # ("migrating": releasing would tear down the session the
+            # restore is bringing back) — detach instead.
+            if result is None or result.status not in ("running", "migrating"):
+                client.release_session(session.session_id)
+            else:
+                client.detach_session(session.session_id)
+        _exec_result(result, streamed=on_output is not None,
+                     still_running_exit=0 if wait_s == 0 else 75)
+
+
+def _parse_exec_args(
+    args: list[str],
+) -> tuple[str, str, int, str, int | None, list[dict[str, str]], int, str, list[str]]:
+    args, trailing = _split_trailing_args(args)
+    volume_mounts, args = _extract_volumes(args)
+    sku, remaining = _extract_sku(args)
+    min_disk_gb, remaining = _extract_min_disk(remaining)
+    on_sid, remaining = _extract_on(remaining)
+    wait_s, remaining = _extract_wait(remaining)
+    emit_pid, remaining = _extract_flag(remaining, "--emit-pid")
+    output_range, output_limit, remaining = _extract_output_policy(remaining)
+    if not on_sid:
+        on_sid, remaining = _consume_positional_session(remaining)
+    return (
+        sku, on_sid, wait_s, output_range, output_limit,
+        volume_mounts, min_disk_gb, emit_pid, remaining + trailing,
+    )