fcloud-sdk 0.1.0__py3-none-any.whl

fcloud/cli/job.py

@@ -0,0 +1,300 @@
+"""`fcloud job`: run a command on an ephemeral session (no saved filesystem)."""
+
+import sys
+import time
+
+from fcloud.cli import output
+from fcloud.cli.common import (
+    _extract_flag,
+    _extract_secrets,
+    _extract_upload_options,
+    _parse_duration_seconds,
+    _parse_min_disk,
+    _parse_volume_mount,
+    _reject_volume_upload_collision,
+    _shape_exec_command,
+)
+from fcloud.cli.context import (
+    _resolve_min_disk_gb,
+    _resolve_project,
+    _resolve_volume_mounts,
+)
+from fcloud.cli.processes import cmd_logs
+from fcloud.cli.registry import cmd
+from fcloud.cli.sessions import _session_state
+from fcloud.cli.wait import cmd_wait
+from fcloud.client import Client
+
+_JOB_LONG_HELP = """\
+A job runs a command to completion on an EPHEMERAL session: its filesystem
+is the image + --include files + volumes, and it is never persisted — the
+session cannot be resumed, mounted, or downloaded after it closes. Write
+durable outputs to a volume (--volume) or download them while the job runs;
+the exit code and logs always survive (fcloud job logs).
+
+Jobs are never checkpoint/migrated: if the host is lost mid-run (spot
+preemption), --retries N re-runs the command on a fresh session.
+
+--include PATH uploads a file or directory before the command starts (same
+rules as `fcloud upload`: hidden entries, dependency dirs and symlinked files
+are skipped unless --include-hidden / --follow-symlinks say otherwise).
+
+`job run` returns once the command has exited AND its volumes are
+committed — read them the moment it returns. If the host is unreachable at
+that point the commit is deferred to its teardown replay, and the command
+says so instead of returning a silent lie.
+
+Examples:
+  fcloud job run --sku gpu_1x_l4 --volume results -- python train.py
+  fcloud job run --include ./src --env RUN=7 --detach -- python src/gen.py
+  fcloud job ls
+  fcloud job logs s-abc123            # durable output, live or after close
+  fcloud job wait s-abc123            # block until the job ends
+  fcloud job kill s-abc123
+
+Sweep management moved to `fcloud sweep <status|logs|retry|cancel|wait>`."""
+
+
+_JOB_SUBCOMMANDS = ("run", "ls", "logs", "wait", "kill")
+
+
+@cmd("job", "fcloud job <run|ls|logs|wait|kill> …",
+     "Run a job: ephemeral session, no saved filesystem",
+     long_help=_JOB_LONG_HELP)
+def cmd_job(args: list[str]) -> None:
+    if not args:
+        print("Usage: fcloud job <run|ls|logs|wait|kill> …  "
+              "(see `fcloud help job`)", file=sys.stderr)
+        sys.exit(2)
+    sub, rest = args[0], args[1:]
+    if sub == "run":
+        _job_run(rest)
+    elif sub == "ls":
+        _job_ls(rest)
+    elif sub == "logs":
+        _job_logs(rest)
+    elif sub == "wait":
+        _job_wait(rest)
+    elif sub == "kill":
+        _job_kill(rest)
+    else:
+        print(f"Unknown job subcommand {sub!r}. Jobs: fcloud job "
+              f"<run|ls|logs|wait|kill>. Sweeps: fcloud sweep <name>.",
+              file=sys.stderr)
+        sys.exit(2)
+
+
+def _job_run(args: list[str]) -> None:
+    try:
+        secrets, args = _extract_secrets(args)
+    except ValueError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(2)
+    upload_opts, args = _extract_upload_options(args)
+    sku, min_disk, retries, detach = "", 0, 0, False
+    volumes: list[dict[str, str]] = []
+    include: list[str] = []
+    env: dict[str, str] = {}
+    cmd_args: list[str] = []
+    i = 0
+    while i < len(args):
+        a = args[i]
+        if a == "--":
+            cmd_args = args[i + 1:]
+            break
+        if a == "--sku" and i + 1 < len(args):
+            sku = args[i + 1]
+            i += 2
+        elif a == "--volume" and i + 1 < len(args):
+            volumes.append(_parse_volume_mount(args[i + 1]))
+            i += 2
+        elif a == "--include" and i + 1 < len(args):
+            include.append(args[i + 1])
+            i += 2
+        elif a == "--env" and i + 1 < len(args):
+            key, _, value = args[i + 1].partition("=")
+            env[key] = value
+            i += 2
+        elif a in ("--min-disk-gb", "--min-disk") and i + 1 < len(args):
+            try:
+                min_disk = _parse_min_disk(args[i + 1])
+            except ValueError as exc:
+                print(f"Error: {exc}", file=sys.stderr)
+                sys.exit(2)
+            i += 2
+        elif a == "--retries" and i + 1 < len(args):
+            retries = int(args[i + 1])
+            i += 2
+        elif a == "--detach":
+            detach = True
+            i += 1
+        elif a.startswith("--"):
+            print(f"Error: unknown job run option {a!r} (put the command "
+                  f"after `--`)", file=sys.stderr)
+            sys.exit(2)
+        else:
+            cmd_args = args[i:]
+            break
+        continue
+    if not cmd_args:
+        print("Usage: fcloud job run [--sku SKU] [--volume NAME[:MOUNT]] "
+              "[--include PATH] [--include-hidden] [--follow-symlinks] "
+              "[--env K=V] [--secret KEY[=VALUE]] "
+              "[--retries N] [--detach] -- <command...>", file=sys.stderr)
+        sys.exit(2)
+    env.update(secrets)
+
+    client = Client()
+    project, default_sku = _resolve_project(client, sku=sku)
+    volume_mounts = _resolve_volume_mounts(volumes)
+    # --include uploads land at /workspace/<basename>: same collision
+    # hazard as `fcloud run` (BUG-1) — a mount there absorbs the upload.
+    from pathlib import Path
+    for inc in include:
+        _reject_volume_upload_collision(volume_mounts, Path(inc).resolve())
+    job = project.job(
+        _shape_exec_command(cmd_args),
+        sku=sku or default_sku,
+        volume_mounts=volume_mounts or None,
+        include=include,
+        include_hidden=upload_opts.include_hidden,
+        follow_symlinks=upload_opts.follow_symlinks,
+        env=env or None,
+        retries=retries,
+        min_disk_gb=_resolve_min_disk_gb(min_disk),
+        # Teardown status on stderr: the close that commits the job's
+        # volumes runs after the command exits, and it is not instant.
+        on_progress=lambda line: print(line, file=sys.stderr),
+    )
+    print(f"Job {job.session_id} (pid {job.process_id}) — ephemeral: "
+          f"outputs go to volumes/logs.", file=sys.stderr)
+    if detach:
+        if output._JSON_OUTPUT:
+            output._out({"session_id": job.session_id,
+                  "process_id": job.process_id, "detached": True})
+        else:
+            print(f"Detached. Track: fcloud job logs {job.session_id} · "
+                  f"fcloud job wait {job.session_id}", file=sys.stderr)
+        return
+    try:
+        for event in job.watch():
+            data = event.get("data", "")
+            if data:
+                sys.stdout.write(data)
+                sys.stdout.flush()
+    except KeyboardInterrupt:
+        print(f"\nDetached — the job keeps running. Track: "
+              f"fcloud job wait {job.session_id}", file=sys.stderr)
+        return
+    log = job.wait(poll_interval=2)
+    if job.attempts > 1:
+        print(f"(re-ran {job.attempts - 1} time(s) after host loss)",
+              file=sys.stderr)
+    if output._JSON_OUTPUT:
+        output._out({"session_id": job.session_id, "process_id": job.process_id,
+              "exit_code": log.exit_code, "status": job.status,
+              "attempts": job.attempts, "volume_sync": job.volume_sync})
+    if log.exit_code is None:
+        print("Job ended without an exit code (host lost; retries "
+              "exhausted).", file=sys.stderr)
+        sys.exit(1)
+    sys.exit(log.exit_code)
+
+
+def _job_ls(args: list[str]) -> None:
+    show_all = "--all" in args
+    limit, _rest = _extract_flag([a for a in args if a != "--all"], "--limit")
+    limit_n = int(limit) if limit else 0
+    client = Client()
+    jobs = [s for s in client.list_sessions(status="all") if s.ephemeral]
+    if not show_all:
+        jobs = [s for s in jobs
+                if _session_state(s) not in
+                {"cold", "ended", "failed", "dead"}]
+    jobs.sort(key=lambda s: (s.last_active or s.created_at or ""),
+              reverse=True)
+    if limit_n > 0:
+        jobs = jobs[:limit_n]
+    if output._JSON_OUTPUT:
+        output._out({"jobs": [
+            {"session_id": s.session_id, "status": s.status,
+             "state": _session_state(s), "lifecycle": s.lifecycle,
+             "created_at": s.created_at}
+            for s in jobs]})
+        return
+    if not jobs:
+        print("No jobs. Run one with: fcloud job run -- <cmd>"
+              + ("" if show_all else "  (--all includes finished jobs)"))
+        return
+    print(f"{'JOB':<22} {'STATE':<12} {'CREATED'}")
+    print("-" * 56)
+    for s in jobs:
+        print(f"{s.session_id:<22} {_session_state(s):<12} "
+              f"{s.created_at or ''}")
+
+
+def _job_logs(args: list[str]) -> None:
+    if not args:
+        print("Usage: fcloud job logs <job-id> [--follow]", file=sys.stderr)
+        sys.exit(2)
+    cmd_logs(args)
+
+
+def _job_wait(args: list[str]) -> None:
+    timeout, rest = _extract_flag(args, "--timeout")
+    if not rest:
+        print("Usage: fcloud job wait <job-id> [pid] [--timeout DUR]",
+              file=sys.stderr)
+        sys.exit(2)
+    if len(rest) >= 2:
+        cmd_wait(args)
+        return
+    # No pid: wait for the job's SESSION to reach a terminal state — the
+    # run-to-completion contract releases the session when the command
+    # exits, so "session ended" == "job done".
+    sid = rest[0]
+    deadline = (
+        time.monotonic() + _parse_duration_seconds(timeout)
+    ) if timeout else None
+    client = Client()
+    while True:
+        info = client.describe_session(sid)
+        stage = (info.lifecycle or info.status or "").lower()
+        if stage in ("closed", "failed", "dead"):
+            if output._JSON_OUTPUT:
+                output._out({"session_id": sid, "lifecycle": stage})
+            else:
+                print(f"Job {sid} ended ({stage}).")
+            sys.exit(0 if stage == "closed" else 1)
+        if deadline is not None and time.monotonic() >= deadline:
+            print(f"Job {sid} still {stage} after {timeout}.",
+                  file=sys.stderr)
+            sys.exit(1)
+        time.sleep(3)
+
+
+def _job_kill(args: list[str]) -> None:
+    if not args:
+        print("Usage: fcloud job kill <job-id>", file=sys.stderr)
+        sys.exit(2)
+    sid = args[0]
+    client = Client()
+    # Idempotent like `fcloud stop`: killing an already-ended job is the no-op
+    # the caller wanted — but say so instead of claiming a fresh kill.
+    try:
+        close_result = client.close_session(sid) or {}
+    except Exception as exc:
+        if "session_not_active" not in str(exc):
+            raise
+        close_result = {"status": "already_ended"}
+    status = close_result.get("status", "")
+    if output._JSON_OUTPUT:
+        output._out({"ok": True, "session_id": sid, "status": status})
+        return
+    if status == "already_ended":
+        print(f"Job {sid} already ended (no action needed).")
+    elif status in ("close_in_progress", "close_deferred"):
+        print(f"Job {sid} is already shutting down ({status}).")
+    else:
+        print(f"Job {sid} killed. Workspace discarded; volume commits and "
+              f"logs survive (fcloud job logs {sid}).")

fcloud/cli/main.py

@@ -0,0 +1,140 @@
+"""CLI entry point: global flags, dispatch, top-level error handling."""
+
+import sys
+
+from fcloud.cli import output
+from fcloud.cli.help import print_command_help, print_help
+from fcloud.cli.registry import COMMANDS
+from fcloud.cli_args import SPECS, ArgError, split
+from fcloud.client import PaymentOverdueError
+
+
+def _strip_precommand_json(rest: list[str], command: list[str]) -> list[str]:
+    """Drop a fcloud-level ``--json`` that appears before the command begins.
+
+    ``command`` is the opaque remote-command suffix from :func:`split` (empty
+    for non-passthrough commands). Everything from that boundary onward is the
+    user's own argv and is left untouched, so ``fcloud exec -- tool --json``
+    still passes ``--json`` to ``tool``.
+    """
+    boundary = len(rest) - len(command) if command else len(rest)
+    head = [t for t in rest[:boundary] if t != "--json"]
+    return head + rest[boundary:]
+
+
+def main():
+    argv = sys.argv[1:]
+    # A --json *before* the command name is fcloud's global flag; a --json after
+    # the command is parsed per-spec below (and belongs to a passthrough
+    # command's own argv when it falls after `--`).
+    while argv and argv[0] == "--json":
+        output._JSON_OUTPUT = True
+        argv = argv[1:]
+
+    if not argv or argv[0] in ("-h", "--help"):
+        print_help()
+        sys.exit(0 if argv else 1)
+
+    if argv[0] in ("--version", "-V", "version"):
+        from fcloud import __version__
+        print(f"fcloud {__version__}")
+        sys.exit(0)
+
+    name = argv[0]
+    if name == "help":
+        if len(argv) > 1 and argv[1] in COMMANDS:
+            print_command_help(argv[1])
+            sys.exit(0)
+        print_help()
+        sys.exit(0)
+
+    if name not in COMMANDS:
+        print(f"Unknown command: {name}")
+        print("Run 'fcloud help' for a list of commands.")
+        sys.exit(1)
+
+    if len(argv) > 1 and argv[1] in ("-h", "--help", "help"):
+        print_command_help(name)
+        sys.exit(0)
+
+    rest = argv[1:]
+    # Validate flags against the command's spec: reject an undeclared leading
+    # flag (the typo'd-flag class) and resolve --json without eating the user's own
+    # trailing --json. Handlers then parse the (json-stripped) rest themselves.
+    spec = SPECS.get(name)
+    if spec is not None:
+        try:
+            parsed = split(spec, rest)
+        except ArgError as exc:
+            if output._JSON_OUTPUT:
+                output._out({"error": str(exc)})
+            else:
+                print(f"Error: {exc}", file=sys.stderr)
+            sys.exit(exc.exit_code)
+        if parsed.flags.get("--json"):
+            output._JSON_OUTPUT = True
+        rest = _strip_precommand_json(rest, parsed.command)
+
+    handler, _, _ = COMMANDS[name]
+    try:
+        handler(rest)
+    except SystemExit:
+        raise
+    except PaymentOverdueError as e:
+        # A missed payment suspends the whole account: the key is rejected
+        # and running sessions are killed. Make the cause unmistakable so the
+        # user fixes billing instead of debugging a phantom auth failure.
+        if output._JSON_OUTPUT:
+            output._out({"error": str(e), "code": "payment_overdue"})
+        else:
+            print(f"Payment overdue: {e}", file=sys.stderr)
+            print(
+                "Your account is suspended for a missed payment. Complete the "
+                "outstanding payment in your billing dashboard to restore "
+                "access.",
+                file=sys.stderr,
+            )
+        sys.exit(3)
+    except KeyboardInterrupt:
+        # The command's own cleanup (kill remote process, release the
+        # session) has already run as the interrupt unwound through it;
+        # exit on the conventional SIGINT code without a noisy traceback.
+        print("Interrupted.", file=sys.stderr)
+        sys.exit(130)
+    except Exception as e:
+        # Best-effort client telemetry for an otherwise-invisible CLI crash
+        # the dispatcher can't see an exception that dies
+        # in the client process, so report it before surfacing it locally.
+        # Strictly fire-and-forget — report_client_event swallows everything.
+        _report_cli_error(e)
+        if output._JSON_OUTPUT:
+            output._out({"error": str(e)})
+            sys.exit(1)
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _report_cli_error(exc: Exception) -> None:
+    """Best-effort telemetry for an uncaught CLI error, then return.
+
+    Resolves the dispatcher URL + api key the same way the commands do and
+    fires a ``client_cli_error`` event. The error_code is the exception class
+    name and the detail is the (truncated) message — never a traceback, and
+    never the message of an OS/IO error, since those routinely embed local
+    filesystem paths. ``FCLOUD_TELEMETRY=0`` disables this along with every
+    other client event (enforced centrally in ``report_client_event``). Any
+    failure here is swallowed so telemetry never masks the original error.
+    """
+    try:
+        from fcloud import v2_connect
+        from fcloud.config import resolve_api_key, resolve_url
+
+        detail = "" if isinstance(exc, OSError) else str(exc)
+        v2_connect.report_client_event(
+            resolve_url(None), resolve_api_key(None),
+            event_type="client_cli_error",
+            error_code=type(exc).__name__,
+            detail=detail,
+        )
+    except Exception:
+        return