gpu4agents 0.5.0__py3-none-any.whl

gpu4agents/cli/setup.py

@@ -0,0 +1,164 @@
+"""`gpu setup` / `init` / `offers` - provisioning entry points."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from ..core.config import (
+    DEFAULT_CONFIG_TEMPLATE,
+    DEFAULT_CONFIG_TEMPLATE_FLAT,
+    CONFIG_FILENAMES,
+)
+from ..core.errors import JambuError
+from ._common import console, fmt_money, get_context, handle, print_json, register_project
+
+
+def init_command(
+    ctx: typer.Context,
+    model: str = typer.Option("empero-ai/Qwythos-9B-v2", "--model", help="HuggingFace model id."),
+    force: bool = typer.Option(False, "--force", help="Overwrite an existing config file."),
+    flat: bool = typer.Option(
+        False,
+        "--flat",
+        help=(
+            "Write a legacy, gpu-runtime-only config.yml (no gpu_runtime: "
+            "wrapper) instead of the shared jambu.yaml."
+        ),
+    ),
+) -> None:
+    """Write a starter jambu.yaml (and .env.example) in the current directory."""
+    if flat:
+        target = Path.cwd() / "config.yml"
+        content = DEFAULT_CONFIG_TEMPLATE_FLAT.format(model_id=model)
+    else:
+        target = Path.cwd() / CONFIG_FILENAMES[0]  # jambu.yaml
+        content = DEFAULT_CONFIG_TEMPLATE.format(model_id=model)
+
+    if target.exists() and not force:
+        console.print(Text(f"! {target} already exists (use --force)", style="yellow"))
+        raise typer.Exit(1)
+    target.write_text(content)
+    console.print(Text(f"+ wrote {target}", style="green"))
+    if not flat:
+        console.print(
+            Text(
+                "  gpu-runtime config lives under the gpu_runtime: key - other "
+                "Jambu Lab tools can share this file",
+                style="dim",
+            )
+        )
+
+    env_example = Path.cwd() / ".env.example"
+    if not env_example.exists():
+        env_example.write_text(
+            "# Credentials never belong in jambu.yaml / config.yml.\n"
+            "VAST_API_KEY=\n"
+            "# Optional, for gated HuggingFace models:\n"
+            "HF_TOKEN=\n"
+        )
+        console.print(Text(f"+ wrote {env_example}", style="green"))
+    console.print("next: export VAST_API_KEY=... && gpu validate")
+
+
+def setup_command(
+    ctx: typer.Context,
+    recreate: bool = typer.Option(
+        False, "--recreate", help="Replace an existing instance that no longer matches jambu.yaml."
+    ),
+    allow_cost_override: bool = typer.Option(
+        False, "--allow-cost-override", help="Provision even above budget.max_hourly_cost_usd."
+    ),
+    no_wait: bool = typer.Option(
+        False, "--no-wait", help="Return as soon as the instance is created."
+    ),
+) -> None:
+    """Provision the instance, install the watchdog and start the model server."""
+    context = get_context(ctx)
+    try:
+        engine = context.engine()
+        register_project(context)
+        state = engine.setup(
+            recreate=recreate, allow_cost_override=allow_cost_override, wait=not no_wait
+        )
+    except JambuError as exc:
+        handle(exc)
+        return
+
+    payload = engine.connection(state)
+    watchdog = state.watchdog.to_dict()
+    watchdog.pop("token", None)
+    payload["watchdog"] = watchdog
+    if context.json:
+        print_json(payload)
+        return
+    if state.endpoint:
+        console.print(
+            f"\n  endpoint  [bold]{state.endpoint}[/bold]\n"
+            f"  openai    [bold]{payload['base_url']}[/bold]\n"
+            f"  model     [bold]{payload['model']}[/bold]\n"
+            "  api key   [dim]gpu env[/dim]"
+        )
+    console.print("\nnext: gpu run python your_experiment.py")
+
+
+def offers_command(
+    ctx: typer.Context,
+    limit: int = typer.Option(10, "--limit", "-n", help="How many offers to show."),
+) -> None:
+    """Show matching capacity and its price before committing to it."""
+    context = get_context(ctx)
+    try:
+        engine = context.engine()
+        engine.provider.require_capability("pricing_query")
+        spec = engine.build_spec()
+        offers = engine.provider.find_offers(spec, limit=limit)
+    except JambuError as exc:
+        handle(exc)
+        return
+
+    rows = [
+        {
+            "id": o.id,
+            "gpu": o.gpu_name,
+            "count": o.gpu_count,
+            "vram_gb": o.gpu_vram_gb,
+            "disk_gb": o.disk_gb,
+            "hourly_cost_usd": o.hourly_cost_usd,
+            "region": o.region,
+            "cuda_max": o.cuda_max,
+        }
+        for o in offers
+    ]
+    if context.json:
+        print_json({"offers": rows})
+        return
+
+    if not rows:
+        console.print(Text("no matching offers", style="yellow"))
+        return
+
+    ceiling = engine.config.budget.max_hourly_cost_usd
+    table = Table(title=f"{engine.provider.display_name} offers", title_justify="left")
+    for column in ("OFFER", "GPU", "N", "VRAM", "DISK", "$/H", "REGION"):
+        table.add_column(column)
+    for row in rows:
+        over = ceiling is not None and row["hourly_cost_usd"] > float(ceiling)
+        table.add_row(
+            row["id"],
+            row["gpu"],
+            str(row["count"]),
+            f"{row['vram_gb']:.0f} GB",
+            f"{row['disk_gb']:.0f} GB",
+            Text(
+                fmt_money(row["hourly_cost_usd"]) or "-",
+                style="red" if over else "green",
+            ),
+            row["region"] or "-",
+        )
+    console.print(table)
+    if ceiling:
+        console.print(Text(f"  budget ceiling: ${float(ceiling):.3f}/h", style="dim"))

gpu4agents/cli/status.py

@@ -0,0 +1,224 @@
+"""`gpu status` / `logs` / `events` - reconciled observability."""
+
+from __future__ import annotations
+
+import time
+
+import typer
+from rich.table import Table
+from rich.text import Text
+
+from ..core.durations import format_duration
+from ..core.errors import JambuError
+from ._common import (
+    console,
+    fmt_money,
+    fmt_time,
+    get_context,
+    handle,
+    kv_table,
+    print_json,
+    state_text,
+)
+
+
+def status_command(
+    ctx: typer.Context,
+    watch: float = typer.Option(
+        0, "--watch", "-w", help="Refresh every N seconds (0 = once)."
+    ),
+) -> None:
+    """Reconcile local and provider state, then report the normalized result."""
+    context = get_context(ctx)
+    while True:
+        try:
+            payload = context.engine().status()
+        except JambuError as exc:
+            handle(exc)
+            return
+
+        if context.json:
+            print_json(payload)
+            return
+
+        _render(payload)
+        if not watch:
+            return
+        time.sleep(max(2.0, watch))
+        console.rule(style="dim")
+
+
+def _render(payload: dict) -> None:
+    decision = payload.get("decision") or {}
+    watchdog = payload.get("watchdog") or {}
+    rows = [
+        ("state", state_text(payload.get("state", "unknown"))),
+        ("provider", payload.get("provider")),
+        ("instance", payload.get("instance_id") or "-"),
+        ("model", payload.get("model")),
+        ("endpoint", payload.get("endpoint") or "-"),
+        ("created", fmt_time(payload.get("created_at"))),
+        (
+            "age",
+            format_duration(payload.get("age_seconds"))
+            if payload.get("age_seconds") is not None
+            else None,
+        ),
+        ("idle", format_duration(payload.get("idle_seconds"))),
+        ("cost", _cost(payload)),
+        ("locked", _lock(payload)),
+        (
+            "watchdog",
+            _watchdog(watchdog),
+        ),
+        ("guard", _guard(decision, payload)),
+    ]
+    console.print(kv_table("Runtime", rows))
+
+    workloads = [w for w in payload.get("workloads", []) if w["state"] in ("running", "queued")]
+    recent = sorted(
+        payload.get("workloads", []), key=lambda w: w.get("started_at") or 0, reverse=True
+    )[:5]
+    if recent:
+        table = Table(title="Workloads", title_justify="left", box=None)
+        for column in ("ID", "STATE", "LABEL", "STARTED", "RC"):
+            table.add_column(column)
+        for record in recent:
+            table.add_row(
+                record["id"],
+                Text(
+                    record["state"],
+                    style="cyan" if record["state"] == "running" else "dim",
+                ),
+                (record.get("label") or "")[:48],
+                fmt_time(record.get("started_at")) or "-",
+                "-" if record.get("returncode") is None else str(record["returncode"]),
+            )
+        console.print(table)
+    if workloads:
+        console.print(
+            Text(f"  {len(workloads)} active workload(s) hold the instance open", style="cyan")
+        )
+
+
+def _cost(payload: dict) -> str:
+    hourly = payload.get("hourly_cost_usd")
+    session = payload.get("session_cost_usd")
+    if hourly is None and session is None:
+        return "-"
+    parts = []
+    if hourly is not None:
+        parts.append(f"{fmt_money(hourly)}/h")
+    if session:
+        parts.append(f"{fmt_money(session)} this session")
+    return "  ".join(parts) or "-"
+
+
+def _lock(payload: dict) -> Text:
+    if not payload.get("locked"):
+        return Text("no", style="dim")
+    until = fmt_time(payload.get("locked_until"))
+    return Text(f"yes (until {until})" if until else "yes (indefinite)", style="yellow")
+
+
+def _watchdog(watchdog: dict) -> Text:
+    if not watchdog.get("enabled"):
+        return Text("disabled - shutdown depends on this CLI staying alive", style="yellow")
+    if watchdog.get("reachable"):
+        return Text(f"live at {watchdog.get('url')}", style="green")
+    if watchdog.get("installed"):
+        return Text(
+            f"unreachable ({watchdog.get('last_error') or 'no answer'})", style="yellow"
+        )
+    return Text("not installed", style="yellow")
+
+
+def _guard(decision: dict, payload: dict) -> Text:
+    action = decision.get("action", "none")
+    reason = decision.get("reason") or "-"
+    if action == "none":
+        policy = payload.get("policy") or {}
+        idle_timeout = policy.get("idle_timeout_s")
+        idle = payload.get("idle_seconds") or 0
+        if idle_timeout and reason == "within_policy":
+            remaining = max(0.0, float(idle_timeout) - float(idle))
+            return Text(
+                f"ok ({reason}); auto-stop in {format_duration(remaining)} if idle",
+                style="green",
+            )
+        if (decision.get("detail") or {}).get("idle_skipped"):
+            return Text(f"ok ({reason}); idle left to the unreachable watchdog", style="yellow")
+        return Text(f"ok ({reason})", style="green")
+    return Text(f"{action} pending: {reason}", style="bold yellow")
+
+
+def logs_command(
+    ctx: typer.Context,
+    lines: int = typer.Option(200, "--lines", "-n", help="How many lines to show."),
+    source: str = typer.Option(
+        "",
+        "--source",
+        "-s",
+        help="runtime | onstart | watchdog (read on the instance through the "
+        "watchdog) or provider (container log). Default: runtime when the "
+        "watchdog answers, else provider.",
+    ),
+) -> None:
+    """Show the instance's logs: the model server's by default."""
+    context = get_context(ctx)
+    try:
+        used, output = context.engine().read_logs(limit=lines, source=source or None)
+    except JambuError as exc:
+        handle(exc)
+        return
+    if context.json:
+        print_json({"source": used, "lines": output})
+        return
+    for line in output:
+        console.print(line, markup=False, highlight=False)
+
+
+def events_command(
+    ctx: typer.Context,
+    limit: int = typer.Option(40, "--limit", "-n", help="How many events to show."),
+    event: str = typer.Option("", "--event", help="Filter by event name."),
+) -> None:
+    """Show the structured lifecycle event log (spec section 24)."""
+    context = get_context(ctx)
+    try:
+        engine = context.engine()
+    except JambuError as exc:
+        handle(exc)
+        return
+    records = engine.events.tail(limit=limit, event=event or None)
+    if context.json:
+        print_json({"events": records})
+        return
+    if not records:
+        console.print(Text("no events recorded yet", style="dim"))
+        return
+    table = Table(title="Lifecycle events", title_justify="left", box=None)
+    for column in ("TS", "EVENT", "DETAIL"):
+        table.add_column(column)
+    for record in records:
+        detail = {
+            k: v
+            for k, v in record.items()
+            if k not in ("ts", "event", "pid", "provider", "source")
+        }
+        table.add_row(
+            str(record.get("ts", "")),
+            Text(str(record.get("event", "")), style=_event_style(record.get("event", ""))),
+            ", ".join(f"{k}={v}" for k, v in detail.items())[:110],
+        )
+    console.print(table)
+
+
+def _event_style(event: str) -> str:
+    if "error" in event or "failed" in event:
+        return "red"
+    if "guard" in event or "stopped" in event or "destroyed" in event:
+        return "yellow"
+    if "ready" in event or "created" in event:
+        return "green"
+    return "white"

gpu4agents/cli/timer.py

@@ -0,0 +1,143 @@
+"""systemd *user* units that run `gpu sweep --apply` on a timer.
+
+One ``ExecStart`` covers every project: the sweep guards each project in
+``$GPU_HOME/projects.json`` with its own config, so the unit does not depend
+on its working directory. A user unit needs no root, but it does not inherit
+the shell's environment: credentials must live in ``~/.gpu/credentials`` or
+the project ``.env``. Writing the files is all this module does; enabling them
+is left to the printed ``systemctl`` command.
+"""
+
+from __future__ import annotations
+
+import os
+import shlex
+import sys
+from pathlib import Path
+from typing import Optional
+
+UNIT_NAME = "jambu-gpu-sweep"
+# A stuck provider API must not wedge the oneshot (and with it every later run).
+START_TIMEOUT = "10min"
+
+
+def unit_dir() -> Path:
+    base = os.environ.get("XDG_CONFIG_HOME")
+    root = Path(base).expanduser() if base else Path.home() / ".config"
+    return root / "systemd" / "user"
+
+
+def unit_paths() -> tuple[Path, Path]:
+    directory = unit_dir()
+    return directory / f"{UNIT_NAME}.service", directory / f"{UNIT_NAME}.timer"
+
+
+def _specifiers(value: str) -> str:
+    # systemd expands %h, %u, ... in unit values; a literal % must be doubled.
+    return value.replace("%", "%%")
+
+
+def _quote(arg: str) -> str:
+    """One ExecStart= argument: % and $ escaped, quoted when it needs it."""
+    arg = _specifiers(arg).replace("$", "$$")
+    if not arg or any(c in arg for c in " \t\"'\\;"):
+        return '"' + arg.replace("\\", "\\\\").replace('"', '\\"') + '"'
+    return arg
+
+
+def _environment(name: str, value: str) -> str:
+    text = _specifiers(f"{name}={value}").replace("\\", "\\\\").replace('"', '\\"')
+    return f'Environment="{text}"'
+
+
+def render_units(
+    *, every_s: float, python: str, jambu_home: Optional[str] = None
+) -> tuple[str, str]:
+    lines = [
+        "# Generated by `gpu sweep --install-timer`. Remove with --uninstall-timer.",
+        "[Unit]",
+        "Description=gpu4agents: guard every registered project and clean up orphans",
+        "",
+        "[Service]",
+        "Type=oneshot",
+        "WorkingDirectory=~",
+        f"TimeoutStartSec={START_TIMEOUT}",
+    ]
+    if jambu_home:
+        lines.append(_environment("GPU_HOME", jambu_home))
+    # No "-" prefix: a failing sweep must show up as a failed unit.
+    lines.append("ExecStart=" + " ".join(_quote(a) for a in [python, "-m", "gpu4agents", "sweep", "--apply"]))
+    lines.append("")
+    service = "\n".join(lines)
+    seconds = max(60, int(every_s))
+    timer = "\n".join(
+        [
+            "# Generated by `gpu sweep --install-timer`.",
+            "[Unit]",
+            f"Description=Run {UNIT_NAME}.service every {seconds}s",
+            "",
+            "[Timer]",
+            "OnActiveSec=60s",
+            f"OnUnitActiveSec={seconds}s",
+            "AccuracySec=30s",
+            f"Unit={UNIT_NAME}.service",
+            "",
+            "[Install]",
+            "WantedBy=timers.target",
+            "",
+        ]
+    )
+    return service, timer
+
+
+def legacy_guard_configs(service: Path) -> list[Path]:
+    """Config paths of the per-project `guard` lines older units carried."""
+    if not service.is_file():
+        return []
+    found = []
+    for line in service.read_text().splitlines():
+        if not line.startswith("ExecStart="):
+            continue
+        try:
+            args = shlex.split(line[len("ExecStart="):].lstrip("-"))
+        except ValueError:
+            continue
+        if args and args[-1] == "guard" and "--config" in args:
+            index = args.index("--config") + 1
+            if index < len(args):
+                found.append(Path(args[index].replace("%%", "%")))
+    return found
+
+
+def install_timer(every_s: float, python: Optional[str] = None) -> dict:
+    """Write (or rewrite) the units."""
+    python = python or sys.executable
+    service_path, timer_path = unit_paths()
+    legacy = legacy_guard_configs(service_path)
+    service, timer = render_units(
+        every_s=every_s,
+        python=python,
+        jambu_home=os.environ.get("GPU_HOME") or os.environ.get("JAMBU_HOME") or None,
+    )
+    service_path.parent.mkdir(parents=True, exist_ok=True)
+    service_path.write_text(service)
+    timer_path.write_text(timer)
+    return {
+        "service": str(service_path),
+        "timer": str(timer_path),
+        "every_s": max(60, int(every_s)),
+        "legacy_guard_configs": [str(p) for p in legacy],
+        "enable": f"systemctl --user daemon-reload && systemctl --user enable --now {UNIT_NAME}.timer",
+    }
+
+
+def uninstall_timer() -> dict:
+    removed = []
+    for path in unit_paths():
+        if path.exists():
+            path.unlink()
+            removed.append(str(path))
+    return {
+        "removed": removed,
+        "disable": f"systemctl --user disable --now {UNIT_NAME}.timer; systemctl --user daemon-reload",
+    }

gpu4agents/cli/validate.py

@@ -0,0 +1,57 @@
+"""`gpu validate` - fail before provisioning (spec sections 5, 15)."""
+
+from __future__ import annotations
+
+import typer
+from rich.text import Text
+
+from ..core.errors import JambuError
+from ._common import console, get_context, handle, print_json
+
+
+def validate_command(
+    ctx: typer.Context,
+    offline: bool = typer.Option(
+        False, "--offline", help="Skip provider API checks (schema and policy only)."
+    ),
+) -> None:
+    """Validate jambu.yaml, credentials, capabilities and available capacity."""
+    context = get_context(ctx)
+    try:
+        engine = context.engine()
+        result = engine.validate(check_provider=not offline)
+    except JambuError as exc:
+        if context.json:
+            print_json({"ok": False, "errors": [{"message": str(exc)}]})
+            raise typer.Exit(getattr(exc, "exit_code", 1)) from exc
+        handle(exc)
+        return
+
+    payload = {
+        "ok": result.ok,
+        "provider": engine.provider.name,
+        "model": engine.config.model.id,
+        "runtime": engine.config.runtime.engine,
+        "fingerprint": engine.config.fingerprint(),
+        "errors": [{"field": i.field, "message": i.message} for i in result.errors],
+        "warnings": [{"field": i.field, "message": i.message} for i in result.warnings],
+    }
+    if context.json:
+        print_json(payload)
+        raise typer.Exit(0 if result.ok else 2)
+
+    console.print(
+        f"config: [bold]{engine.config.source_path}[/bold]\n"
+        f"provider: [bold]{engine.provider.display_name}[/bold]  "
+        f"runtime: [bold]{engine.runtime.describe()}[/bold]"
+    )
+    for issue in result.warnings:
+        console.print(Text(f"! {issue.field or 'config'}: {issue.message}", style="yellow"))
+    for issue in result.errors:
+        console.print(Text(f"x {issue.field or 'config'}: {issue.message}", style="bold red"))
+
+    if result.ok:
+        console.print(Text("+ configuration is valid", style="bold green"))
+    else:
+        console.print(Text(f"{len(result.errors)} error(s) must be fixed", style="bold red"))
+        raise typer.Exit(2)