wafer-cli 0.2.26__py3-none-any.whl → 0.2.28__py3-none-any.whl

wafer/cli.py

@@ -268,6 +268,32 @@ Configure targets with: wafer config targets init ..."""
 )
 app.add_typer(targets_ops_app, name="targets", rich_help_panel="Infrastructure")
 
+# Specs management (new: local TOML configs)
+from wafer.specs_cli import specs_app
+
+app.add_typer(specs_app, name="specs", rich_help_panel="Configuration")
+
+# Live resource management (new: API-backed commands on `wafer targets`)
+# These become: wafer targets list, wafer targets terminate, etc.
+from wafer.targets_cli import (
+    targets_list as _targets_list_cmd,
+)
+from wafer.targets_cli import (
+    targets_provision as _targets_provision_cmd,
+)
+from wafer.targets_cli import (
+    targets_reconcile as _targets_reconcile_cmd,
+)
+from wafer.targets_cli import (
+    targets_terminate as _targets_terminate_cmd,
+)
+from wafer.targets_cli import (
+    targets_pools as _targets_pools_cmd,
+)
+from wafer.targets_cli import (
+    targets_probe as _targets_probe_cmd,
+)
+
 # Billing management - nested under config
 billing_app = typer.Typer(help="Manage billing, credits, and subscription")
 config_app.add_typer(billing_app, name="billing")
@@ -612,7 +638,9 @@ def skill_status() -> None:
 auth_app = typer.Typer(help="Authenticate with Wafer and cloud GPU providers")
 app.add_typer(auth_app, name="auth", rich_help_panel="Configuration")
 
-providers_app = typer.Typer(help="Manage API keys for cloud GPU providers (RunPod, DigitalOcean, etc.)")
+providers_app = typer.Typer(
+    help="Manage API keys for cloud GPU providers (RunPod, DigitalOcean, etc.)"
+)
 auth_app.add_typer(providers_app, name="providers")
 
 
@@ -1813,6 +1841,93 @@ def kernelbench_list_problems() -> None:
         raise typer.Exit(1) from None
 
 
+def _resolve_pool_query(pool: str, collector) -> tuple[str, object]:
+    """Resolve a PoolQuery pool to a target spec name + lock context.
+
+    Queries live providers, matches by pool query, locks one target,
+    returns (spec_name, lock_context) for the evaluator.
+    """
+    import trio
+    from wafer_core.targets.pool import resolve_pool
+
+    from .target_lock import acquire_from_pool
+
+    matched_targets = trio.run(resolve_pool, pool)
+
+    if not matched_targets:
+        collector.set_error("pool", "NoMatchingTargets", pool=pool)
+        collector.finalize()
+        raise typer.Exit(1)
+
+    # Filter to targets with a spec (evaluator needs spec fields)
+    spec_targets = [t for t in matched_targets if t.spec_name]
+    if not spec_targets:
+        collector.set_error(
+            "pool", "NoSpecTargets", pool=pool,
+            message="Matched targets have no spec binding — evaluator needs spec fields",
+        )
+        collector.finalize()
+        raise typer.Exit(1)
+
+    # Lock one by resource_id
+    resource_ids = [t.resource_id for t in spec_targets]
+    collector.emit("pool_acquire", pool=pool, count=len(resource_ids))
+
+    lock_ctx = acquire_from_pool(resource_ids)
+    acquired_id = lock_ctx.__enter__()
+
+    if acquired_id is None:
+        lock_ctx.__exit__(None, None, None)
+        collector.set_error("pool", "AllTargetsBusy", pool=pool, targets=resource_ids)
+        collector.finalize()
+        raise typer.Exit(1)
+
+    # Map resource_id back to spec_name
+    acquired_target = next(t for t in spec_targets if t.resource_id == acquired_id)
+    spec_name = acquired_target.spec_name
+
+    collector.emit("pool_acquired", target=spec_name, resource_id=acquired_id)
+    return spec_name, lock_ctx
+
+
+def _resolve_pool_legacy(pool: str, collector) -> tuple[str, object]:
+    """Resolve an old-style pool (static target name list) to a target name + lock context.
+
+    Old format: [pools.name] targets = ["t1", "t2"]
+    """
+    from .target_lock import acquire_from_pool
+    from .targets import filter_pool_by_auth, get_pool
+
+    try:
+        pool_targets = get_pool(pool)
+    except FileNotFoundError as e:
+        collector.set_error("pool", "PoolNotFound", pool=pool, message=str(e))
+        collector.finalize()
+        raise typer.Exit(1) from None
+
+    usable_targets, skipped = filter_pool_by_auth(pool_targets)
+    if skipped:
+        collector.emit("pool_auth_skip", targets=skipped)
+
+    if not usable_targets:
+        collector.set_error("pool", "NoUsableTargets", pool=pool)
+        collector.finalize()
+        raise typer.Exit(1) from None
+
+    collector.emit("pool_acquire", pool=pool, count=len(usable_targets))
+    lock_ctx = acquire_from_pool(usable_targets)
+    acquired_target = lock_ctx.__enter__()
+
+    if acquired_target is None:
+        lock_ctx.__exit__(None, None, None)
+        collector.set_error("pool", "AllTargetsBusy", pool=pool, targets=usable_targets)
+        collector.finalize()
+        raise typer.Exit(1)
+
+    collector.emit("pool_acquired", target=acquired_target)
+    return acquired_target, lock_ctx
+
+
 @kernelbench_app.callback(invoke_without_command=True)
 def kernelbench_evaluate(  # noqa: PLR0913, PLR0915
     ctx: typer.Context,
@@ -1943,39 +2058,12 @@ def kernelbench_evaluate(  # noqa: PLR0913, PLR0915
     pool_lock_context = None
 
     if pool:
-        from .target_lock import acquire_from_pool
-        from .targets import filter_pool_by_auth, get_pool
-
-        try:
-            pool_targets = get_pool(pool)
-        except FileNotFoundError as e:
-            collector.set_error("pool", "PoolNotFound", pool=pool, message=str(e))
-            collector.finalize()
-            raise typer.Exit(1) from None
+        from wafer_core.targets.pool import is_query_pool
 
-        # Filter to only targets with valid auth
-        usable_targets, skipped = filter_pool_by_auth(pool_targets)
-        if skipped:
-            collector.emit("pool_auth_skip", targets=skipped)
-
-        if not usable_targets:
-            collector.set_error("pool", "NoUsableTargets", pool=pool)
-            collector.finalize()
-            raise typer.Exit(1) from None
-
-        collector.emit("pool_acquire", pool=pool, count=len(usable_targets))
-        pool_lock_context = acquire_from_pool(usable_targets)
-        acquired_target = pool_lock_context.__enter__()
-
-        if acquired_target is None:
-            # Exit context manager before raising to avoid resource leak
-            pool_lock_context.__exit__(None, None, None)
-            collector.set_error("pool", "AllTargetsBusy", pool=pool, targets=usable_targets)
-            collector.finalize()
-            raise typer.Exit(1)
-
-        collector.emit("pool_acquired", target=acquired_target)
-        resolved_target = acquired_target
+        if is_query_pool(pool):
+            resolved_target, pool_lock_context = _resolve_pool_query(pool, collector)
+        else:
+            resolved_target, pool_lock_context = _resolve_pool_legacy(pool, collector)
 
     collector.target = resolved_target
 
@@ -5254,6 +5342,18 @@ def workspaces_pull(
         raise typer.Exit(1) from None
 
 
+# =============================================================================
+# Live resource commands (list/terminate/reconcile/provision)
+# =============================================================================
+
+targets_ops_app.command("list")(_targets_list_cmd)
+targets_ops_app.command("terminate")(_targets_terminate_cmd)
+targets_ops_app.command("reconcile")(_targets_reconcile_cmd)
+targets_ops_app.command("provision")(_targets_provision_cmd)
+targets_ops_app.command("pools")(_targets_pools_cmd)
+targets_ops_app.command("probe")(_targets_probe_cmd)
+
+
 # =============================================================================
 # Target operations commands (exec/ssh/sync)
 # =============================================================================

wafer/specs_cli.py

@@ -0,0 +1,157 @@
+"""CLI commands for wafer specs — TargetSpec TOML management.
+
+These are the local config commands (no API calls).
+Registered as: wafer specs list|show|add|remove|default|init
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+specs_app = typer.Typer(
+    help="""Manage GPU target specs (provisioning blueprints).
+
+Specs define how to access or provision GPUs. They are TOML files in ~/.wafer/specs/.
+
+  wafer specs list                        # List all specs
+  wafer specs show runpod-mi300x          # Show one spec
+  wafer specs add /path/to/spec.toml      # Add from file
+  wafer specs remove old-target           # Remove a spec
+  wafer specs default runpod-mi300x       # Set default
+
+To create a new spec interactively:
+  wafer config targets init ssh           # (legacy, still works)
+  wafer config targets init runpod
+"""
+)
+
+
+@specs_app.command("list")
+def specs_list() -> None:
+    """List all configured specs.
+
+    Example:
+        wafer specs list
+    """
+    from wafer_core.targets.spec_store import list_spec_names, load_spec
+
+    from .targets import get_default_target
+
+    names = list_spec_names()
+    default = get_default_target()
+
+    if not names:
+        typer.echo("No specs configured.")
+        typer.echo("Add one with: wafer specs add <path/to/spec.toml>")
+        typer.echo("Or interactively: wafer config targets init ssh")
+        return
+
+    typer.echo("Configured specs:")
+    for name in names:
+        marker = " (default)" if name == default else ""
+        try:
+            spec = load_spec(name)
+            type_name = type(spec).__name__.replace("Target", "")
+            typer.echo(f"  {name}{marker}  [{type_name}]  gpu={spec.gpu_type}")
+        except Exception as e:
+            typer.echo(f"  {name}{marker}  [error: {e}]")
+
+
+@specs_app.command("show")
+def specs_show(
+    name: str = typer.Argument(..., help="Spec name"),
+) -> None:
+    """Show details for a spec.
+
+    Example:
+        wafer specs show runpod-mi300x
+    """
+    from wafer_core.targets.spec_store import load_spec
+
+    from .targets import get_target_info
+
+    try:
+        spec = load_spec(name)
+    except FileNotFoundError:
+        typer.echo(f"Spec not found: {name}", err=True)
+        raise typer.Exit(1) from None
+
+    typer.echo(f"Spec: {name}")
+    for key, value in get_target_info(spec).items():
+        typer.echo(f"  {key}: {value}")
+
+
+@specs_app.command("add")
+def specs_add(
+    file_path: Path = typer.Argument(..., help="Path to TOML spec file"),
+) -> None:
+    """Add a spec from a TOML file.
+
+    Example:
+        wafer specs add ./my-target.toml
+    """
+    import tomllib
+
+    from wafer_core.targets.spec_store import parse_spec, save_spec
+
+    if not file_path.exists():
+        typer.echo(f"File not found: {file_path}", err=True)
+        raise typer.Exit(1) from None
+
+    try:
+        with open(file_path, "rb") as f:
+            data = tomllib.load(f)
+        spec = parse_spec(data)
+        save_spec(spec)
+        typer.echo(f"Added spec: {spec.name}")
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1) from None
+
+
+@specs_app.command("remove")
+def specs_remove(
+    name: str = typer.Argument(..., help="Spec name to remove"),
+    force: bool = typer.Option(False, "--force", "-f", help="Skip confirmation"),
+) -> None:
+    """Remove a spec.
+
+    Example:
+        wafer specs remove old-target
+    """
+    from wafer_core.targets.spec_store import remove_spec
+
+    if not force:
+        confirm = typer.confirm(f"Remove spec '{name}'?")
+        if not confirm:
+            return
+
+    try:
+        remove_spec(name)
+        typer.echo(f"Removed spec: {name}")
+    except FileNotFoundError:
+        typer.echo(f"Spec not found: {name}", err=True)
+        raise typer.Exit(1) from None
+
+
+@specs_app.command("default")
+def specs_default(
+    name: str = typer.Argument(..., help="Spec name to set as default"),
+) -> None:
+    """Set the default spec.
+
+    Example:
+        wafer specs default runpod-mi300x
+    """
+    from wafer_core.targets.spec_store import list_spec_names
+
+    from .targets import set_default_target
+
+    if name not in list_spec_names():
+        typer.echo(f"Spec not found: {name}", err=True)
+        raise typer.Exit(1) from None
+
+    set_default_target(name)
+    typer.echo(f"Default spec set to: {name}")

wafer/targets_cli.py

@@ -0,0 +1,472 @@
+"""CLI commands for wafer targets — live resource management.
+
+These commands always hit provider APIs to show real state.
+Registered as: wafer targets list|show|terminate|sync|provision
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import typer
+
+targets_live_app = typer.Typer(
+    name="targets",
+    help="""Manage live GPU resources across cloud providers.
+
+Unlike 'wafer specs' (local config files), these commands query provider APIs
+to show what's actually running.
+
+  wafer targets list                    # All running resources
+  wafer targets list --unbound          # Orphans (no matching spec)
+  wafer targets list --provider runpod  # Filter by provider
+  wafer targets terminate <resource-id> # Kill a resource
+  wafer targets terminate --unbound     # Kill all orphans
+  wafer targets sync                    # Refresh bindings
+  wafer targets provision <spec-name>   # Provision from a spec
+""",
+)
+
+
+@targets_live_app.command("list")
+def targets_list(
+    provider: str | None = typer.Option(None, "--provider", "-p", help="Filter by provider"),
+    pool: str | None = typer.Option(None, "--pool", help="Filter by pool query from config.toml"),
+) -> None:
+    """List all running GPU resources across providers.
+
+    Queries RunPod and DigitalOcean APIs to show live state.
+
+    Examples:
+        wafer targets list
+        wafer targets list --provider runpod
+        wafer targets list --pool mi300x-rocm7
+    """
+    import trio
+    from wafer_core.targets.providers import get_all_cloud_providers, get_provider
+    from wafer_core.targets.types import Target, TargetProvider
+
+    async def _list() -> list[Target]:
+        all_targets: list[Target] = []
+
+        if provider:
+            prov = get_provider(provider)
+            all_targets = await prov.list_targets()
+        else:
+            providers = get_all_cloud_providers()
+
+            async def _fetch(prov_impl: TargetProvider, results: list[Target]) -> None:
+                try:
+                    targets = await prov_impl.list_targets()
+                    results.extend(targets)
+                except Exception as e:
+                    typer.echo(
+                        f"  Warning: failed to query {type(prov_impl).__name__}: {e}", err=True
+                    )
+
+            async with trio.open_nursery() as nursery:
+                for _, prov_impl in providers:
+                    nursery.start_soon(_fetch, prov_impl, all_targets)
+
+        return all_targets
+
+    all_targets = trio.run(_list)
+
+    # Hydrate targets with cached labels
+    from dataclasses import replace
+    from wafer_core.targets.state_cache import load_all_labels
+
+    cached_labels = load_all_labels()
+    all_targets = [
+        replace(t, labels=cached_labels[t.resource_id])
+        if t.resource_id in cached_labels
+        else t
+        for t in all_targets
+    ]
+
+    # Apply pool filter if specified
+    if pool:
+        from wafer_core.targets.pool import load_pool_query, match_targets
+
+        try:
+            query = load_pool_query(pool)
+        except KeyError as e:
+            typer.echo(str(e), err=True)
+            raise typer.Exit(1) from None
+
+        all_targets = match_targets(query, all_targets)
+        typer.echo(f"Pool {pool!r}: {len(all_targets)} matching target(s)\n")
+
+    if not all_targets:
+        typer.echo("No running resources found.")
+        return
+
+    typer.echo(f"{len(all_targets)} resource(s):\n")
+    for target in all_targets:
+        _print_target(target)
+
+
+def _print_target(target: Target) -> None:
+    """Print a single target's info."""
+    ssh_info = ""
+    if target.public_ip and target.ssh_port:
+        ssh_info = f"  ssh={target.ssh_username}@{target.public_ip}:{target.ssh_port}"
+
+    name_part = f"  name={target.name}" if target.name else ""
+    spec_part = f"  spec={target.spec_name}" if target.spec_name else ""
+    price_part = f"  ${target.price_per_hour:.2f}/hr" if target.price_per_hour else ""
+
+    # Show interesting labels (skip 'image' — too long)
+    label_keys = sorted(k for k in target.labels if k != "image")
+    labels_part = ""
+    if label_keys:
+        labels_part = "  " + " ".join(f"{k}={target.labels[k]}" for k in label_keys)
+
+    typer.echo(
+        f"  {target.resource_id}  [{target.provider}]  "
+        f"status={target.status}  gpu={target.gpu_type}"
+        f"{spec_part}{name_part}{ssh_info}{price_part}{labels_part}"
+    )
+    typer.echo()
+
+
+@targets_live_app.command("terminate")
+def targets_terminate(
+    resource_id: str | None = typer.Argument(None, help="Resource ID to terminate"),
+    pool_name: str | None = typer.Option(
+        None, "--pool", help="Terminate all targets matching a pool query"
+    ),
+    provider_name: str | None = typer.Option(
+        None, "--provider", "-p", help="Provider hint (avoids querying all providers)"
+    ),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"),
+) -> None:
+    """Terminate a running resource by ID, or all targets matching a pool query.
+
+    Examples:
+        wafer targets terminate tkru24z7npcgth
+        wafer targets terminate --pool mi300x --yes
+        wafer targets terminate --pool runpod-only --provider runpod
+    """
+    import trio
+    from wafer_core.targets.providers import get_all_cloud_providers, get_provider
+    from wafer_core.targets.state_cache import remove_binding
+
+    if pool_name:
+        _terminate_pool(pool_name, provider_name, yes)
+        return
+
+    if not resource_id:
+        typer.echo("Provide a resource ID or use --pool <name>.", err=True)
+        raise typer.Exit(1)
+
+    async def _terminate() -> bool:
+        if provider_name:
+            prov = get_provider(provider_name)
+            return await prov.terminate(resource_id)
+
+        for name, prov in get_all_cloud_providers():
+            target = await prov.get_target(resource_id)
+            if target is not None:
+                success = await prov.terminate(resource_id)
+                if success:
+                    remove_binding(resource_id)
+                    typer.echo(f"Terminated {resource_id} ({name})")
+                return success
+
+        typer.echo(f"Resource {resource_id} not found on any provider.", err=True)
+        return False
+
+    success = trio.run(_terminate)
+    if not success:
+        raise typer.Exit(1)
+
+
+def _terminate_pool(pool_name: str, provider_name: str | None, yes: bool) -> None:
+    """Terminate all targets matching a pool query."""
+    import trio
+    from wafer_core.targets.pool import load_pool_query, match_targets
+    from wafer_core.targets.providers import get_all_cloud_providers, get_provider
+    from wafer_core.targets.state_cache import remove_binding
+    from wafer_core.targets.types import Target
+
+    try:
+        query = load_pool_query(pool_name)
+    except KeyError as e:
+        typer.echo(str(e), err=True)
+        raise typer.Exit(1) from None
+
+    async def _do_terminate() -> int:
+        all_targets: list[Target] = []
+        if provider_name:
+            prov = get_provider(provider_name)
+            all_targets = await prov.list_targets()
+        else:
+            for _, prov in get_all_cloud_providers():
+                try:
+                    all_targets.extend(await prov.list_targets())
+                except Exception:
+                    pass
+
+        matched = match_targets(query, all_targets)
+
+        if not matched:
+            typer.echo(f"No targets match pool {pool_name!r}.")
+            return 0
+
+        typer.echo(f"Found {len(matched)} target(s) matching pool {pool_name!r}:")
+        for t in matched:
+            name_part = f"  name={t.name}" if t.name else ""
+            typer.echo(f"  {t.resource_id}  [{t.provider}]  gpu={t.gpu_type}{name_part}")
+
+        if not yes:
+            confirm = typer.confirm("Terminate all?")
+            if not confirm:
+                return 0
+
+        count = 0
+        for t in matched:
+            prov = get_provider(t.provider)
+            if await prov.terminate(t.resource_id):
+                remove_binding(t.resource_id)
+                typer.echo(f"  Terminated {t.resource_id}")
+                count += 1
+            else:
+                typer.echo(f"  Failed to terminate {t.resource_id}", err=True)
+
+        return count
+
+    count = trio.run(_do_terminate)
+    typer.echo(f"\nTerminated {count} resource(s).")
+
+
+@targets_live_app.command("reconcile")
+def targets_reconcile() -> None:
+    """Refresh local binding cache from provider APIs.
+
+    Queries all cloud providers, matches resources to specs, and updates
+    the local state cache. Reports any drift.
+
+    Example:
+        wafer targets reconcile
+    """
+    import trio
+    from wafer_core.targets.providers import get_all_cloud_providers
+    from wafer_core.targets.reconcile import reconcile
+    from wafer_core.targets.spec_store import load_all_specs
+    from wafer_core.targets.state_cache import (
+        BindingEntry,
+        get_binding_hints,
+        save_bindings,
+    )
+    from wafer_core.targets.types import Target
+
+    async def _sync() -> None:
+        specs = load_all_specs()
+
+        all_targets: list[Target] = []
+        for name, prov in get_all_cloud_providers():
+            typer.echo(f"Querying {name}...")
+            try:
+                targets = await prov.list_targets()
+                typer.echo(f"  Found {len(targets)} resource(s)")
+                all_targets.extend(targets)
+            except Exception as e:
+                typer.echo(f"  Failed: {e}", err=True)
+
+        hints = get_binding_hints()
+        result = reconcile(specs, all_targets, binding_hints=hints)
+
+        # Update binding cache with bound results
+        new_bindings = {}
+        now = datetime.now(UTC).isoformat()
+        for spec, target in result.bound:
+            new_bindings[target.resource_id] = BindingEntry(
+                spec_name=spec.name,
+                provider=target.provider,
+                bound_at=now,
+            )
+        save_bindings(new_bindings)
+
+        typer.echo("\nSync complete:")
+        typer.echo(f"  Total resources: {len(all_targets)}")
+        typer.echo(f"  Matched to specs: {len(result.bound)}")
+        typer.echo(f"  No matching spec: {len(result.unbound)}")
+
+    trio.run(_sync)
+
+
+@targets_live_app.command("provision")
+def targets_provision(
+    spec_name: str = typer.Argument(..., help="Spec name to provision from"),
+) -> None:
+    """Explicitly provision a resource from a spec.
+
+    Creates a new cloud resource and binds it to the spec.
+
+    Example:
+        wafer targets provision runpod-mi300x
+    """
+    import trio
+    from wafer_core.targets.providers import get_provider
+    from wafer_core.targets.spec_store import load_spec
+    from wafer_core.targets.state_cache import BindingEntry, add_binding
+    from wafer_core.utils.kernel_utils.targets.config import (
+        DigitalOceanTarget,
+        RunPodTarget,
+    )
+
+    try:
+        spec = load_spec(spec_name)
+    except FileNotFoundError:
+        typer.echo(f"Spec not found: {spec_name}", err=True)
+        raise typer.Exit(1) from None
+
+    if isinstance(spec, RunPodTarget):
+        provider_name = "runpod"
+    elif isinstance(spec, DigitalOceanTarget):
+        provider_name = "digitalocean"
+    else:
+        typer.echo(f"Spec type {type(spec).__name__} cannot be provisioned.", err=True)
+        raise typer.Exit(1) from None
+
+    async def _provision() -> None:
+        from wafer_core.targets.probe import probe_target_labels
+        from wafer_core.targets.state_cache import save_labels
+
+        prov = get_provider(provider_name)
+        typer.echo(f"Provisioning {spec_name} via {provider_name}...")
+        target = await prov.provision(spec)
+
+        # Cache the binding
+        add_binding(
+            target.resource_id,
+            BindingEntry(
+                spec_name=spec_name,
+                provider=provider_name,
+                bound_at=datetime.now(UTC).isoformat(),
+            ),
+        )
+
+        typer.echo(f"\nProvisioned: {target.resource_id}")
+        if target.public_ip:
+            typer.echo(f"  SSH: {target.ssh_username}@{target.public_ip}:{target.ssh_port}")
+
+        # Probe software labels (sync — runs subprocess ssh)
+        if target.public_ip and target.ssh_port:
+            typer.echo("  Probing software versions...")
+            try:
+                ssh_key = spec.ssh_key if hasattr(spec, "ssh_key") else None
+                labels = probe_target_labels(
+                    host=target.public_ip,
+                    port=target.ssh_port,
+                    username=target.ssh_username,
+                    ssh_key_path=ssh_key,
+                )
+                save_labels(target.resource_id, labels)
+                if labels:
+                    typer.echo(f"  Labels: {' '.join(f'{k}={v}' for k, v in sorted(labels.items()))}")
+            except Exception as e:
+                typer.echo(f"  Warning: probe failed: {e}", err=True)
+
+    trio.run(_provision)
+
+
+@targets_live_app.command("pools")
+def targets_pools() -> None:
+    """List configured pool queries from config.toml.
+
+    Example:
+        wafer targets pools
+    """
+    from wafer_core.targets.pool import list_pool_names, load_pool_query
+
+    names = list_pool_names()
+    if not names:
+        typer.echo("No pools configured in ~/.wafer/config.toml.")
+        typer.echo("\nAdd a pool:\n")
+        typer.echo("  [pools.mi300x]")
+        typer.echo('  gpu_type = "MI300X"')
+        typer.echo("")
+        typer.echo("  [pools.mi300x-rocm7]")
+        typer.echo('  gpu_type = "MI300X"')
+        typer.echo("  [pools.mi300x-rocm7.labels]")
+        typer.echo('  rocm_version = "7.0.2"')
+        return
+
+    typer.echo(f"{len(names)} pool(s):\n")
+    for name in names:
+        query = load_pool_query(name)
+        parts = []
+        if query.gpu_type:
+            parts.append(f"gpu_type={query.gpu_type}")
+        if query.provider:
+            parts.append(f"provider={query.provider}")
+        if query.status and query.status != "running":
+            parts.append(f"status={query.status}")
+        for k, v in sorted(query.labels.items()):
+            parts.append(f"{k}={v}")
+        criteria = "  ".join(parts) if parts else "(match all)"
+        typer.echo(f"  {name}:  {criteria}")
+
+
+@targets_live_app.command("probe")
+def targets_probe(
+    resource_id: str = typer.Argument(..., help="Resource ID to probe"),
+    provider_name: str | None = typer.Option(
+        None, "--provider", "-p", help="Provider hint (avoids querying all providers)"
+    ),
+) -> None:
+    """Probe a running target's software versions via SSH.
+
+    Results are cached in ~/.wafer/target_state.json and shown
+    by wafer targets list. Used for targets not provisioned by wafer
+    (e.g. dashboard-created pods).
+
+    Examples:
+        wafer targets probe ewfo5ckpxlg7y2
+        wafer targets probe 543538453 --provider digitalocean
+    """
+    import trio
+    from wafer_core.targets.probe import probe_target_labels
+    from wafer_core.targets.providers import get_all_cloud_providers, get_provider
+    from wafer_core.targets.state_cache import save_labels
+
+    # Find the target (async — needs provider API)
+    async def _find_target():
+        if provider_name:
+            prov = get_provider(provider_name)
+            return await prov.get_target(resource_id)
+
+        for _, prov in get_all_cloud_providers():
+            target = await prov.get_target(resource_id)
+            if target is not None:
+                return target
+        return None
+
+    target = trio.run(_find_target)
+
+    if target is None:
+        typer.echo(f"Resource {resource_id} not found.", err=True)
+        raise typer.Exit(1)
+
+    if not target.public_ip or not target.ssh_port:
+        typer.echo(f"Resource {resource_id} has no SSH info (status={target.status}).", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Probing {resource_id} ({target.ssh_username}@{target.public_ip}:{target.ssh_port})...")
+
+    labels = probe_target_labels(
+        host=target.public_ip,
+        port=target.ssh_port,
+        username=target.ssh_username,
+    )
+
+    save_labels(resource_id, labels)
+
+    if labels:
+        typer.echo(f"Labels cached for {resource_id}:")
+        for k, v in sorted(labels.items()):
+            typer.echo(f"  {k}={v}")
+    else:
+        typer.echo("Probe returned no labels.")

wafer/targets_ops.py

@@ -15,6 +15,7 @@ import logging
 import subprocess
 from collections.abc import Callable
 from dataclasses import dataclass, replace
+from datetime import UTC
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -30,6 +31,26 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def _update_binding_cache(resource_id: str, spec_name: str, provider: str) -> None:
+    """Update the new target state cache when provisioning through the legacy path.
+
+    This bridges the old per-provider state files with the new unified cache
+    so that `wafer targets list` can see resources provisioned via the old flow.
+    """
+    from datetime import datetime
+
+    from wafer_core.targets.state_cache import BindingEntry, add_binding
+
+    add_binding(
+        resource_id,
+        BindingEntry(
+            spec_name=spec_name,
+            provider=provider,
+            bound_at=datetime.now(UTC).isoformat(),
+        ),
+    )
+
+
 @dataclass(frozen=True)
 class TargetSSHInfo:
     """SSH connection info for a target."""
@@ -135,7 +156,8 @@ async def _get_runpod_ssh_info(target: RunPodTarget) -> TargetSSHInfo:
     # Check if pod already exists and is running
     existing = get_pod_state(target.name)
     if existing and await check_pod_running(existing.pod_id):
-        # Reuse existing pod
+        # Reuse existing pod — also update the new state cache
+        _update_binding_cache(existing.pod_id, target.name, "runpod")
         return TargetSSHInfo(
             host=existing.public_ip,
             port=existing.ssh_port,
@@ -151,6 +173,8 @@ async def _get_runpod_ssh_info(target: RunPodTarget) -> TargetSSHInfo:
     target_keep_alive = replace(target, keep_alive=True)
 
     async with runpod_ssh_context(target_keep_alive) as ssh_info:
+        # Update new state cache with provisioned pod
+        _update_binding_cache(ssh_info.pod_id, target.name, "runpod")
         return TargetSSHInfo(
             host=ssh_info.host,
             port=ssh_info.port,
@@ -172,7 +196,8 @@ async def _get_digitalocean_ssh_info(target: DigitalOceanTarget) -> TargetSSHInf
     # Check if droplet already exists and is running
     existing = get_droplet_state(target.name)
     if existing and await check_droplet_running(existing.droplet_id):
-        # Reuse existing droplet
+        # Reuse existing droplet — also update the new state cache
+        _update_binding_cache(existing.droplet_id, target.name, "digitalocean")
         return TargetSSHInfo(
             host=existing.public_ip,
             port=22,  # DigitalOcean uses standard SSH port
@@ -184,6 +209,8 @@ async def _get_digitalocean_ssh_info(target: DigitalOceanTarget) -> TargetSSHInf
     target_keep_alive = replace(target, keep_alive=True)
 
     async with digitalocean_ssh_context(target_keep_alive) as ssh_info:
+        # Update new state cache with provisioned droplet
+        _update_binding_cache(ssh_info.droplet_id, target.name, "digitalocean")
         return TargetSSHInfo(
             host=ssh_info.host,
             port=ssh_info.port,

{wafer_cli-0.2.26.dist-info → wafer_cli-0.2.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wafer-cli
-Version: 0.2.26
+Version: 0.2.28
 Summary: CLI for running GPU workloads, managing remote workspaces, and evaluating/optimizing kernels
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{wafer_cli-0.2.26.dist-info → wafer_cli-0.2.28.dist-info}/RECORD

@@ -6,7 +6,7 @@ wafer/api_client.py,sha256=i_Az2b2llC3DSW8yOL-BKqa7LSKuxOr8hSN40s-oQXY,6313
 wafer/auth.py,sha256=dwss_se5P-FFc9IN38q4kh_dBrA6k-CguDBkivgcdj0,14003
 wafer/autotuner.py,sha256=41WYP41pTDvMijv2h42vm89bcHtDMJXObDlWmn6xpFU,44416
 wafer/billing.py,sha256=hEEwtrtIsbPQ3lLJNcyTLMsapUbcuvcVW_e9_0SxzVo,7199
-wafer/cli.py,sha256=s3m6SJzK1vRJxaQCrd_I4rcxrt3skty0GBdFHzIBc6U,279424
+wafer/cli.py,sha256=zuVZhPdML5AOBtLUqLwAwjl8XMNe9EwQkffZxtBGLx4,282748
 wafer/cli_instructions.py,sha256=bziUKDNDAXABVMvKPLEMXm-hFSD2TcFSh-FKRYa949k,4693
 wafer/config.py,sha256=h5Eo9_yfWqWGoPNdVQikI9GoZVUeysunSYiixf1mKcw,3411
 wafer/corpus.py,sha256=CY9T7wXENNDJxnrtI-XsQmXeptrFfKG4x-lngrc9_3s,24748
@@ -23,10 +23,12 @@ wafer/problems.py,sha256=ce2sy10A1nnNUG3VGsseTS8jL7LZsku4dE8zVf9JHQ4,11296
 wafer/rocprof_compute.py,sha256=n_yOGZaFbOXna_ghhmYWXeyUoSabgH4KkjlYq38DlHo,19888
 wafer/rocprof_sdk.py,sha256=0Q7Ye6dUfa1anFZbqKc21rItgqva8V8VIZoSB7wqbmA,10085
 wafer/rocprof_systems.py,sha256=4IWbMcbYk1x_8iS7P3FC_u5sgH6EXADCtR2lV9id80M,18629
+wafer/specs_cli.py,sha256=frMEKwMflxVNpFlAuxprmr33ZZ1Oeh2lB0KWZ4oZWzw,4360
 wafer/ssh_keys.py,sha256=MxiHlSm6wuDUFzkOQtx5K7OIbx_a6bXxE-m8OpwLx98,8130
 wafer/target_lock.py,sha256=SDKhNzv2N7gsphGflcNni9FE5YYuAMuEthngAJEo4Gs,7809
 wafer/targets.py,sha256=9r-iRWoKSH5cQl1LcamaX-T7cNVOg99ngIm_hlRk-qU,26922
-wafer/targets_ops.py,sha256=jN1oIBx0mutxRNE9xpIc7SaBxPkVmOyus2eqn0kEKNI,21475
+wafer/targets_cli.py,sha256=Oe3e02rSXeNrMbe_Qv9DNfQ8dEOKodtU7BbQQWxlNwA,16348
+wafer/targets_ops.py,sha256=wLPyq55H_wz0wEAEg8KFLYs9LIIyiVIphcsXD2NLa-E,22623
 wafer/trace_compare.py,sha256=COuxxKY874DteOSLUvJuJFREPMBSybq9dtANi3ATsg4,10803
 wafer/tracelens.py,sha256=g9ZIeFyNojZn4uTd3skPqIrRiL7aMJOz_-GOd3aiyy4,7998
 wafer/wevin_cli.py,sha256=eo1ETsXIsCftXSG5AxEYYZipNGcXayKyIevs5F6MjXg,26140
@@ -38,8 +40,8 @@ wafer/templates/optimize_kernel.py,sha256=4-MaKm_C9BQHQEllrNLLYkcdhJpcj6D-8zbJ4F
 wafer/templates/optimize_kernelbench.py,sha256=T3co9Y9eSLWDrZG66gwQVFMdnGVoyUQos-TxnMMBLL8,3747
 wafer/templates/trace_analyze.py,sha256=B7CiRlsokERzBjLL-k49kGjpU2zlJZqzTE05xbRS1WI,2878
 wafer/tests/test_eval_cli_parity.py,sha256=SGmaj2NGBZ7GdDF53bXsECvQbV21iHZw8YeL_MJOLk0,7206
-wafer_cli-0.2.26.dist-info/METADATA,sha256=IM8Eatar1KYIBo1hHEBjvpX6J272f0PWfV4mwhV1jIY,2799
-wafer_cli-0.2.26.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-wafer_cli-0.2.26.dist-info/entry_points.txt,sha256=WqB7hB__WhtPY8y1cO2sZiUz7fCq6Ik-usAigpeFvWE,41
-wafer_cli-0.2.26.dist-info/top_level.txt,sha256=2MK1IVMWfpLL8BZCQ3E9aG6L6L666gSA_teYlwan4fs,6
-wafer_cli-0.2.26.dist-info/RECORD,,
+wafer_cli-0.2.28.dist-info/METADATA,sha256=2AsfmsBcnVfG8UTxileEvOjpEYOO1OF5YcpXpm9Mf2w,2799
+wafer_cli-0.2.28.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+wafer_cli-0.2.28.dist-info/entry_points.txt,sha256=WqB7hB__WhtPY8y1cO2sZiUz7fCq6Ik-usAigpeFvWE,41
+wafer_cli-0.2.28.dist-info/top_level.txt,sha256=2MK1IVMWfpLL8BZCQ3E9aG6L6L666gSA_teYlwan4fs,6
+wafer_cli-0.2.28.dist-info/RECORD,,