valkyrie-cli 2026.11.9__py3-none-any.whl

valkyrie/cli/agent/__init__.py

@@ -0,0 +1,27 @@
+import click
+
+from valkyrie.cli.lazy import LazyGroup
+
+
+@click.group(
+    cls=LazyGroup,
+    lazy_commands={
+        "install": (
+            "valkyrie.cli.agent.lifecycle:install",
+            "Install a GitHub agent through Tracker, replacing an existing alias.",
+        ),
+        "push": (
+            "valkyrie.cli.agent.lifecycle:push",
+            "Upload a local agent through Tracker, replacing an existing alias.",
+        ),
+        "remove": ("valkyrie.cli.agent.lifecycle:agent_remove", "Remove an installed agent."),
+        "download": ("valkyrie.cli.agent.lifecycle:download", "Download an installed agent."),
+        "list": ("valkyrie.cli.agent.lifecycle:list_installed_agents", "List installed agents."),
+    },
+    help="Install, upload, list, download, and remove agents.",
+)
+def agent():
+    pass
+
+
+__all__ = ["agent"]

valkyrie/cli/agent/lifecycle.py

@@ -0,0 +1,130 @@
+import json
+from datetime import datetime
+from pathlib import Path
+
+import click
+from valkyrie.sdk import ValkyrieClient
+
+from valkyrie.display import terminal_safe
+
+from valkyrie.cli.client import call
+from valkyrie.cli.display import format_option, format_table, local_time, paginate_cli_pages
+
+
+@click.command(name="install", help="Install a GitHub agent through Tracker, replacing an existing alias.")
+@click.argument("github_url", type=str)
+@click.option(
+    "--name",
+    "-n",
+    type=str,
+    required=False,
+    help="Agent name (defaults to the contract name).",
+)
+def install(github_url: str, name: str | None):
+    entry = call(lambda client: client.agents.install(github_url, name=name))
+    click.echo(click.style(f"✓ Agent '{entry.name}' installed successfully!", fg="green", bold=True))
+
+
+@click.command(name="push", help="Upload a local agent through Tracker, replacing an existing alias.")
+@click.argument("agent_path", type=click.Path(exists=True, path_type=Path, file_okay=False, dir_okay=True))
+@click.option(
+    "--name",
+    "-n",
+    type=str,
+    required=False,
+    help="Agent name (defaults to the contract name).",
+)
+def push(agent_path: Path, name: str | None):
+    entry = call(lambda client: client.agents.push(agent_path, name=name or None))
+    click.echo(click.style(f"✓ Agent '{entry.name}' pushed successfully!", fg="green", bold=True))
+
+
+@click.command(name="remove", help="Remove an installed agent.")
+@click.argument("agent_name", type=str)
+@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.")
+def agent_remove(agent_name: str, yes: bool):
+    if not yes and not click.confirm(f"Are you sure you want to remove agent '{agent_name}'?"):
+        click.echo("Cancelled.")
+        return
+
+    call(lambda client: client.agents.remove(agent_name))
+    click.echo(click.style(f"✓ Agent '{agent_name}' removed successfully!", fg="green", bold=True))
+
+
+@click.command(name="download", help="Download an installed agent.")
+@click.argument("agent_name", type=str)
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(path_type=Path, file_okay=False, dir_okay=True),
+    default=None,
+    required=False,
+    help="Output directory for downloaded agent (default: current directory).",
+)
+@click.option("--overwrite", is_flag=True, help="Replace an existing agent directory after validating the download.")
+def download(agent_name: str, output_dir: Path | None, overwrite: bool):
+    call(lambda client: client.agents.download(agent_name, output_dir, overwrite=overwrite))
+    click.echo(click.style(f"✓ Agent '{agent_name}' downloaded successfully!", fg="green", bold=True))
+
+
+@click.command(name="list", help="List installed agents.")
+@click.option("--all", "all_agents", is_flag=True, help="Show every agent without interactive paging.")
+@format_option("Output format. JSON includes every agent without interactive paging.")
+def list_installed_agents(all_agents: bool, output_format: str):
+
+    async def list_agents(client: ValkyrieClient) -> list[tuple[str, datetime | None]]:
+        response = await client.agents.list()
+        try:
+            return [
+                (agent.name, datetime.fromisoformat(agent.last_modified) if agent.last_modified else None)
+                for agent in response.agents
+            ]
+        except ValueError as error:
+            raise click.ClickException(terminal_safe(str(error), preserve_newlines=True)) from error
+
+    agents = call(list_agents)
+
+    if output_format == "json":
+        click.echo(
+            json.dumps(
+                {
+                    "agents": [
+                        {"name": name, "last_modified": modified.isoformat() if modified else None}
+                        for name, modified in agents
+                    ]
+                }
+            )
+        )
+        return
+
+    if not agents:
+        click.echo(click.style("No agents found.", fg="yellow"))
+        return
+
+    if all_agents:
+        _format_agents_response(agents, 1, 1, len(agents))
+    else:
+        paginate_agents(agents)
+
+
+def _format_agents_response(
+    agents: list[tuple[str, datetime | None]],
+    current_page: int,
+    total_pages: int,
+    total_count: int,
+) -> None:
+    rows = [
+        {"Agent": name, "Last Modified": local_time(last_modified) if last_modified else "-"}
+        for name, last_modified in agents
+    ]
+
+    format_table(rows, ["Agent", "Last Modified"], current_page, total_pages, total_count, "agent")
+
+
+def paginate_agents(agents: list[tuple[str, datetime | None]], limit: int = 10) -> None:
+    paginate_cli_pages(
+        lambda offset, page_limit: (len(agents), agents[offset : offset + page_limit]),
+        _format_agents_response,
+        limit=limit,
+        render_empty=lambda: click.echo(click.style("No agents found.", fg="yellow")),
+    )

valkyrie/cli/benchmark/__init__.py

@@ -0,0 +1,29 @@
+import click
+
+from valkyrie.cli.lazy import LazyGroup
+
+
+@click.group(
+    cls=LazyGroup,
+    lazy_commands={
+        "tasks": ("valkyrie.cli.benchmark.tasks:tasks", "Export the task IDs of a benchmark dataset."),
+        "service": (
+            "valkyrie.cli.benchmark.services:service",
+            "Save benchmark service URLs that your runs use instead of the hosted services.",
+        ),
+    },
+    hidden_commands={
+        "create": "valkyrie.cli.benchmark.definitions:create_benchmark",
+        "list": "valkyrie.cli.benchmark.definitions:list_benchmarks",
+        "show": "valkyrie.cli.benchmark.definitions:show_benchmark",
+        "update": "valkyrie.cli.benchmark.definitions:update_benchmark",
+        "remove": "valkyrie.cli.benchmark.definitions:remove_benchmark",
+        "restore": "valkyrie.cli.benchmark.definitions:restore_benchmark",
+    },
+    help="Export the task IDs in a benchmark dataset, and save benchmark services.",
+)
+def benchmark():
+    pass
+
+
+__all__ = ["benchmark"]

valkyrie/cli/benchmark/definitions.py

@@ -0,0 +1,73 @@
+import click
+
+from valkyrie.cli.client import call
+from valkyrie.cli.resource import INCLUDE_ARCHIVED, RESOURCE_NAME, list_resources, show_resource
+from valkyrie.cli.display import format_option, one_line
+
+
+@click.command(
+    name="create",
+    help=(
+        "Create a benchmark that datasets can belong to.\n\n"
+        "NAME is lowercase letters and digits with single '.', '_' or '-' separators, such as swe-bench.\n\n"
+        "Example:\n"
+        '    valkyrie benchmark create swe-bench --description "SWE-bench tasks"'
+    ),
+)
+@click.argument("name", metavar="NAME", type=RESOURCE_NAME)
+@click.option("--description", default="", help="What the benchmark covers.")
+def create_benchmark(name: str, description: str) -> None:
+    created = call(lambda client: client.benchmark_definitions.create(name, description=description))
+    click.echo(click.style(f"✓ Benchmark created: {one_line(created.name)}", fg="green", bold=True))
+
+
+@click.command(name="list", help="List your organization's benchmarks.")
+@INCLUDE_ARCHIVED
+@format_option()
+def list_benchmarks(include_archived: bool, output_format: str) -> None:
+    response = call(lambda client: client.benchmark_definitions.list(include_archived=include_archived))
+    list_resources(response, response.benchmarks, output_format, "benchmark")
+
+
+@click.command(name="show", help="Show one benchmark, archived or not.")
+@click.argument("name", metavar="NAME", type=RESOURCE_NAME)
+@format_option()
+def show_benchmark(name: str, output_format: str) -> None:
+    found = call(lambda client: client.benchmark_definitions.get(name))
+    show_resource(found, output_format)
+
+
+@click.command(
+    name="update",
+    help=(
+        "Change a benchmark's description. Its name never changes.\n\n"
+        "Example:\n"
+        '    valkyrie benchmark update swe-bench --description "SWE-bench family"'
+    ),
+)
+@click.argument("name", metavar="NAME", type=RESOURCE_NAME)
+@click.option("--description", required=True, help="The description the benchmark now has.")
+def update_benchmark(name: str, description: str) -> None:
+    updated = call(lambda client: client.benchmark_definitions.update(name, description=description))
+    click.echo(click.style(f"✓ Benchmark updated: {one_line(updated.name)}", fg="green", bold=True))
+
+
+@click.command(
+    name="remove",
+    help="Archive a benchmark once it has no active datasets. It stays readable, keeps its name, and can be restored.",
+)
+@click.argument("name", metavar="NAME", type=RESOURCE_NAME)
+@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.")
+def remove_benchmark(name: str, yes: bool) -> None:
+    if not yes and not click.confirm(f"Archive benchmark '{name}'?"):
+        click.echo("Cancelled.")
+        return
+    call(lambda client: client.benchmark_definitions.remove(name))
+    click.echo(click.style(f"✓ Benchmark '{name}' archived.", fg="green", bold=True))
+
+
+@click.command(name="restore", help="Restore an archived benchmark so it can change again.")
+@click.argument("name", metavar="NAME", type=RESOURCE_NAME)
+def restore_benchmark(name: str) -> None:
+    restored = call(lambda client: client.benchmark_definitions.restore(name))
+    click.echo(click.style(f"✓ Benchmark restored: {one_line(restored.name)}", fg="green", bold=True))

valkyrie/cli/benchmark/services.py

@@ -0,0 +1,147 @@
+from collections.abc import Callable, Sequence
+from urllib.parse import urlparse
+
+import click
+from valkyrie.sdk import BenchmarkServiceEntry, BenchmarkServiceHealth
+
+from valkyrie.cli.client import call
+from valkyrie.cli.display import format_table, paginate_cli_pages
+
+
+@click.group(help="Save benchmark service URLs that your runs use instead of the hosted services.")
+def service() -> None:
+    pass
+
+
+@service.command(
+    "set",
+    help=(
+        "Use a service URL for your runs of a benchmark.\n"
+        "\n"
+        "Replaces the URL and credential saved before. The Tracker must be able to reach the URL.\n"
+        "\n"
+        "Example:\n"
+        "    valkyrie benchmark service set swebench https://my-tunnel.ngrok.io --auth-secret swebench-key"
+    ),
+)
+@click.argument("benchmark")
+@click.argument("url")
+@click.option(
+    "--auth-secret",
+    metavar="SECRET",
+    help="Organization secret whose value the Tracker sends to the service. Create it with `valkyrie secret set`.",
+)
+@click.option("--auth-header", metavar="HEADER", help="Header that carries the secret. Defaults to Authorization.")
+def service_set(benchmark: str, url: str, auth_secret: str | None, auth_header: str | None) -> None:
+    settings = call(
+        lambda client: client.settings.set_benchmark_service(
+            benchmark, url, auth_secret=auth_secret, auth_header=auth_header
+        )
+    )
+    saved = settings.benchmark_services[benchmark]
+    message = f"Runs of {benchmark} now use {saved.url}"
+    if saved.auth_secret_name:
+        message += f", sending the secret {saved.auth_secret_name} as {saved.auth_header_name}"
+    click.echo(f"{message}.")
+
+
+@service.command(
+    "remove",
+    help=(
+        "Go back to the hosted service for your runs of a benchmark.\n"
+        "\n"
+        "Example:\n"
+        "    valkyrie benchmark service remove swebench"
+    ),
+)
+@click.argument("benchmark")
+def service_remove(benchmark: str) -> None:
+    call(lambda client: client.settings.remove_benchmark_service(benchmark))
+    click.echo(f"Runs of {benchmark} now use the hosted service.")
+
+
+@service.command(
+    "list",
+    help=(
+        "List hosted benchmark services and your saved URLs, with their latency.\n"
+        "\n"
+        "A URL you saved replaces the hosted service with the same name."
+    ),
+)
+def service_list() -> None:
+    saved = call(lambda client: client.settings.get()).benchmark_services
+
+    catalog = call(lambda client: client.services.catalog())
+    services_by_name = {service.name: service for service in catalog.services}
+    services_by_name.update({name: BenchmarkServiceEntry(name=name, url=entry.url) for name, entry in saved.items()})
+    if not services_by_name:
+        click.echo(click.style("No benchmark services configured.", fg="yellow"))
+        return
+
+    paginate_services(
+        list(services_by_name.values()),
+        check_services=lambda entries: call(lambda client: client.services.check(entries)).services,
+    )
+
+
+def _service_source_domain(service: BenchmarkServiceHealth) -> str:
+    host = urlparse(service.url).netloc or service.url
+    return host.removeprefix(f"{service.name}.")
+
+
+def _health_checked_page(
+    services: Sequence[BenchmarkServiceEntry],
+    cache: dict[str, BenchmarkServiceHealth],
+    check_services: Callable[[list[BenchmarkServiceEntry]], list[BenchmarkServiceHealth]],
+) -> list[BenchmarkServiceHealth]:
+    missing = [service for service in services if service.name not in cache]
+    if missing:
+        cache.update({service.name: service for service in check_services(missing)})
+    for service in missing:
+        cache.setdefault(
+            service.name, BenchmarkServiceHealth(name=service.name, url=service.url, healthy=False, latency_ms=None)
+        )
+    return [cache[service.name] for service in services]
+
+
+def paginate_services(
+    services: Sequence[BenchmarkServiceEntry],
+    check_services: Callable[[list[BenchmarkServiceEntry]], list[BenchmarkServiceHealth]],
+    limit: int = 10,
+) -> None:
+    health_cache: dict[str, BenchmarkServiceHealth] = {}
+
+    def load_page(offset: int, page_limit: int) -> tuple[int, list[BenchmarkServiceHealth]]:
+        page_services = _health_checked_page(services[offset : offset + page_limit], health_cache, check_services)
+        return len(services), page_services
+
+    def render_page(
+        page_services: list[BenchmarkServiceHealth],
+        current_page: int,
+        total_pages: int,
+        total_count: int,
+    ) -> None:
+        rows = [
+            {
+                "Benchmark": service.name,
+                "Service URL": service.url,
+                "Source": _service_source_domain(service),
+                "Latency": f"{service.latency_ms} ms" if service.latency_ms is not None else "-",
+            }
+            for service in page_services
+        ]
+        format_table(
+            rows,
+            ["Benchmark", "Service URL", "Source", "Latency"],
+            current_page,
+            total_pages,
+            total_count,
+            "service",
+        )
+
+    paginate_cli_pages(
+        load_page,
+        render_page,
+        limit=limit,
+        render_empty=lambda: click.echo(click.style("No benchmark services found.", fg="yellow")),
+    )

valkyrie/cli/benchmark/tasks.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+
+import click
+
+from valkyrie.cli.client import call
+
+
+@click.command(
+    name="tasks",
+    help=(
+        "Export the task IDs of a benchmark dataset.\n"
+        "\n"
+        "Example:\n"
+        "    valkyrie benchmark tasks swebench --dataset default"
+    ),
+)
+@click.argument("benchmark_name", type=str)
+@click.option(
+    "--dataset",
+    type=str,
+    required=False,
+    default=None,
+    help="Dataset name to fetch from the benchmark service (defaults to 'default').",
+)
+@click.option(
+    "--header",
+    "-H",
+    "headers",
+    multiple=True,
+    nargs=2,
+    type=(str, str),
+    help="Custom header for benchmark service requests (e.g., -H Authorization my-credential).",
+)
+@click.option(
+    "--ignore-custom-services",
+    "--ics",
+    is_flag=True,
+    help="Use the hosted benchmark service instead of the URL saved with `valkyrie benchmark service set`.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(path_type=Path, file_okay=True, dir_okay=False),
+    required=False,
+    default=None,
+    help="Path to save task IDs, one per line. Defaults to <benchmark>-<dataset>-tasks.txt.",
+)
+def tasks(
+    benchmark_name: str,
+    dataset: str | None,
+    headers: tuple[tuple[str, str]],
+    ignore_custom_services: bool,
+    output: Path | None,
+):
+    task_ids = call(
+        lambda client: client.services.task_ids(
+            benchmark_name,
+            dataset=dataset,
+            service_headers=dict(headers),
+            ignore_custom_services=ignore_custom_services,
+        )
+    )
+    output_path = output or Path(f"{benchmark_name}-{dataset or 'default'}-tasks.txt")
+    output_path.write_text("\n".join(task_ids) + "\n", encoding="utf-8")
+    click.echo(f"Wrote {len(task_ids)} task IDs to {output_path}")

valkyrie/cli/client.py

@@ -0,0 +1,50 @@
+import asyncio
+import traceback
+from collections.abc import Awaitable, Callable
+from typing import Any, TypeVar
+
+import click
+from valkyrie.sdk import SessionAuth, ValkyrieAPIError, ValkyrieClient, ValkyrieConfig, ValkyrieSDKError
+
+from valkyrie.display import terminal_safe
+
+Result = TypeVar("Result")
+
+
+def call(operation: Callable[[ValkyrieClient], Awaitable[Result]], auth: SessionAuth | None = None) -> Result:
+
+    async def run() -> Result:
+        async with ValkyrieClient(auth=auth) as client:
+            return await operation(client)
+
+    try:
+        return asyncio.run(run())
+    except ValkyrieAPIError as error:
+        if error.status_code == 401:
+            raise click.ClickException(
+                _unauthorized(auth is not None or ValkyrieConfig().http_auth() is not None)
+            ) from error
+        message = f"{_describe(error.detail)} ({error.status_code})" if isinstance(error.detail, list) else str(error)
+        raise click.ClickException(terminal_safe(message, preserve_newlines=True)) from error
+    except (ValkyrieSDKError, ValueError, OSError) as error:
+        if isinstance(error, ValueError) and not _rejected_by_sdk(error):
+            raise
+        raise click.ClickException(terminal_safe(str(error), preserve_newlines=True)) from error
+
+
+def _rejected_by_sdk(error: ValueError) -> bool:
+    return isinstance(error, ValkyrieSDKError) or any(
+        frame.f_globals.get("__name__", "").startswith("valkyrie.sdk")
+        for frame, _ in traceback.walk_tb(error.__traceback__)
+    )
+
+
+def _unauthorized(has_credentials: bool) -> str:
+    problem = "The Tracker rejected your credentials." if has_credentials else "You are not logged in."
+    return f"{problem} Run `valkyrie login`."
+
+
+def _describe(detail: Any) -> Any:
+    if isinstance(detail, list):
+        return "; ".join(f"{error['loc'][-1]}: {error['msg']}" for error in detail)
+    return detail