manifold-cli 0.0.39__py3-none-any.whl

manifold_cli/commands/run.py

@@ -0,0 +1,337 @@
+"""The `manifold run` command group — submit and track policy x benchmark runs."""
+
+from typing import Annotated, assert_never
+from uuid import UUID
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+
+from manifold_cli.commands import run_log
+from manifold_cli.commands.organization import (
+    ORGANIZATION_CALL_ERRORS,
+    exit_for_call_error,
+    resolve_organization,
+)
+from manifold_cli.dependencies import Dependencies
+from manifold_cli.models.catalog import (
+    InvalidReferenceError,
+    join_versioned_ref,
+    split_versioned_ref,
+)
+from manifold_cli.models.organization import OrganizationNotFoundError
+from manifold_cli.models.run import (
+    Run,
+    RunCancelRejectedError,
+    RunDeleteRejectedError,
+    RunNotFoundError,
+    RunStatus,
+    RunSubmitError,
+)
+
+ORG_OPTION = typer.Option(
+    "--org", help="Organization identifier; required if you belong to more than one."
+)
+
+# How many runs `run list` shows when the caller does not say; the same default
+# the API applies to one page.
+DEFAULT_LIST_LIMIT = 50
+MIN_RUN_NAME_LEN = 3
+
+
+def _status(run: Run) -> str:
+    # NOTE: the assert_never arm is what makes a new RunStatus a typecheck
+    # failure here rather than a KeyError at print time.
+    if run.status is RunStatus.QUEUED:
+        color = "yellow"
+    elif (
+        run.status is RunStatus.DISPATCHED
+        or run.status is RunStatus.PREPARING
+        or run.status is RunStatus.STARTING
+        or run.status is RunStatus.RUNNING
+    ):
+        color = "cyan"
+    elif run.status is RunStatus.COMPLETED:
+        color = "green"
+    elif run.status is RunStatus.FAILED or run.status is RunStatus.CANCELED:
+        color = "red"
+    else:
+        assert_never(run.status)
+    return f"[{color}]{run.status.value}[/]"
+
+
+def _progress(run: Run) -> str:
+    """Episodes recorded against the count the benchmark version declares.
+
+    Includes the success rate once the first episode has landed.
+    """
+    progress = f"{run.completed_episodes}/{run.episodes} episodes"
+    if run.mean_success is None:
+        return progress
+    return f"{progress}, {run.mean_success:.0%} success"
+
+
+def _duration(run: Run) -> str:
+    """How long the run has been executing, or nothing while it has not started.
+
+    A queued run shows no duration rather than a zero: the server measures from
+    the first episode, so queue time is outside it.
+    """
+    if run.duration_sec is None:
+        return ""
+    return f"   [dim]duration[/dim] {run.duration_sec:.1f}s"
+
+
+def _pairing(run: Run) -> str:
+    return (
+        f"{escape(run.policy_slug)}:{escape(run.policy_version)} x "
+        f"{escape(run.benchmark_slug)}:{escape(run.benchmark_version)}"
+    )
+
+
+def _print_run(console: Console, run: Run) -> None:
+    label = f"  [dim]{escape(run.name)}[/dim]" if run.name else ""
+    console.print(
+        f"[bold]{run.id}[/bold]  "
+        f"{_pairing(run)}  "
+        f"{_progress(run)}  "
+        f"{_status(run)}{label}",
+        highlight=False,
+    )
+
+
+def _run_webapp_url(webapp_url: str, organization: str, run_id: UUID) -> str:
+    return f"{webapp_url.rstrip('/')}/{organization}/runs/{run_id}"
+
+
+def _validate_run_name(value: str) -> str:
+    name = value.strip()
+    if len(name) < MIN_RUN_NAME_LEN:
+        raise typer.BadParameter(
+            f"must be at least {MIN_RUN_NAME_LEN} characters", param_hint="'--name'"
+        )
+    return name
+
+
+def register(app: typer.Typer, deps: Dependencies) -> None:
+    console = Console()
+    run_app = typer.Typer(
+        name="run",
+        help="Submit and track policy x benchmark runs.",
+        no_args_is_help=True,
+    )
+    app.add_typer(run_app)
+    run_log.register(run_app, deps)
+
+    @run_app.command()
+    def submit(
+        policy: Annotated[
+            str,
+            typer.Argument(
+                help="Policy to run, as [<org>/]<identifier>[:<version>], for "
+                "example my-policy:0.0.1. Without a version, the newest "
+                "version runs."
+            ),
+        ],
+        benchmark: Annotated[
+            str,
+            typer.Argument(
+                help="Benchmark to run against, as "
+                "[<org>/]<identifier>[:<version>]. Without a version, the "
+                "newest version runs."
+            ),
+        ],
+        name: Annotated[
+            str,
+            typer.Option(
+                "--name",
+                callback=_validate_run_name,
+                help="Label for this run in the runs list.",
+            ),
+        ],
+        # NOTE: the default is None rather than 1. A default of 1 would make
+        # every submit that leaves the flag off override the benchmark version's
+        # count with a serial run.
+        shards: Annotated[
+            int | None,
+            typer.Option(
+                "--shards",
+                min=1,
+                help="Parallel shards to fan out over, overriding the benchmark "
+                "version's own count. Left off, the version's count stands. A "
+                "count above the run's episode count is lowered to it, and "
+                "nothing reports that.",
+            ),
+        ] = None,
+        shards_per_gpu: Annotated[
+            int | None,
+            typer.Option(
+                "--shards-per-gpu",
+                min=1,
+                help="Parallel shards per GPU of each benchmark runner, "
+                "overriding the benchmark version's own counts and "
+                "--shards.",
+            ),
+        ] = None,
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """Queue a run of a policy against a benchmark.
+
+        The run belongs to the organization --org names, and a private policy
+        or benchmark has to be owned by that organization; a public one may
+        come from anywhere. So the private entries of two organizations cannot
+        be paired, and a private pair cannot be run under a third
+        organization, even by a member of all of them.
+        """
+        try:
+            # Settled before the session is touched, so a mistyped reference
+            # costs no round trip.
+            policy_owner, policy_slug, policy_version = split_versioned_ref(policy)
+            benchmark_owner, benchmark_slug, benchmark_version = split_versioned_ref(
+                benchmark
+            )
+            org = resolve_organization(console, deps.organization_service, organization)
+            run = deps.run_service.submit(
+                org,
+                join_versioned_ref(policy_owner, policy_slug, None),
+                join_versioned_ref(benchmark_owner, benchmark_slug, None),
+                name=name,
+                shards=shards,
+                shards_per_gpu=shards_per_gpu,
+                policy_version=policy_version,
+                benchmark_version=benchmark_version,
+            )
+        except InvalidReferenceError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except RunSubmitError as exc:
+            console.print(f"[red]Run rejected:[/red] {escape(str(exc))}")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(f"[green]✓ Run queued[/green] [bold]{run.id}[/bold]")
+        console.print(f"  {_pairing(run)}, {run.episodes} episodes", highlight=False)
+        console.print(f"Follow it with: manifold run watch {run.id}")
+        run_url = _run_webapp_url(deps.manifold_webapp_url, org, run.id)
+        console.print(f"View it at: {escape(run_url)}", soft_wrap=True)
+
+    @run_app.command("list")
+    def list_runs(
+        limit: Annotated[
+            int, typer.Option("--limit", min=1, help="Runs to show (newest first).")
+        ] = DEFAULT_LIST_LIMIT,
+        organization: Annotated[str | None, ORG_OPTION] = None,
+    ) -> None:
+        """List your organization's runs, newest first."""
+        try:
+            org = resolve_organization(console, deps.organization_service, organization)
+            runs, total = deps.run_service.list(org, limit=limit)
+        except OrganizationNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if len(runs) == 0:
+            console.print("No runs yet. Submit one with: manifold run submit")
+            return
+        for run in runs:
+            _print_run(console, run)
+        if len(runs) < total:
+            console.print(
+                f"[dim]Showing {len(runs)} of {total}; raise --limit for more.[/dim]"
+            )
+
+    @run_app.command()
+    def get(
+        run_id: Annotated[UUID, typer.Argument(help="Run id.")],
+        episodes: Annotated[
+            bool,
+            typer.Option("--episodes", help="Also list the episodes recorded so far."),
+        ] = False,
+    ) -> None:
+        """Show one run's progress and outcome."""
+        try:
+            run = deps.run_service.get(run_id)
+            recorded = deps.run_service.episodes(run_id) if episodes else []
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        _print_run(console, run)
+        console.print(
+            f"  [dim]visibility[/dim] {run.visibility.value}   "
+            f"[dim]submitted[/dim] {run.created_at:%Y-%m-%d %H:%M:%S}"
+            f"{_duration(run)}"
+        )
+        if not episodes:
+            return
+        if len(recorded) == 0:
+            console.print("  No episodes recorded yet.")
+            return
+        for episode in recorded:
+            outcome = "[green]✓[/green]" if episode.success else "[red]✗[/red]"
+            console.print(
+                f"  {outcome} [bold]{episode.episode_idx}[/bold]  "
+                f"{escape(episode.task_name)}  "
+                f"[dim]{episode.steps} steps, {episode.elapsed_sec:.1f}s[/dim]"
+            )
+
+    @run_app.command()
+    def cancel(
+        run_id: Annotated[UUID, typer.Argument(help="Run id.")],
+    ) -> None:
+        """Stop a run, keeping the episodes it has already recorded."""
+        try:
+            run = deps.run_service.cancel(run_id)
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except RunCancelRejectedError as exc:
+            console.print(f"[red]Run not canceled:[/red] {escape(str(exc))}")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(f"[green]✓ Run canceled[/green] [bold]{run.id}[/bold]")
+        console.print(f"  {_pairing(run)}, {_progress(run)}", highlight=False)
+
+    @run_app.command()
+    def delete(
+        run_id: Annotated[UUID, typer.Argument(help="Run id.")],
+    ) -> None:
+        """Delete a run that has reached a terminal status.
+
+        A deleted run is left out of `run list`, and `run get` reports it as not
+        found. Stop a run that is still in progress with `run cancel` first.
+        """
+        try:
+            deps.run_service.delete(run_id)
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except RunDeleteRejectedError as exc:
+            console.print(f"[red]Run not deleted:[/red] {escape(str(exc))}")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        console.print(f"[green]✓ Run deleted[/green] [bold]{run_id}[/bold]")
+
+    @run_app.command()
+    def watch(
+        run_id: Annotated[UUID, typer.Argument(help="Run id.")],
+    ) -> None:
+        """Follow a run until it reaches a terminal status."""
+        try:
+            final = deps.run_service.watch(
+                run_id, on_update=lambda run: _print_run(console, run)
+            )
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Stopped watching (the run continues).[/yellow]")
+            raise typer.Exit(130) from None
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if final.status is not RunStatus.COMPLETED:
+            raise typer.Exit(1)

manifold_cli/commands/run_log.py

@@ -0,0 +1,291 @@
+"""The `manifold run logs` command group — list, search and read a run's output.
+
+A stream is identified by its assignment id, producer, worker index and source.
+`streams` prints those four values in the order `run logs read` takes them, and
+`search` prints the `run logs read` command for each match, so an agent can run
+the next step without rebuilding its arguments.
+"""
+
+from datetime import datetime
+from typing import Annotated
+from uuid import UUID
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+from rich.table import Table
+
+from manifold_cli.commands.organization import (
+    ORGANIZATION_CALL_ERRORS,
+    exit_for_call_error,
+)
+from manifold_cli.commands.table import print_table
+from manifold_cli.dependencies import Dependencies
+from manifold_cli.models.log import LogLine, LogMatch, LogProducer, LogSource
+from manifold_cli.models.pagination import SortOrder
+from manifold_cli.models.run import RunNotFoundError
+
+RUN_ID_ARGUMENT = typer.Argument(help="Run id.")
+
+
+def register(run_app: typer.Typer, deps: Dependencies) -> None:
+    console = Console()
+    logs_app = typer.Typer(
+        name="logs",
+        help="List, search and read the output of a run's runners and workers.",
+        no_args_is_help=True,
+    )
+    run_app.add_typer(logs_app)
+
+    @logs_app.command()
+    def streams(run_id: Annotated[UUID, RUN_ID_ARGUMENT]) -> None:
+        """List every output stream of every assignment of a run.
+
+        Start here when investigating a run's output. Prints each stream's line
+        count and its process's exit code. The first four columns are the
+        arguments `run logs read` takes.
+        """
+        try:
+            run_streams = deps.run_service.log_streams(run_id)
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if len(run_streams) == 0:
+            console.print("This run has not recorded any output yet.")
+            return
+        table = Table(box=None, pad_edge=False)
+        for heading in (
+            "Assignment",
+            "Producer",
+            "Worker",
+            "Source",
+            "Side",
+            "Status",
+            "Lines",
+            "Exit",
+        ):
+            table.add_column(heading, no_wrap=True)
+        for stream in run_streams:
+            status = stream.status.value
+            if stream.reason is not None:
+                status = f"{status} ({stream.reason.value})"
+            table.add_row(
+                str(stream.assignment_id),
+                stream.producer.value,
+                str(stream.worker_index),
+                stream.source.value,
+                stream.side.value,
+                status,
+                str(stream.line_count),
+                "-" if stream.exit_code is None else str(stream.exit_code),
+            )
+        print_table(console, table)
+        console.print(
+            f"Search them with: manifold run logs search {run_id} PATTERN\n"
+            f"Read one with: manifold run logs read {run_id} "
+            "ASSIGNMENT_ID PRODUCER WORKER SOURCE",
+            soft_wrap=True,
+        )
+
+    @logs_app.command()
+    def search(
+        run_id: Annotated[UUID, RUN_ID_ARGUMENT],
+        pattern: Annotated[
+            str,
+            typer.Argument(
+                help="Case-insensitive POSIX regular expression, such as "
+                "'traceback|error'."
+            ),
+        ],
+        assignment_id: Annotated[
+            UUID | None,
+            typer.Option(
+                "--assignment-id", help="Search only this assignment's streams."
+            ),
+        ] = None,
+        context_lines: Annotated[
+            int | None,
+            typer.Option(
+                "--context",
+                "-C",
+                min=0,
+                help="Lines to show before and after each match.  [default: 3]",
+            ),
+        ] = None,
+        since: Annotated[
+            datetime | None,
+            typer.Option(
+                parser=_parse_aware_datetime,
+                metavar="TIMESTAMP",
+                help="Search only lines written at or after this ISO 8601 "
+                "timestamp with a time zone.",
+            ),
+        ] = None,
+        until: Annotated[
+            datetime | None,
+            typer.Option(
+                parser=_parse_aware_datetime,
+                metavar="TIMESTAMP",
+                help="Search only lines written before this ISO 8601 timestamp "
+                "with a time zone.",
+            ),
+        ] = None,
+        order: Annotated[
+            SortOrder,
+            typer.Option(
+                help="Print the oldest match first with asc, or the newest first "
+                "with desc."
+            ),
+        ] = SortOrder.ASC,
+        cursor: Annotated[
+            str | None,
+            typer.Option(
+                help="Print the page that follows this cursor from an earlier search."
+            ),
+        ] = None,
+    ) -> None:
+        """Search a run's output for lines matching a regular expression.
+
+        Prints up to 20 matches per page. Each match's header is the `run logs
+        read` command that centres on it, followed by the time the line was
+        written, and `>` marks each matching line.
+        """
+        try:
+            page = deps.run_service.search_log(
+                run_id,
+                pattern=pattern,
+                assignment_id=assignment_id,
+                context_lines=context_lines,
+                since=since,
+                until=until,
+                order=order,
+                cursor=cursor,
+            )
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if page.total == 0:
+            console.print("0 lines match.")
+            return
+        for match in page.items:
+            _print_match(console, run_id, match)
+        console.print(
+            f"Showing {len(page.items)} of {page.total} matches.", soft_wrap=True
+        )
+        if page.next_cursor is not None:
+            console.print(
+                "Next page: repeat the command with --cursor "
+                f"{escape(page.next_cursor)}, or narrow the search with "
+                "--assignment-id, --since or --until.",
+                soft_wrap=True,
+            )
+
+    @logs_app.command()
+    def read(
+        run_id: Annotated[UUID, RUN_ID_ARGUMENT],
+        assignment_id: Annotated[
+            UUID,
+            typer.Argument(
+                metavar="ASSIGNMENT_ID", help="Assignment id from `run logs streams`."
+            ),
+        ],
+        producer: Annotated[LogProducer, typer.Argument(help="runner or worker.")],
+        worker_index: Annotated[
+            int,
+            typer.Argument(
+                metavar="WORKER", min=0, help="Worker index; 0 for a runner stream."
+            ),
+        ],
+        source: Annotated[LogSource, typer.Argument(help="stdout or stderr.")],
+        around_seq: Annotated[
+            int | None,
+            typer.Option(
+                "--around",
+                min=0,
+                help="Centre the window on this seq. Without it, print the stream's "
+                "tail.",
+            ),
+        ] = None,
+        lines: Annotated[
+            int | None,
+            typer.Option(
+                "--lines",
+                "-n",
+                min=1,
+                help="Lines to show.  [default: 50]",
+            ),
+        ] = None,
+    ) -> None:
+        """Read a window of one stream from `run logs streams`.
+
+        Each line starts with its seq, and `>` marks the `--around` line. Move
+        the window by passing a seq near either end of the lines shown.
+        """
+        try:
+            stream_lines = deps.run_service.read_log(
+                run_id,
+                assignment_id=assignment_id,
+                producer=producer,
+                worker_index=worker_index,
+                source=source,
+                around_seq=around_seq,
+                lines=lines,
+            )
+        except RunNotFoundError as exc:
+            console.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(1) from exc
+        except ORGANIZATION_CALL_ERRORS as exc:
+            raise exit_for_call_error(console, exc) from exc
+        if len(stream_lines) == 0:
+            console.print("This stream is empty or does not exist.")
+            return
+        _print_lines(stream_lines, marked_seq=around_seq)
+        if around_seq is not None and all(
+            line.seq != around_seq for line in stream_lines
+        ):
+            console.print(f"[yellow]seq {around_seq} is not in this stream.[/yellow]")
+
+
+def _parse_aware_datetime(value: str) -> datetime:
+    """An ISO 8601 timestamp with a time zone, which the search API requires."""
+    try:
+        instant = datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise typer.BadParameter(f"'{value}' is not an ISO 8601 timestamp") from exc
+    if instant.tzinfo is None:
+        raise typer.BadParameter(
+            f"'{value}' does not include a time zone; add one, such as {value}Z "
+            f"or {value}+00:00"
+        )
+    return instant
+
+
+def _print_match(console: Console, run_id: UUID, match: LogMatch) -> None:
+    """The `run logs read` command centred on a match and the time the matched
+    line was written, then the match's lines with context."""
+    matched = next(line for line in match.lines if line.seq == match.seq)
+    console.print(
+        f"manifold run logs read {run_id} {match.assignment_id} "
+        f"{match.producer.value} {match.worker_index} {match.source.value} "
+        f"--around {match.seq}  # {matched.ts.isoformat()}",
+        soft_wrap=True,
+        highlight=False,
+    )
+    _print_lines(match.lines, marked_seq=match.seq)
+    console.print()
+
+
+def _print_lines(stream_lines: list[LogLine], *, marked_seq: int | None) -> None:
+    """One line per log line: a `>` marker, the seq, and the text as written.
+
+    Log text bypasses Rich, which would substitute emoji codes such as `:100:`,
+    drop carriage returns and expand tabs.
+    """
+    seq_width = len(str(stream_lines[-1].seq))
+    for line in stream_lines:
+        marker = ">" if line.seq == marked_seq else " "
+        typer.echo(f"{marker} {line.seq:>{seq_width}}  {line.text}")