endpoints-submission-cli 0.1.0__py3-none-any.whl

endpoints_submission_cli/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""endpoints-submission-cli: CLI for managing MLPerf rolling submission runs and submissions."""
+
+__version__ = "0.1.0"
+__all__ = ["__version__"]

endpoints_submission_cli/_http.py

@@ -0,0 +1,125 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""HTTP transport layer for the MLPerf Submission API (httpx-based).
+
+All HTTP plumbing lives here: timeouts, base-URL resolution, auth header,
+error translators, and the four verb helpers used by the domain API modules.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, NoReturn
+
+import httpx
+
+from .exceptions import APIError, AuthError
+
+__all__ = [
+    "get_token",
+    "_base_url",
+    "_headers",
+    "_raise_status",
+    "_raise_request",
+    "_get",
+    "_post",
+    "_patch",
+    "_delete",
+    "_UPLOAD_TIMEOUT",
+    "_DOWNLOAD_TIMEOUT",
+    "_API_TIMEOUT",
+]
+
+_DEFAULT_BASE_URL = "http://localhost:8080"
+
+# All values are per-operation idle timeouts (time without a byte transferred), not wall-clock totals.
+# A slow-but-steady transfer never times out; only a stalled connection does.
+_UPLOAD_TIMEOUT = httpx.Timeout(connect=10.0, write=300.0, read=120.0, pool=5.0)   # write: each chunk send; read: wait for server ack after full upload
+_DOWNLOAD_TIMEOUT = httpx.Timeout(connect=10.0, read=300.0, write=30.0, pool=5.0)  # read: gap between received bytes; accommodates large archives on slow links
+_API_TIMEOUT = httpx.Timeout(connect=10.0, read=30.0, write=30.0, pool=5.0)        # standard JSON calls; no large payloads expected
+
+
+def _base_url() -> str:
+    return os.environ.get("MLPERF_API_BASE_URL", _DEFAULT_BASE_URL)
+
+
+def get_token(explicit_token: str | None) -> str:
+    """Resolve the API token from the explicit flag or PRISM_USER_API_TOKEN env var."""
+    token = explicit_token or os.environ.get("PRISM_USER_API_TOKEN")
+    if not token:
+        raise AuthError(
+            "No API token provided. Pass --token or set PRISM_USER_API_TOKEN."
+        )
+    return token
+
+
+def _headers(token: str) -> dict[str, str]:
+    return {"X-API-Key": token}
+
+
+# Translates an httpx HTTP error (4xx/5xx response) into our own APIError/AuthError.
+def _raise_status(exc: httpx.HTTPStatusError) -> NoReturn:
+    if exc.response.status_code in (401, 403):
+        raise AuthError(f"Authentication failed (HTTP {exc.response.status_code})") from exc
+    body = exc.response.text[:500]  # truncate to avoid dumping full HTML/JSON error pages into logs
+    raise APIError(f"API error {exc.response.status_code}: {body}") from exc
+
+
+# Translates an httpx network/transport error (timeout, connection refused, DNS failure, etc.)
+def _raise_request(exc: httpx.RequestError) -> NoReturn:
+    raise APIError(f"Request failed: {type(exc).__name__}: {exc}") from exc
+
+
+def _get(path: str, token: str, **kwargs: Any) -> Any:
+    try:
+        r = httpx.get(
+            f"{_base_url()}{path}", headers=_headers(token), timeout=_API_TIMEOUT, **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as exc:
+        _raise_status(exc)
+    except httpx.RequestError as exc:
+        _raise_request(exc)
+
+
+def _post(path: str, token: str, **kwargs: Any) -> Any:
+    try:
+        r = httpx.post(
+            f"{_base_url()}{path}", headers=_headers(token), timeout=_API_TIMEOUT, **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as exc:
+        _raise_status(exc)
+    except httpx.RequestError as exc:
+        _raise_request(exc)
+
+
+def _patch(path: str, token: str, **kwargs: Any) -> Any:
+    try:
+        r = httpx.patch(
+            f"{_base_url()}{path}", headers=_headers(token), timeout=_API_TIMEOUT, **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+    except httpx.HTTPStatusError as exc:
+        _raise_status(exc)
+    except httpx.RequestError as exc:
+        _raise_request(exc)
+
+
+def _delete(path: str, token: str, **kwargs: Any) -> Any:
+    try:
+        r = httpx.delete(
+            f"{_base_url()}{path}", headers=_headers(token), timeout=_API_TIMEOUT, **kwargs
+        )
+        r.raise_for_status()
+        # 204 No Content returns empty body
+        if r.status_code == 204:
+            return None
+        return r.json()
+    except httpx.HTTPStatusError as exc:
+        _raise_status(exc)
+    except httpx.RequestError as exc:
+        _raise_request(exc)

endpoints_submission_cli/commands/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""CLI command groups for endpoints-submission-cli."""

endpoints_submission_cli/commands/common.py

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""Shared helpers for all CLI commands."""
+
+from __future__ import annotations
+
+import datetime
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.json import JSON
+from rich.table import Table
+
+from .. import _http
+from ..exceptions import AuthError, SubmissionCheckError
+
+__all__ = ["_console", "_get_token", "_SEVERITY_STYLE", "_run_submission_checker", "output_json"]
+
+_console = Console(stderr=True)
+_stdout_console = Console()
+
+_SEVERITY_STYLE = {
+    "error": "bold red",
+    "warning": "yellow",
+    "info": "dim",
+}
+
+
+def output_json(data: Any) -> None:
+    """Print *data* as syntax-highlighted JSON (plain when piped)."""
+    _stdout_console.print(JSON(json.dumps(data, default=str)))
+
+
+def _get_token(token: str | None) -> str:
+    try:
+        return _http.get_token(token)
+    except AuthError as exc:
+        _console.print(f"[bold red]Auth error:[/bold red] {exc}")
+        sys.exit(1)
+
+
+def _run_submission_checker(submission_dir: Path) -> None:
+    """Run the SubmissionChecker on *submission_dir*; print results table; raise on errors."""
+    from submission_checker.checker import SubmissionChecker
+
+    report = SubmissionChecker(submission_dir).run()
+
+    # --- write Rich table to log file ---
+    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    log_path = Path.cwd() / f"submission_checker_{timestamp}.log"
+
+    table = Table(show_lines=True, expand=False)
+    table.add_column("Rule", style="cyan", no_wrap=True)
+    table.add_column("§ Ref", no_wrap=True)
+    table.add_column("Severity", no_wrap=True)
+    table.add_column("Message")
+    table.add_column("Path")
+
+    for r in report.results:
+        sev = r.severity.value
+        style = _SEVERITY_STYLE.get(sev, "")
+        table.add_row(
+            r.rule,
+            r.spec_ref,
+            f"[{style}]{sev}[/{style}]" if style else sev,
+            r.message,
+            str(r.path) if r.path else "",
+        )
+
+    with open(log_path, "w", encoding="utf-8") as fh:
+        file_console = Console(file=fh, no_color=True, width=220)
+        file_console.print(
+            f"Submission Checker Report — {timestamp}\n"
+            f"Directory : {submission_dir}\n"
+            f"Results   : {len(report.results)} checks"
+            f" ({len(report.errors)} error(s), {len(report.warnings)} warning(s))\n"
+        )
+        file_console.print(table)
+
+    _console.print(f"[dim]Checker report written to {log_path}[/dim]")
+
+    errors = report.errors
+    if errors:
+        msgs = "\n".join(f"  [{e.rule}] {e.message}" for e in errors)
+        raise SubmissionCheckError(f"Submission checker found {len(errors)} error(s):\n{msgs}")

endpoints_submission_cli/commands/runs/__init__.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs command group."""
+
+import click
+
+from .create import runs_create
+from .delete import runs_delete
+from .get import runs_get
+from .list import runs_list
+from .pin import runs_pin
+from .unpin import runs_unpin
+
+__all__ = ["runs"]
+
+
+@click.group(name="runs")
+def runs() -> None:
+    """Manage benchmark runs."""
+
+
+runs.add_command(runs_list)
+runs.add_command(runs_create)
+runs.add_command(runs_get)
+runs.add_command(runs_delete)
+runs.add_command(runs_pin)
+runs.add_command(runs_unpin)

endpoints_submission_cli/commands/runs/create.py

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs create command."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+import click
+
+from ...exceptions import APIError, ArchiveError, RunFolderError
+from ...runs import api as runs_api
+from ...runs.parser import build_archive, parse_run_folder
+from ..common import _console, _get_token
+
+__all__ = ["runs_create"]
+
+
+@click.command("create")
+@click.option(
+    "--path",
+    "path",
+    required=True,
+    type=click.Path(path_type=Path),
+    help="Path to the local run folder.",
+)
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+@click.option(
+    "--expires-at",
+    default=None,
+    help="Expiry datetime in ISO 8601 format (e.g. 2026-01-01T00:00:00). Defaults to server policy.",
+)
+@click.option(
+    "--pinned",
+    is_flag=True,
+    default=False,
+    help="Pin the run immediately to prevent automatic expiry.",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    help="Print the parsed payload as JSON and exit without calling the API.",
+)
+def runs_create(path: Path, token: str | None, expires_at: str | None, pinned: bool, dry_run: bool) -> None:
+    """Create a run from a local benchmark result folder.
+
+    Parses system_info.json, config.yaml, and result_summary.json from PATH,
+    registers the run with the Submission API, and uploads the run folder as
+    an archive.  If the archive upload fails the run record is deleted
+    (rollback to clean state).
+    """
+    try:
+        payload = parse_run_folder(path)
+    except RunFolderError as exc:
+        _console.print(f"[bold red]Run folder error:[/bold red] {exc}")
+        sys.exit(1)
+
+    if expires_at is not None:
+        payload["expires_at"] = expires_at
+    if pinned:
+        payload["pinned"] = True
+
+    if dry_run:
+        print(json.dumps(payload, indent=2, default=str))
+        return
+
+    resolved_token = _get_token(token)
+
+    try:
+        run_out = runs_api.create_run(resolved_token, payload)
+    except APIError as exc:
+        _console.print(f"[bold red]API error creating run:[/bold red] {exc}")
+        sys.exit(1)
+
+    run_id: str = run_out["id"]
+
+    # At this point the run record exists in the DB but has no archive yet.
+    # If the upload fails we must roll back by deleting the run record so we
+    # don't leave an orphaned entry with no associated data.
+    with tempfile.TemporaryDirectory() as tmp:
+        archive_path = build_archive(path, Path(tmp) / f"{path.name}.tar.gz")
+        try:
+            upload_result = runs_api.upload_run_archive(resolved_token, run_id, archive_path)
+        except (APIError, ArchiveError, OSError) as exc:
+            _console.print(
+                f"[bold red]Archive upload failed:[/bold red] {exc}\n"
+                f"[yellow]Rolling back run {run_id}…[/yellow]"
+            )
+            # Step 1: attempt to delete the partial archive from storage (best-effort,
+            # ignore failure — storage may not have received anything).
+            with contextlib.suppress(APIError):
+                runs_api.delete_run_archive(resolved_token, run_id)
+            # Step 2: delete the run DB record to restore clean state.
+            try:
+                runs_api.delete_run(resolved_token, run_id)
+                _console.print("[green]Rollback successful — run deleted.[/green]")
+            except APIError as rb_exc:
+                _console.print(
+                    f"[bold red]Rollback also failed:[/bold red] {rb_exc}\n"
+                    f"Orphaned run ID: {run_id}"
+                )
+            sys.exit(1)
+
+    archive_uri = upload_result.get("archive_uri") if upload_result else None
+    _console.print(f"[bold green]Run created:[/bold green] {run_id}")
+    if archive_uri:
+        _console.print(f"[dim]Archive:[/dim] {archive_uri}")

endpoints_submission_cli/commands/runs/delete.py

@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs delete command."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ...exceptions import APIError
+from ...runs import api as runs_api
+from ..common import _console, _get_token
+
+__all__ = ["runs_delete"]
+
+
+@click.command("delete")
+@click.option("--run-id", required=True, help="Run UUID.")
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+def runs_delete(run_id: str, token: str | None) -> None:
+    """Delete a run and its stored archive.
+
+    If the run is part of an active submission the API will reject the request.
+    Withdraw the submission first in that case.
+
+    The archive is removed from GCS first so that a DB-delete failure leaves
+    nothing orphaned in storage.  A 404 on archive deletion means the run has
+    no archive and is silently skipped.
+    """
+    resolved_token = _get_token(token)
+
+    try:
+        runs_api.delete_run_archive(resolved_token, run_id)
+    except APIError as exc:
+        if "404" not in str(exc):
+            _console.print(
+                f"[yellow]Warning:[/yellow] Archive deletion failed: {exc}\n"
+                "Continuing to delete the run record."
+            )
+
+    try:
+        runs_api.delete_run(resolved_token, run_id)
+    except APIError as exc:
+        _console.print(f"[bold red]Error deleting run:[/bold red] {exc}")
+        sys.exit(1)
+
+    _console.print(f"[bold green]Run deleted:[/bold green] {run_id}")

endpoints_submission_cli/commands/runs/get.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs get command."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ...exceptions import APIError
+from ...runs import api as runs_api
+from ..common import _console, _get_token, output_json
+
+__all__ = ["runs_get"]
+
+
+@click.command("get")
+@click.option("--run-id", required=True, help="Run UUID.")
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+def runs_get(run_id: str, token: str | None) -> None:
+    """Get full details of a specific run."""
+    resolved_token = _get_token(token)
+    try:
+        run = runs_api.get_run(resolved_token, run_id)
+    except APIError as exc:
+        _console.print(f"[bold red]Error:[/bold red] {exc}")
+        sys.exit(1)
+
+    output_json(run)

endpoints_submission_cli/commands/runs/list.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs list command."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ...exceptions import APIError
+from ...runs import api as runs_api
+from ...runs.formatters import print_runs_table
+from ..common import _console, _get_token, output_json
+
+__all__ = ["runs_list"]
+
+
+@click.command("list")
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+@click.option("-j", "--json", "as_json", is_flag=True, default=False, help="Output raw JSON.")
+def runs_list(token: str | None, as_json: bool) -> None:
+    """List all runs for the authenticated user."""
+    resolved_token = _get_token(token)
+    try:
+        run_list = runs_api.list_runs(resolved_token)
+    except APIError as exc:
+        _console.print(f"[bold red]Error:[/bold red] {exc}")
+        sys.exit(1)
+
+    if as_json:
+        output_json(run_list)
+    else:
+        print_runs_table(run_list)

endpoints_submission_cli/commands/runs/pin.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs pin command."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ...exceptions import APIError
+from ...runs import api as runs_api
+from ..common import _console, _get_token
+
+__all__ = ["runs_pin"]
+
+
+@click.command("pin")
+@click.option("--run-id", required=True, help="Run UUID.")
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+def runs_pin(run_id: str, token: str | None) -> None:
+    """Pin a run to prevent expiry (sets expires_at = null)."""
+    resolved_token = _get_token(token)
+    try:
+        runs_api.pin_run(resolved_token, run_id)
+    except APIError as exc:
+        _console.print(f"[bold red]Error:[/bold red] {exc}")
+        sys.exit(1)
+    _console.print(f"[bold green]Run pinned:[/bold green] {run_id}")

endpoints_submission_cli/commands/runs/unpin.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""runs unpin command."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from ...exceptions import APIError
+from ...runs import api as runs_api
+from ..common import _console, _get_token
+
+__all__ = ["runs_unpin"]
+
+
+@click.command("unpin")
+@click.option("--run-id", required=True, help="Run UUID.")
+@click.option(
+    "--token",
+    envvar="PRISM_USER_API_TOKEN",
+    default=None,
+    help="PRISM API key (mlc_...).",
+)
+def runs_unpin(run_id: str, token: str | None) -> None:
+    """Unpin a run to restore normal expiry behaviour."""
+    resolved_token = _get_token(token)
+    try:
+        runs_api.unpin_run(resolved_token, run_id)
+    except APIError as exc:
+        _console.print(f"[bold red]Error:[/bold red] {exc}")
+        sys.exit(1)
+    _console.print(f"[bold green]Run unpinned:[/bold green] {run_id}")

endpoints_submission_cli/commands/submissions/__init__.py

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""submissions command group."""
+
+import click
+
+from .add_run import submissions_add_run
+from .create import submissions_create
+from .get import submissions_get
+from .list import submissions_list
+from .remove_run import submissions_remove_run
+from .update import submissions_update
+from .withdraw import submissions_withdraw
+
+__all__ = ["submissions"]
+
+
+@click.group(name="submissions")
+def submissions() -> None:
+    """Manage MLPerf submissions."""
+
+
+submissions.add_command(submissions_list)
+submissions.add_command(submissions_create)
+submissions.add_command(submissions_get)
+submissions.add_command(submissions_update)
+submissions.add_command(submissions_withdraw)
+submissions.add_command(submissions_add_run)
+submissions.add_command(submissions_remove_run)