benchloop-client 0.1.6__py3-none-any.whl

aliyun/bmcli/__init__.py

@@ -0,0 +1,3 @@
+"""aliyun.bmcli — bmcli command line tool for benchmark analysis data."""
+
+__version__ = "0.1.0"

aliyun/bmcli/__main__.py

@@ -0,0 +1,26 @@
+"""Executable entry point for the ``bmcli`` command."""
+
+import sys
+
+from aliyun.bmcli import output
+from aliyun.bmcli.cli import app
+from aliyun.bmcli.client import ApiError, AuthError, NetworkError
+from aliyun.bmcli.exit_codes import AUTH, NETWORK, PARTIAL
+
+
+def main() -> None:
+    try:
+        app()
+    except AuthError as exc:
+        output.error(str(exc))
+        sys.exit(AUTH)
+    except NetworkError as exc:
+        output.error(str(exc))
+        sys.exit(NETWORK)
+    except ApiError as exc:
+        output.error(str(exc))
+        sys.exit(PARTIAL)
+
+
+if __name__ == "__main__":
+    main()

aliyun/bmcli/cli.py

@@ -0,0 +1,136 @@
+"""Typer application: global options, subcommand groups, and `config set`."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+
+from aliyun.bmcli import __version__, output
+from aliyun.bmcli.commands import (
+    artifacts as artifacts_cmd,
+    jobs as jobs_cmd,
+    metrics as metrics_cmd,
+    overview as overview_cmd,
+    pull as pull_cmd,
+    report as report_cmd,
+    tasks as tasks_cmd,
+    trajectory as trajectory_cmd,
+)
+from aliyun.bmcli.config import load_config, write_config
+from aliyun.bmcli.exit_codes import OK
+from aliyun.bmcli.runtime import Runtime
+
+app = typer.Typer(
+    name="bmcli",
+    help="Benchloop CLI — fetch and download benchmark analysis data.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"bmcli {__version__}")
+        raise typer.Exit(code=OK)
+
+
+@app.callback()
+def main(
+    ctx: typer.Context,
+    api_key: Optional[str] = typer.Option(
+        None, "--api-key", help="Personal api_key (highest precedence).", show_default=False
+    ),
+    host: Optional[str] = typer.Option(
+        None, "--host", help="Override backend base URL.", show_default=False
+    ),
+    output_dir: Optional[str] = typer.Option(
+        None, "-o", "--output-dir", help="Download output root directory (default '.').",
+        show_default=False,
+    ),
+    pretty: bool = typer.Option(
+        True, "--pretty/--no-pretty", help="Pretty-print JSON output."
+    ),
+    concurrency: int = typer.Option(
+        5, "--concurrency", help="Concurrency for multi-id operations."
+    ),
+    timeout: float = typer.Option(
+        300.0, "--timeout", help="Per-request timeout in seconds."
+    ),
+    retry: int = typer.Option(
+        2, "--retry", help="Retry count on failure (exponential backoff)."
+    ),
+    cluster: Optional[str] = typer.Option(
+        None, "--cluster", help="Cluster name (resolved by backend if omitted).",
+        show_default=False,
+    ),
+    verbose: bool = typer.Option(
+        False, "-v", "--verbose", help="Debug logs to stderr."
+    ),
+    quiet: bool = typer.Option(
+        False, "-q", "--quiet", help="Only output errors."
+    ),
+    config_path: Optional[str] = typer.Option(
+        None, "--config", help="Path to a config file.", show_default=False
+    ),
+    version: bool = typer.Option(
+        False, "--version", callback=_version_callback, is_eager=True,
+        help="Show version and exit.",
+    ),
+) -> None:
+    """Resolve config + runtime options and stash them on the context."""
+    output.set_verbosity(quiet=quiet, verbose=verbose)
+    cfg = load_config(
+        api_key=api_key,
+        host=host,
+        output_dir=output_dir,
+        config_path=config_path,
+    )
+    ctx.obj = Runtime(
+        config=cfg,
+        concurrency=concurrency,
+        timeout=timeout,
+        retry=retry,
+        cluster=cluster,
+        pretty=pretty,
+        output_dir=cfg.output_dir,
+        verbose=verbose,
+        quiet=quiet,
+    )
+
+
+# --- config command group -------------------------------------------------
+
+config_app = typer.Typer(help="Manage local configuration (~/.benchloop/config.toml).")
+app.add_typer(config_app, name="config")
+
+
+@config_app.command("set")
+def config_set(
+    api_key: Optional[str] = typer.Option(None, "--api-key", show_default=False),
+    host: Optional[str] = typer.Option(None, "--host", show_default=False),
+    output_dir: Optional[str] = typer.Option(None, "-o", "--output-dir", show_default=False),
+    config_path: Optional[str] = typer.Option(None, "--config", show_default=False),
+) -> None:
+    """Write api_key / host / output_dir to ~/.benchloop/config.toml."""
+    if api_key is None and host is None and output_dir is None:
+        raise typer.BadParameter("provide at least one of --api-key/--host/--output-dir")
+    path = write_config(
+        api_key=api_key,
+        host=host,
+        output_dir=output_dir,
+        config_path=config_path,
+    )
+    output.log(f"wrote config to {path}")
+
+
+# --- resource sub-apps -----------------------------------------------------
+
+app.add_typer(metrics_cmd.app, name="metrics")
+app.add_typer(trajectory_cmd.app, name="trajectory")
+app.add_typer(report_cmd.app, name="report")
+app.add_typer(overview_cmd.app, name="overview")
+app.add_typer(tasks_cmd.app, name="tasks")
+app.add_typer(jobs_cmd.app, name="jobs")
+app.add_typer(artifacts_cmd.app, name="artifacts")
+app.command("pull")(pull_cmd.pull)

aliyun/bmcli/client.py

@@ -0,0 +1,145 @@
+"""HTTP client wrapping httpx with retries, typed errors and file downloads."""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+import httpx
+
+
+class ClientError(Exception):
+    """Base class for all client-side errors."""
+
+
+class AuthError(ClientError):
+    """Raised on HTTP 401 (invalid / missing api key)."""
+
+
+class NetworkError(ClientError):
+    """Raised on timeout or connection failure."""
+
+
+class ApiError(ClientError):
+    """Raised on a non-2xx HTTP response other than 401."""
+
+    def __init__(self, status: int, body: Any) -> None:
+        self.status = status
+        self.body = body
+        super().__init__(f"API error {status}: {body}")
+
+
+class ApiClient:
+    """Thin wrapper over ``httpx.Client`` for the benchmark openapi."""
+
+    def __init__(
+        self,
+        host: str,
+        api_key: Optional[str] = None,
+        timeout: float = 300.0,
+        retry: int = 2,
+        cluster: Optional[str] = None,
+    ) -> None:
+        self.host = host.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.retry = max(0, retry)
+        self.cluster = cluster
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Accept": "application/json"}
+        if self.api_key:
+            headers["X-API-Key"] = self.api_key
+        return headers
+
+    def _url(self, path: str) -> str:
+        if path.startswith("http://") or path.startswith("https://"):
+            return path
+        return f"{self.host}/{path.lstrip('/')}"
+
+    def get_json(
+        self, path: str, params: Optional[dict[str, Any]] = None
+    ) -> Any:
+        """GET ``path`` and return parsed JSON, with retry + typed errors."""
+        merged: dict[str, Any] = {}
+        if self.cluster:
+            merged["cluster"] = self.cluster
+        if params:
+            merged.update({k: v for k, v in params.items() if v is not None})
+
+        url = self._url(path)
+        last_exc: Optional[Exception] = None
+        for attempt in range(self.retry + 1):
+            try:
+                with httpx.Client(timeout=self.timeout) as client:
+                    resp = client.get(url, params=merged, headers=self._headers())
+            except (httpx.TimeoutException, httpx.TransportError) as exc:
+                last_exc = NetworkError(str(exc))
+                if attempt < self.retry:
+                    time.sleep(2**attempt * 0.5)
+                    continue
+                raise last_exc from exc
+
+            if resp.status_code == 401:
+                raise AuthError("authentication failed (HTTP 401)")
+            if resp.status_code >= 400:
+                body = self._safe_body(resp)
+                # Retry on 5xx; fail fast on 4xx.
+                if resp.status_code >= 500 and attempt < self.retry:
+                    last_exc = ApiError(resp.status_code, body)
+                    time.sleep(2**attempt * 0.5)
+                    continue
+                raise ApiError(resp.status_code, body)
+
+            try:
+                return resp.json()
+            except ValueError as exc:
+                raise ApiError(resp.status_code, resp.text) from exc
+
+        if last_exc:
+            raise last_exc
+        raise NetworkError("request failed")  # pragma: no cover
+
+    def download_file(self, url: str, dest_path: str | Path) -> Path:
+        """Stream ``url`` to ``dest_path`` on disk. Returns the path written."""
+        dest = Path(dest_path)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        last_exc: Optional[Exception] = None
+        for attempt in range(self.retry + 1):
+            try:
+                with httpx.Client(timeout=self.timeout, follow_redirects=True) as client:
+                    with client.stream("GET", self._url(url), headers=self._headers()) as resp:
+                        if resp.status_code == 401:
+                            raise AuthError("authentication failed (HTTP 401)")
+                        if resp.status_code >= 400:
+                            body = self._safe_stream_body(resp)
+                            raise ApiError(resp.status_code, body)
+                        with dest.open("wb") as fh:
+                            for chunk in resp.iter_bytes(chunk_size=65536):
+                                fh.write(chunk)
+                return dest
+            except (httpx.TimeoutException, httpx.TransportError) as exc:
+                last_exc = NetworkError(str(exc))
+                if attempt < self.retry:
+                    time.sleep(2**attempt * 0.5)
+                    continue
+                raise last_exc from exc
+        if last_exc:
+            raise last_exc
+        return dest  # pragma: no cover
+
+    @staticmethod
+    def _safe_body(resp: httpx.Response) -> Any:
+        try:
+            return resp.json()
+        except ValueError:
+            return resp.text
+
+    @staticmethod
+    def _safe_stream_body(resp: httpx.Response) -> str:
+        try:
+            resp.read()
+            return resp.text
+        except Exception:  # pragma: no cover
+            return ""

aliyun/bmcli/commands/__init__.py

@@ -0,0 +1 @@
+"""Resource command groups for the bmcli Typer app."""

aliyun/bmcli/commands/_common.py

@@ -0,0 +1,36 @@
+"""Shared option definitions and small helpers for command modules."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from aliyun.bmcli.ids import expand_ids
+
+# Reusable Typer option objects (lists for repeatable / comma-separated flags).
+GroupOption = typer.Option(
+    None, "-g", "--group", help="Group ID (repeatable, comma-separated, or '-' for stdin).",
+    show_default=False,
+)
+JobOption = typer.Option(
+    None, "-j", "--job", help="Job ID (repeatable, comma-separated, or '-' for stdin).",
+    show_default=False,
+)
+OutputDirOption = typer.Option(
+    None, "-o", "--output-dir", help="Override download output root directory.",
+    show_default=False,
+)
+SkipExistingOption = typer.Option(
+    False, "--skip-existing", help="Skip files that already exist on disk."
+)
+
+
+def expand(values: Optional[list[str]]) -> list[str]:
+    return expand_ids(values)
+
+
+def resolve_output_dir(runtime_output: str, override: Optional[str]) -> Path:
+    """Pick the per-command override if given, else the runtime output dir."""
+    return Path(override) if override else Path(runtime_output)

aliyun/bmcli/commands/artifacts.py

@@ -0,0 +1,138 @@
+"""`bmcli artifacts` — download artifacts (Group, Job); download only."""
+
+from __future__ import annotations
+
+import fnmatch
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Optional
+
+import typer
+
+from aliyun.bmcli import output
+from aliyun.bmcli.commands._common import (
+    GroupOption,
+    JobOption,
+    OutputDirOption,
+    SkipExistingOption,
+    expand,
+    resolve_output_dir,
+)
+from aliyun.bmcli.ids import enforce_mutually_exclusive, require_any
+from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
+
+app = typer.Typer(help="Download artifacts (Group, Job).", no_args_is_help=True)
+
+
+def _matches(name: str, include: Optional[str], exclude: Optional[str]) -> bool:
+    if include and not fnmatch.fnmatch(name, include):
+        return False
+    if exclude and fnmatch.fnmatch(name, exclude):
+        return False
+    return True
+
+
+def _list_job(client, job_id: str) -> list[dict[str, Any]]:
+    data = client.get_json(f"/openapi/jobs/{job_id}/artifacts")
+    arts = data.get("artifacts", []) if isinstance(data, dict) else []
+    return [{**a, "job_id": job_id} for a in arts]
+
+
+def _list_group(client, group_id: str) -> list[dict[str, Any]]:
+    data = client.get_json(f"/openapi/groups/{group_id}/artifacts")
+    arts = data.get("artifacts", []) if isinstance(data, dict) else []
+    # tag each entry with its group so downloads can nest under {group_id}/{job_id}/
+    return [{**a, "group_id": group_id} for a in arts]
+
+
+@app.command("download")
+def download(
+    ctx: typer.Context,
+    group: Optional[list[str]] = GroupOption,
+    job: Optional[list[str]] = JobOption,
+    include: Optional[str] = typer.Option(None, "--include", help="Glob to include by name.", show_default=False),
+    exclude: Optional[str] = typer.Option(None, "--exclude", help="Glob to exclude by name.", show_default=False),
+    list_only: bool = typer.Option(False, "--list-only", help="List artifacts without downloading."),
+    output_dir: Optional[str] = OutputDirOption,
+    skip_existing: bool = SkipExistingOption,
+) -> None:
+    """Fetch presigned URLs then concurrently download (or list) artifacts."""
+    rt = get_runtime(ctx)
+    groups = expand(group)
+    jobs = expand(job)
+    enforce_mutually_exclusive(groups, jobs)
+    require_any(groups, jobs)
+    root = resolve_output_dir(rt.output_dir, output_dir)
+
+    # Phase 1: list artifacts per id (concurrent, partial-failure tolerant).
+    if jobs:
+        listing = safe_run_concurrent(
+            jobs, lambda jid: _list_job(rt.client, jid), rt.concurrency
+        )
+    else:
+        listing = safe_run_concurrent(
+            groups, lambda gid: _list_group(rt.client, gid), rt.concurrency
+        )
+
+    # Flatten and filter by name glob.
+    flat: list[dict[str, Any]] = []
+    for arts in listing.items:
+        for a in arts:
+            if _matches(a.get("name") or "", include, exclude):
+                flat.append(a)
+
+    if list_only:
+        # Listing shows every entry, including jobs with no artifact (error_code).
+        output.emit_json(flat, pretty=rt.pretty)
+        report_and_exit(listing, emit=False)
+        return
+
+    # Only entries with a real presigned url are downloadable; the rest (e.g.
+    # error_code=no_artifact) are reported to stderr but are not hard failures.
+    downloadable = [a for a in flat if a.get("url")]
+    for a in flat:
+        if not a.get("url"):
+            output.log(
+                f"no artifact for job={a.get('job_id')} "
+                f"({a.get('error_code', 'no url')})"
+            )
+
+    # Phase 2: concurrent downloads. Job dim -> {job_id}/<name>;
+    # Group dim -> {group_id}/{job_id}/<name> (avoids same-name collisions).
+    failures = list(listing.failures)
+
+    def _dest_for(art: dict[str, Any]) -> Path:
+        name = art.get("name") or "artifact"
+        jid = str(art.get("job_id") or "")
+        if jobs:
+            sub = Path(jid) if jid else Path()
+        else:
+            gid = str(art.get("group_id") or "")
+            sub = Path(gid) / jid if (gid and jid) else Path(gid or jid)
+        return root / sub / name
+
+    def _download_one(art: dict[str, Any]) -> Optional[tuple[str, str]]:
+        name = art.get("name") or "artifact"
+        url = art.get("url")
+        if not url:  # already reported above; defensive guard
+            return None
+        dest = _dest_for(art)
+        if skip_existing and dest.exists():
+            output.log(f"skip existing {dest}")
+            return None
+        try:
+            rt.client.download_file(url, dest)
+            output.log(f"downloaded {dest}")
+            return None
+        except Exception as exc:  # noqa: BLE001 - per-artifact failure tolerated
+            return (f"{art.get('job_id')}:{name}", str(exc))
+
+    if downloadable:
+        max_workers = max(1, min(rt.concurrency, len(downloadable)))
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            for result in pool.map(_download_one, downloadable):
+                if result is not None:
+                    failures.append(result)
+
+    listing.failures = failures
+    report_and_exit(listing, emit=False)

aliyun/bmcli/commands/jobs.py

@@ -0,0 +1,131 @@
+"""`bmcli jobs` — Group job list (/openapi/jobs) with pagination."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import typer
+
+from aliyun.bmcli import output
+from aliyun.bmcli.commands._common import (
+    GroupOption,
+    JobOption,
+    OutputDirOption,
+    SkipExistingOption,
+    expand,
+    resolve_output_dir,
+)
+from aliyun.bmcli.ids import (
+    enforce_mutually_exclusive,
+    reject_unsupported_dimension,
+    require_any,
+)
+from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
+
+app = typer.Typer(help="Group job list (/openapi/jobs).", no_args_is_help=True)
+
+
+def _guard(groups: list[str], jobs: list[str]) -> None:
+    enforce_mutually_exclusive(groups, jobs)
+    reject_unsupported_dimension(
+        "jobs", groups=groups, jobs=jobs, allow_group=True, allow_job=False
+    )
+    require_any(groups, jobs)
+
+
+def _params(group_id, skip, limit, template, status, instance_id) -> dict[str, Any]:
+    params: dict[str, Any] = {"group_id": group_id, "skip": skip, "limit": limit}
+    if template:
+        params["template"] = template
+    if status:
+        params["status"] = status
+    if instance_id:
+        params["instance_id"] = instance_id
+    return params
+
+
+def _fetch_all(client, group_id, page_size, template, status, instance_id):
+    skip = 0
+    all_jobs: list[Any] = []
+    total = None
+    while True:
+        data = client.get_json(
+            "/openapi/jobs",
+            params=_params(group_id, skip, page_size, template, status, instance_id),
+        )
+        jobs = data.get("jobs", []) if isinstance(data, dict) else []
+        if isinstance(data, dict) and total is None:
+            total = data.get("total")
+        all_jobs.extend(jobs)
+        if not jobs or len(jobs) < page_size:
+            break
+        skip += page_size
+        if total is not None and len(all_jobs) >= total:
+            break
+    return {"group_id": group_id, "total": total if total is not None else len(all_jobs),
+            "jobs": all_jobs}
+
+
+@app.command("get")
+def get(
+    ctx: typer.Context,
+    group: Optional[list[str]] = GroupOption,
+    job: Optional[list[str]] = JobOption,
+    page: int = typer.Option(1, "--page", help="Page number (1-based)."),
+    page_size: int = typer.Option(50, "--page-size", help="Page size."),
+    all_pages: bool = typer.Option(False, "--all", help="Auto-paginate all jobs."),
+    template: Optional[str] = typer.Option(None, "--template", show_default=False),
+    status: Optional[str] = typer.Option(None, "--status", show_default=False),
+    instance_id: Optional[str] = typer.Option(None, "--instance-id", show_default=False),
+) -> None:
+    """Query the group job list; prints a JSON array."""
+    rt = get_runtime(ctx)
+    groups = expand(group)
+    jobs = expand(job)
+    _guard(groups, jobs)
+
+    def worker(group_id: str):
+        if all_pages:
+            return _fetch_all(rt.client, group_id, page_size, template, status, instance_id)
+        skip = max(0, (page - 1) * page_size)
+        data = rt.client.get_json(
+            "/openapi/jobs",
+            params=_params(group_id, skip, page_size, template, status, instance_id),
+        )
+        return {"group_id": group_id, **(data if isinstance(data, dict) else {"data": data})}
+
+    res = safe_run_concurrent(groups, worker, rt.concurrency)
+    report_and_exit(res, emit=True, pretty=rt.pretty)
+
+
+@app.command("download")
+def download(
+    ctx: typer.Context,
+    group: Optional[list[str]] = GroupOption,
+    job: Optional[list[str]] = JobOption,
+    page_size: int = typer.Option(50, "--page-size", help="Page size for pagination."),
+    template: Optional[str] = typer.Option(None, "--template", show_default=False),
+    status: Optional[str] = typer.Option(None, "--status", show_default=False),
+    instance_id: Optional[str] = typer.Option(None, "--instance-id", show_default=False),
+    output_dir: Optional[str] = OutputDirOption,
+    skip_existing: bool = SkipExistingOption,
+) -> None:
+    """Download the full job list to {group_id}/job_list.json."""
+    rt = get_runtime(ctx)
+    groups = expand(group)
+    jobs = expand(job)
+    _guard(groups, jobs)
+    root = resolve_output_dir(rt.output_dir, output_dir)
+
+    def worker(group_id: str):
+        dest = root / group_id / "job_list.json"
+        if skip_existing and dest.exists():
+            output.log(f"skip existing {dest}")
+            return str(dest)
+        data = _fetch_all(rt.client, group_id, page_size, template, status, instance_id)
+        output.write_json_file(dest, data, pretty=rt.pretty)
+        output.log(f"wrote {dest} ({len(data.get('jobs', []))} jobs)")
+        return str(dest)
+
+    res = safe_run_concurrent(groups, worker, rt.concurrency)
+    report_and_exit(res, emit=False)