benchloop-client 0.1.6__py3-none-any.whl

aliyun/bmcli/config.py

@@ -0,0 +1,120 @@
+"""Configuration loading with precedence: CLI flag > env > config file > default."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+try:  # Python 3.11+
+    import tomllib  # type: ignore
+except ModuleNotFoundError:  # Python 3.10: use the tomli backport
+    try:
+        import tomli as tomllib  # type: ignore
+    except ModuleNotFoundError:  # pragma: no cover - backport not installed
+        tomllib = None  # type: ignore
+
+
+DEFAULT_HOST = "https://benchmark-analysis.alibaba-inc.com"
+DEFAULT_CONFIG_PATH = Path.home() / ".benchloop" / "config.toml"
+
+
+@dataclass
+class Config:
+    """Resolved runtime configuration."""
+
+    api_key: Optional[str] = None
+    host: str = DEFAULT_HOST
+    output_dir: str = "."
+
+
+def _read_config_file(path: Path) -> dict[str, Any]:
+    """Parse the [profile] table from a TOML config file.
+
+    Returns an empty dict if the file is missing, unreadable, or tomllib is
+    unavailable (Python 3.10 without a backport).
+    """
+    if tomllib is None:
+        return {}
+    if not path.exists():
+        return {}
+    try:
+        with path.open("rb") as fh:
+            data = tomllib.load(fh)
+    except (OSError, ValueError):
+        return {}
+    profile = data.get("profile")
+    if isinstance(profile, dict):
+        return profile
+    return {}
+
+
+def load_config(
+    api_key: Optional[str] = None,
+    host: Optional[str] = None,
+    output_dir: Optional[str] = None,
+    config_path: Optional[str] = None,
+) -> Config:
+    """Resolve configuration applying precedence CLI > env > file > default.
+
+    Env vars: BENCHLOOP_API_KEY, BENCHLOOP_HOST, BENCHLOOP_OUTPUT_DIR.
+    BENCHMARK_API_KEY is also accepted as a fallback for the api key.
+    """
+    path = Path(config_path).expanduser() if config_path else DEFAULT_CONFIG_PATH
+    file_cfg = _read_config_file(path)
+
+    resolved_key = (
+        api_key
+        or os.environ.get("BENCHLOOP_API_KEY")
+        or os.environ.get("BENCHMARK_API_KEY")
+        or file_cfg.get("api_key")
+    )
+    resolved_host = (
+        host
+        or os.environ.get("BENCHLOOP_HOST")
+        or file_cfg.get("host")
+        or DEFAULT_HOST
+    )
+    resolved_out = (
+        output_dir
+        or os.environ.get("BENCHLOOP_OUTPUT_DIR")
+        or file_cfg.get("output_dir")
+        or "."
+    )
+
+    return Config(
+        api_key=resolved_key,
+        host=str(resolved_host).rstrip("/"),
+        output_dir=str(resolved_out),
+    )
+
+
+def write_config(
+    api_key: Optional[str] = None,
+    host: Optional[str] = None,
+    output_dir: Optional[str] = None,
+    config_path: Optional[str] = None,
+) -> Path:
+    """Persist provided values to ~/.benchloop/config.toml (merging existing).
+
+    Returns the path written. Only non-None values are updated.
+    """
+    path = Path(config_path).expanduser() if config_path else DEFAULT_CONFIG_PATH
+    existing = _read_config_file(path)
+
+    if api_key is not None:
+        existing["api_key"] = api_key
+    if host is not None:
+        existing["host"] = host
+    if output_dir is not None:
+        existing["output_dir"] = output_dir
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = ["[profile]"]
+    for key in ("host", "api_key", "output_dir"):
+        if key in existing and existing[key] is not None:
+            value = str(existing[key]).replace("\\", "\\\\").replace('"', '\\"')
+            lines.append(f'{key} = "{value}"')
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return path

aliyun/bmcli/exit_codes.py

@@ -0,0 +1,15 @@
+"""Process exit codes used across the CLI.
+
+See design doc section 8:
+- 0 all success
+- 1 partial failure (some ids failed)
+- 2 usage / parameter error
+- 3 auth failure (HTTP 401)
+- 4 network / timeout
+"""
+
+OK = 0
+PARTIAL = 1
+USAGE = 2
+AUTH = 3
+NETWORK = 4

aliyun/bmcli/ids.py

@@ -0,0 +1,87 @@
+"""ID input expansion and dimension validation helpers."""
+
+from __future__ import annotations
+
+import sys
+from typing import Callable, Iterable, Optional
+
+import typer
+
+
+def _read_stdin() -> list[str]:
+    data = sys.stdin.read()
+    return data.split()
+
+
+def expand_ids(
+    values: Optional[Iterable[str]],
+    stdin_reader: Callable[[], list[str]] = _read_stdin,
+) -> list[str]:
+    """Expand a list of raw flag values into a deduped ordered id list.
+
+    Supports:
+    - repeated flags: ``-g G1 -g G2``
+    - comma separation: ``-g G1,G2``
+    - ``-`` meaning read from stdin (whitespace / newline separated)
+
+    Duplicates are removed while preserving first-seen order.
+    """
+    out: list[str] = []
+    seen: set[str] = set()
+
+    def _add(token: str) -> None:
+        token = token.strip()
+        if not token:
+            return
+        if token not in seen:
+            seen.add(token)
+            out.append(token)
+
+    for raw in values or []:
+        if raw is None:
+            continue
+        if raw == "-":
+            for tok in stdin_reader():
+                _add(tok)
+            continue
+        for part in raw.split(","):
+            _add(part)
+
+    return out
+
+
+def enforce_mutually_exclusive(
+    groups: list[str], jobs: list[str]
+) -> None:
+    """Raise a usage error if both -g and -j were supplied."""
+    if groups and jobs:
+        raise typer.BadParameter("-g/--group and -j/--job are mutually exclusive")
+
+
+def reject_unsupported_dimension(
+    resource: str,
+    *,
+    groups: list[str],
+    jobs: list[str],
+    allow_group: bool,
+    allow_job: bool,
+) -> None:
+    """Reject ids for a dimension the resource does not support.
+
+    Raises ``typer.BadParameter`` (mapped to usage exit code 2) when an
+    unsupported dimension is provided (e.g. ``report -g``, ``overview job -g``).
+    """
+    if groups and not allow_group:
+        raise typer.BadParameter(
+            f"{resource} does not support -g/--group (group dimension)"
+        )
+    if jobs and not allow_job:
+        raise typer.BadParameter(
+            f"{resource} does not support -j/--job (job dimension)"
+        )
+
+
+def require_any(groups: list[str], jobs: list[str]) -> None:
+    """Raise a usage error if neither -g nor -j was supplied."""
+    if not groups and not jobs:
+        raise typer.BadParameter("at least one -g/--group or -j/--job is required")

aliyun/bmcli/output.py

@@ -0,0 +1,80 @@
+"""Output helpers: JSON to stdout, files to disk, logs/progress to stderr."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+# Module level verbosity flags, set by the CLI callback.
+_QUIET = False
+_VERBOSE = False
+
+
+def set_verbosity(quiet: bool = False, verbose: bool = False) -> None:
+    global _QUIET, _VERBOSE
+    _QUIET = quiet
+    _VERBOSE = verbose
+
+
+def _dumps(obj: Any, pretty: bool) -> str:
+    if pretty:
+        return json.dumps(obj, ensure_ascii=False, indent=2)
+    return json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
+
+
+def emit_json(data: Any, pretty: bool = True) -> None:
+    """Print JSON data to stdout (data channel, pipeable to jq)."""
+    sys.stdout.write(_dumps(data, pretty) + "\n")
+    sys.stdout.flush()
+
+
+def write_json_file(path: str | Path, obj: Any, pretty: bool = True) -> Path:
+    """Write a JSON object to ``path``, creating parent dirs. Returns path."""
+    p = Path(path)
+    ensure_dir(p.parent)
+    p.write_text(_dumps(obj, pretty) + "\n", encoding="utf-8")
+    return p
+
+
+def write_text_file(path: str | Path, text: str) -> Path:
+    """Write raw text (e.g. markdown report) to ``path``. Returns path."""
+    p = Path(path)
+    ensure_dir(p.parent)
+    p.write_text(text, encoding="utf-8")
+    return p
+
+
+def ensure_dir(path: str | Path) -> Path:
+    """Create a directory (and parents) if needed. Returns the path."""
+    p = Path(path) if str(path) else Path(".")
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def log(message: str) -> None:
+    """Informational log to stderr (suppressed by --quiet)."""
+    if not _QUIET:
+        sys.stderr.write(message + "\n")
+        sys.stderr.flush()
+
+
+def debug(message: str) -> None:
+    """Debug log to stderr (only with --verbose)."""
+    if _VERBOSE and not _QUIET:
+        sys.stderr.write(f"[debug] {message}\n")
+        sys.stderr.flush()
+
+
+def warn(message: str) -> None:
+    """Warning to stderr (suppressed by --quiet)."""
+    if not _QUIET:
+        sys.stderr.write(f"[warn] {message}\n")
+        sys.stderr.flush()
+
+
+def error(message: str) -> None:
+    """Error to stderr (always shown)."""
+    sys.stderr.write(f"[error] {message}\n")
+    sys.stderr.flush()

aliyun/bmcli/runtime.py

@@ -0,0 +1,147 @@
+"""Shared runtime context and concurrent fetch / partial-failure helpers."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+import typer
+
+from aliyun.bmcli import output
+from aliyun.bmcli.client import ApiClient, AuthError, NetworkError
+from aliyun.bmcli.config import Config
+
+
+@dataclass
+class Runtime:
+    """Resolved configuration plus per-invocation runtime options."""
+
+    config: Config
+    concurrency: int = 5
+    timeout: float = 300.0
+    retry: int = 2
+    cluster: str | None = None
+    pretty: bool = True
+    output_dir: str = "."
+    verbose: bool = False
+    quiet: bool = False
+    _client: ApiClient | None = field(default=None, repr=False)
+
+    @property
+    def client(self) -> ApiClient:
+        if self._client is None:
+            self._client = ApiClient(
+                host=self.config.host,
+                api_key=self.config.api_key,
+                timeout=self.timeout,
+                retry=self.retry,
+                cluster=self.cluster,
+            )
+        return self._client
+
+
+def get_runtime(ctx: typer.Context) -> Runtime:
+    """Fetch the Runtime stashed on the Typer context by the root callback."""
+    rt = ctx.obj
+    if not isinstance(rt, Runtime):  # pragma: no cover - defensive
+        raise typer.BadParameter("CLI not initialized")
+    return rt
+
+
+@dataclass
+class FetchResult:
+    """Outcome of a concurrent multi-id fetch."""
+
+    items: list[Any]
+    failures: list[tuple[str, str]]  # (id, error message)
+
+
+def run_concurrent(
+    ids: list[str],
+    worker: Callable[[str], Any],
+    concurrency: int,
+) -> FetchResult:
+    """Run ``worker`` over ``ids`` concurrently, preserving input order.
+
+    A single id failing does not abort the others; its error is collected.
+    Auth errors abort early since they affect every request.
+    """
+    items: list[Any] = [None] * len(ids)
+    failures: list[tuple[str, str]] = []
+
+    if not ids:
+        return FetchResult(items=[], failures=[])
+
+    max_workers = max(1, min(concurrency, len(ids)))
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        future_map = {pool.submit(worker, _id): idx for idx, _id in enumerate(ids)}
+        for future in future_map:
+            idx = future_map[future]
+            _id = ids[idx]
+            try:
+                items[idx] = future.result()
+            except AuthError:
+                # Auth failure is fatal for the whole invocation.
+                raise
+            except NetworkError as exc:
+                failures.append((_id, f"network: {exc}"))
+            except Exception as exc:  # noqa: BLE001 - collect any per-id failure
+                failures.append((_id, str(exc)))
+
+    ordered = [it for it in items if it is not None]
+    return FetchResult(items=ordered, failures=failures)
+
+
+def report_and_exit(
+    result: FetchResult,
+    *,
+    emit: bool,
+    data: Any | None = None,
+    pretty: bool = True,
+) -> None:
+    """Emit data (optional), print failure summary to stderr, and exit.
+
+    Exit codes: 0 all success, 1 partial failure. Raises typer.Exit.
+    """
+    if emit:
+        output.emit_json(data if data is not None else result.items, pretty=pretty)
+
+    if result.failures:
+        output.error(f"{len(result.failures)} id(s) failed:")
+        for _id, msg in result.failures:
+            output.error(f"  - {_id}: {msg}")
+        from aliyun.bmcli.exit_codes import NETWORK, PARTIAL
+
+        # When nothing succeeded and every failure was a network/timeout error,
+        # surface the dedicated network exit code (4) rather than partial (1).
+        all_network = all(msg.startswith("network:") for _id, msg in result.failures)
+        if not result.items and all_network:
+            raise typer.Exit(code=NETWORK)
+        raise typer.Exit(code=PARTIAL)
+
+    from aliyun.bmcli.exit_codes import OK
+
+    raise typer.Exit(code=OK)
+
+
+def safe_run_concurrent(
+    ids: list[str],
+    worker: Callable[[str], Any],
+    concurrency: int,
+) -> "FetchResult":
+    """Like run_concurrent but maps AuthError→exit 3 and NetworkError→exit 4.
+
+    Call this instead of run_concurrent in command handlers so that fatal
+    errors produce the documented exit codes rather than an unhandled exception.
+    """
+    from aliyun.bmcli.exit_codes import AUTH, NETWORK
+
+    try:
+        return run_concurrent(ids, worker, concurrency)
+    except AuthError as exc:
+        output.error(f"authentication error: {exc}")
+        raise typer.Exit(code=AUTH) from exc
+    except NetworkError as exc:
+        output.error(f"network error: {exc}")
+        raise typer.Exit(code=NETWORK) from exc

benchloop_client-0.1.6.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: benchloop_client
+Version: 0.1.6
+Summary: bmcli — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts).
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.25.2
+Requires-Dist: tomli>=1.1.0; python_version < '3.11'
+Requires-Dist: typer>=0.9.4
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# benchloop-cli (`bmcli`)
+
+Command line tool for fetching and downloading benchmark analysis data
+(run metrics, trajectory, pre-analysis reports, task/job lists, and artifacts)
+by `group_id` / `job_id`.
+
+## Install package
+
+```
+pip install -i http://yum.tbsite.net/aliyun-pypi/simple/ --trusted-host=yum.tbsite.net bmcli
+
+bmcli --api-key $BENCH_LOOP_API_KEY overview get -j $JOB_ID
+bmcli --api-key $BENCH_LOOP_API_KEY metrics get -j $JOB_ID
+
+# Option 2: config ~/.benchloop/config.toml by cli, and run bmcli without --api-key option:
+bmcli config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
+
+bmcli overview get -j $JOB_ID
+bmcli metrics get -j $JOB_ID
+
+```
+
+## Install and run from source
+
+```bash
+# with uv (recommended)
+uv sync
+uv run bmcli --help
+
+# or editable install with pip
+python -m venv .venv && source .venv/bin/activate
+pip install -e .
+bmcli --help
+```
+
+## Auth & config
+
+Authentication uses a personal api key sent as the `X-API-Key` header.
+
+Precedence: CLI flag > env var > `~/.benchloop/config.toml` > default.
+
+- Env: `BENCHLOOP_API_KEY` (or `BENCHMARK_API_KEY`), `BENCHLOOP_HOST`, `BENCHLOOP_OUTPUT_DIR`
+- Default host: `https://benchmark-analysis.alibaba-inc.com`
+
+```bash
+bmcli config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
+```
+
+## Usage
+
+Two-segment `bmcli <resource> <action>` form. `get` prints a JSON array to
+stdout (pipeable to `jq`); `download` writes the standard file layout.
+
+```bash
+bmcli metrics get -j J123 | jq '.[0]'
+bmcli metrics download -g G1 --scope all -o ./out
+bmcli trajectory download -g G1,G2 -o ./out
+bmcli report download -j J123 --agent all
+bmcli jobs get -g G1 --all          # get supports --all (auto-paginate)
+bmcli jobs download -g G1           # download is always full
+bmcli artifacts download -g G1 -o ./out
+bmcli pull -g G1 --with metrics,trajectory,jobs,artifacts -o ./out
+```
+
+IDs accept repeated flags, comma-separated values, and `-` to read from stdin.
+`-g` and `-j` are mutually exclusive in a single command.
+
+## Exit codes
+
+| code | meaning |
+| ---- | ------- |
+| 0 | all success |
+| 1 | partial failure (some ids failed) |
+| 2 | usage / parameter error |
+| 3 | auth failure (HTTP 401) |
+| 4 | network / timeout |
+
+stdout carries data only; logs and progress go to stderr. `--quiet` suppresses
+non-error logs, `--verbose` adds debug logs.
+
+
+## Release new package
+
+```
+uv build
+twine check dist/bmcli-0.1.6-py3-none-any.whl
+twine upload -r aliyun-pypi dist/bmcli-0.1.6-py3-none-any.whl --verbose
+```

benchloop_client-0.1.6.dist-info/RECORD

@@ -0,0 +1,23 @@
+aliyun/bmcli/__init__.py,sha256=TkYkNbqespvPiIfcCLuf9m-wQagsVQUM47jjqm0pzqk,99
+aliyun/bmcli/__main__.py,sha256=qQPm9IIfmqXkVmk8zfSG1zkB84dDFFfEcWXWXTImzL4,601
+aliyun/bmcli/cli.py,sha256=WvHmoSaNCNYuO9xleZfjZ8Nn-hB_Qww14MWMklemAZY,4472
+aliyun/bmcli/client.py,sha256=h3pUwmQ0_WG4pih9q3gB5A07cjM0IJ3ErkpNS0rWSx0,5077
+aliyun/bmcli/config.py,sha256=VqRJf0dXINskQTPjubjoWjLZUsffNfOOh-aaPPlzMt4,3545
+aliyun/bmcli/exit_codes.py,sha256=EYGCWM4asW1-rgCt-oO5Gl2a7-VeXmE7gKb0kwLLWoo,257
+aliyun/bmcli/ids.py,sha256=oEj-De0GjPMY9WwaPBNq_p4exJ3B3baOem2IyQCftj8,2369
+aliyun/bmcli/output.py,sha256=xsRIh4rhje6U2gWInglLkCm1SmmmM4viSAYIRVvv4Do,2242
+aliyun/bmcli/runtime.py,sha256=wQK0lOCV5IBndpAQj9B-eks0QucQrBlysiIXrQOKyjI,4688
+aliyun/bmcli/commands/__init__.py,sha256=xK8Hjh-Ic9rab3UI2RQb-lgraIUAEgxOrlGpZsgHlmQ,55
+aliyun/bmcli/commands/_common.py,sha256=a1YcsPLRB-YbmwEn15uUytFugwIonX3cFkYQMjx1kzE,1146
+aliyun/bmcli/commands/artifacts.py,sha256=z_DyEZfHz9pCQSNQwh3fN7_Gb6ca8qoJFpc3N2clJJg,5169
+aliyun/bmcli/commands/jobs.py,sha256=Dl1bZ8MvvoSTYMTul7Ln1cuV759QMwQP_uSxs8T82vA,4751
+aliyun/bmcli/commands/metrics.py,sha256=vSHJcZMg5pWObpM05GHMNcaaQNedu1Aq6cEYzCKzH-Y,4685
+aliyun/bmcli/commands/overview.py,sha256=jjPsLhxIBncYB2xDWRAB6PuQWEFgfpKlYVrLnCr1rS4,2317
+aliyun/bmcli/commands/pull.py,sha256=cdURofZZB4fGHniAfStMkt02rgheTr74DJwFYfr1VBg,6806
+aliyun/bmcli/commands/report.py,sha256=1pdM1oz5o0KSgJ0Z2lFhs7ltY5Sd42rDiznHOVDY2a0,3155
+aliyun/bmcli/commands/tasks.py,sha256=6R831JssRr-NUvv2ftrQQC0DvNm6dt6N0RVMIBYL-dc,4354
+aliyun/bmcli/commands/trajectory.py,sha256=BViov85xYuJ386kLo6xWHsRt_qvJ7nrt2pzMXFiOgqo,2886
+benchloop_client-0.1.6.dist-info/METADATA,sha256=B_QG4kOUeQg7OAjrYN3orHuGVUF51OwzDyu3e4CGu0U,2953
+benchloop_client-0.1.6.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+benchloop_client-0.1.6.dist-info/entry_points.txt,sha256=Nunw6TW3VbmDcuN1pCcDAx7vGmU6G-F7SW87nvI18as,53
+benchloop_client-0.1.6.dist-info/RECORD,,

benchloop_client-0.1.6.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

benchloop_client-0.1.6.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bmcli = aliyun.bmcli.__main__:main