benchloop-client 0.2.21__tar.gz → 0.2.22__tar.gz

{benchloop_client-0.2.21 → benchloop_client-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchloop_client
-Version: 0.2.21
+Version: 0.2.22
 Summary: Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts).
 Requires-Python: >=3.10
 Requires-Dist: httpx>=0.25.2

{benchloop_client-0.2.21 → benchloop_client-0.2.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "benchloop_client"
-version = "0.2.21"
+version = "0.2.22"
 description = "Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts)."
 readme = "README.md"
 requires-python = ">=3.10"

{benchloop_client-0.2.21 → benchloop_client-0.2.22}/src/aliyun/bmcli/cli.py

@@ -11,6 +11,7 @@ from aliyun.bmcli.commands import (
     ap_cmd as ap_cmd_mod,
     artifacts as artifacts_cmd,
     jobs as jobs_cmd,
+    metadata as metadata_cmd,
     metrics as metrics_cmd,
     overview as overview_cmd,
     pull as pull_cmd,
@@ -135,6 +136,7 @@ app.add_typer(overview_cmd.app, name="overview")
 app.add_typer(tasks_cmd.app, name="tasks")
 app.add_typer(jobs_cmd.app, name="jobs")
 app.add_typer(artifacts_cmd.app, name="artifacts")
+app.add_typer(metadata_cmd.app, name="metadata")
 app.command(
     "ap-cmd",
     context_settings={"allow_extra_args": True, "ignore_unknown_options": True},

benchloop_client-0.2.22/src/aliyun/bmcli/commands/metadata.py

@@ -0,0 +1,175 @@
+"""`bmcli metadata` — job metadata for a CLI download task (task-id only)."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Optional
+from urllib.parse import quote
+
+import typer
+
+from aliyun.bmcli import output
+from aliyun.bmcli.commands._common import (
+    OutputDirOption,
+    SkipExistingOption,
+    resolve_output_dir,
+    resolve_task_id,
+    task_job_ids,
+)
+from aliyun.bmcli.exit_codes import OK, PARTIAL
+from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
+
+app = typer.Typer(help="Job metadata (download task).", no_args_is_help=True)
+
+
+def _write_metadata_archive(
+    archive_path: Path,
+    task_payload: dict,
+    task_id: str,
+    downloaded: list[tuple[str, Path]],
+    failures: list[tuple[str, str]],
+    *,
+    pretty: bool,
+) -> Path:
+    """Create the metadata ZIP atomically: metadata/{job_id}.json + manifest.json."""
+    manifest = {
+        "task_id": task_id,
+        "group_id": task_payload.get("group_id"),
+        "created_at": task_payload.get("created_at"),
+        "total_jobs": len(downloaded) + len(failures),
+        "successful_jobs": len(downloaded),
+        "failed_jobs": len(failures),
+        "success_job_ids": [job_id for job_id, _path in downloaded],
+        "failures": [{"job_id": job_id, "error": error} for job_id, error in failures],
+    }
+    indent = 2 if pretty else None
+    separators = None if pretty else (",", ":")
+    manifest_text = (
+        json.dumps(
+            manifest,
+            ensure_ascii=False,
+            indent=indent,
+            separators=separators,
+        )
+        + "\n"
+    )
+
+    # 与轨迹任务包同一套原子写法：临时 zip 落在目标目录内，成功后 replace。
+    with tempfile.NamedTemporaryFile(
+        prefix=f".{archive_path.name}.",
+        suffix=".tmp",
+        dir=archive_path.parent,
+        delete=False,
+    ) as tmp_archive:
+        tmp_archive_path = Path(tmp_archive.name)
+    try:
+        with zipfile.ZipFile(
+            tmp_archive_path,
+            mode="w",
+            compression=zipfile.ZIP_DEFLATED,
+            allowZip64=True,
+        ) as archive:
+            for job_id, path in downloaded:
+                file_job_id = quote(job_id, safe="-_.")
+                archive.write(path, arcname=f"metadata/{file_job_id}.json")
+            archive.writestr("manifest.json", manifest_text)
+        tmp_archive_path.replace(archive_path)
+    except Exception:
+        tmp_archive_path.unlink(missing_ok=True)
+        raise
+    return archive_path
+
+
+@app.command("download")
+def download(
+    ctx: typer.Context,
+    task_id: str = typer.Option(
+        ...,
+        "--task-id",
+        help="Download task ID created by the benchmark analysis page.",
+    ),
+    output_dir: Optional[str] = OutputDirOption,
+    skip_existing: bool = SkipExistingOption,
+) -> None:
+    """Download per-job metadata for a task into {task_id}_metadata.zip."""
+    rt = get_runtime(ctx)
+    # 复用统一校验：task_id 不得包含路径分隔符，保证归档留在输出目录内。
+    task_id = resolve_task_id(task_id, [], [])
+    root = resolve_output_dir(rt.output_dir, output_dir)
+
+    archive_path = root / f"{task_id}_metadata.zip"
+    if skip_existing and archive_path.exists():
+        output.log(f"skip existing {archive_path}")
+        raise typer.Exit(code=OK)
+
+    task_result = safe_run_concurrent(
+        [task_id],
+        lambda tid: rt.client.get_json(
+            f"/openapi/trajectory-download-tasks/{quote(tid, safe='')}"
+        ),
+        1,
+    )
+    if task_result.failures:
+        report_and_exit(task_result, emit=False)
+    if not task_result.items:
+        output.error("download task response is empty")
+        raise typer.Exit(code=PARTIAL)
+
+    task_payload = task_result.items[0]
+    try:
+        task_jobs = task_job_ids(task_payload)
+    except ValueError as exc:
+        output.error(str(exc))
+        raise typer.Exit(code=PARTIAL) from exc
+
+    output.ensure_dir(root)
+    progress = output.ProgressCounter(
+        "task metadata downloads",
+        len(task_jobs),
+        "jobs",
+        action="processed",
+        show_eta=True,
+    )
+    progress.start()
+
+    with tempfile.TemporaryDirectory(
+        prefix=f".{task_id}-metadata-", dir=root
+    ) as temp_dir:
+        temp_root = Path(temp_dir)
+
+        def task_job_worker(job_id: str) -> tuple[str, Path]:
+            try:
+                meta = rt.client.get_json(
+                    f"/openapi/jobs/{quote(job_id, safe='')}/meta"
+                )
+                # job_id comes from the server-side task snapshot. Encode it
+                # before using it as a local filename so an unexpected slash
+                # or traversal segment can never escape the temporary folder.
+                file_job_id = quote(job_id, safe="-_.")
+                dest = temp_root / f"{file_job_id}.json"
+                output.write_json_file(dest, meta, pretty=rt.pretty)
+                return job_id, dest
+            finally:
+                # A failed request is still a completed attempt and must move
+                # the visible task progress forward.
+                progress.advance(job_id)
+
+        result = safe_run_concurrent(task_jobs, task_job_worker, rt.concurrency)
+        if not result.items:
+            report_and_exit(result, emit=False)
+
+        output.progress(f"creating archive {archive_path}")
+        _write_metadata_archive(
+            archive_path,
+            task_payload,
+            task_id,
+            result.items,
+            result.failures,
+            pretty=rt.pretty,
+        )
+
+    output.log(f"wrote {archive_path}")
+    report_and_exit(result, emit=False)

benchloop_client-0.2.22/tests/test_cmd_metadata.py

@@ -0,0 +1,231 @@
+"""Tests for `bmcli metadata download` (task-id only)."""
+
+from __future__ import annotations
+
+import json
+import zipfile
+
+from aliyun.bmcli.cli import app
+from tests.conftest import invoke
+
+
+class TestMetadataDownloadTask:
+    @staticmethod
+    def task_payload(job_ids):
+        return {
+            "task_id": "T1",
+            "group_id": "G1",
+            "job_count": len(job_ids),
+            "job_ids": job_ids,
+            "created_at": "2026-07-14T10:00:00",
+        }
+
+    def test_download_creates_zip_with_metadata_and_manifest(
+        self, runner, mock_client, tmp_path
+    ):
+        def response(path):
+            if path == "/openapi/trajectory-download-tasks/T1":
+                return self.task_payload(["J1", "J2"])
+            job_id = path.split("/")[-2]
+            return {"job_id": job_id, "model": "qwen-max", "tags": ["t1"]}
+
+        mock_client.get_json.side_effect = response
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        archive_path = tmp_path / "T1_metadata.zip"
+        assert result.exit_code == 0
+        assert archive_path.exists()
+        assert f"wrote {archive_path}" in result.stderr
+        called_paths = [call.args[0] for call in mock_client.get_json.call_args_list]
+        assert called_paths.count("/openapi/jobs/J1/meta") == 1
+        assert called_paths.count("/openapi/jobs/J2/meta") == 1
+        with zipfile.ZipFile(archive_path) as archive:
+            assert set(archive.namelist()) == {
+                "metadata/J1.json",
+                "metadata/J2.json",
+                "manifest.json",
+            }
+            assert json.loads(archive.read("metadata/J1.json")) == {
+                "job_id": "J1",
+                "model": "qwen-max",
+                "tags": ["t1"],
+            }
+            manifest = json.loads(archive.read("manifest.json"))
+        assert manifest["task_id"] == "T1"
+        assert manifest["group_id"] == "G1"
+        assert manifest["total_jobs"] == 2
+        assert manifest["successful_jobs"] == 2
+        assert manifest["failed_jobs"] == 0
+        assert manifest["success_job_ids"] == ["J1", "J2"]
+        assert manifest["failures"] == []
+        assert not list(tmp_path.glob(".T1-metadata-*"))
+
+    def test_partial_failure_still_archives_success_and_exits_partial(
+        self, runner, mock_client, tmp_path
+    ):
+        def response(path):
+            if path == "/openapi/trajectory-download-tasks/T1":
+                return self.task_payload(["J1", "J2"])
+            if path == "/openapi/jobs/J2/meta":
+                raise RuntimeError("meta unavailable")
+            return {"job_id": "J1"}
+
+        mock_client.get_json.side_effect = response
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        archive_path = tmp_path / "T1_metadata.zip"
+        assert result.exit_code == 1
+        assert archive_path.exists()
+        with zipfile.ZipFile(archive_path) as archive:
+            assert set(archive.namelist()) == {
+                "metadata/J1.json",
+                "manifest.json",
+            }
+            manifest = json.loads(archive.read("manifest.json"))
+        assert manifest["failed_jobs"] == 1
+        assert manifest["failures"] == [
+            {"job_id": "J2", "error": "meta unavailable"}
+        ]
+
+    def test_all_failures_do_not_create_zip(self, runner, mock_client, tmp_path):
+        def response(path):
+            if path == "/openapi/trajectory-download-tasks/T1":
+                return self.task_payload(["J1", "J2"])
+            raise RuntimeError("meta unavailable")
+
+        mock_client.get_json.side_effect = response
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        assert result.exit_code == 1
+        assert not (tmp_path / "T1_metadata.zip").exists()
+        assert "2 id(s) failed" in result.stderr
+        assert not list(tmp_path.glob(".T1-metadata-*"))
+
+    def test_skip_existing_makes_no_api_calls(self, runner, mock_client, tmp_path):
+        existing = tmp_path / "T1_metadata.zip"
+        existing.write_bytes(b"zip")
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "--skip-existing",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        assert result.exit_code == 0
+        assert f"skip existing {existing}" in result.stderr
+        mock_client.get_json.assert_not_called()
+
+    def test_encodes_traversal_job_id_inside_archive(
+        self, runner, mock_client, tmp_path
+    ):
+        def response(path):
+            if path == "/openapi/trajectory-download-tasks/T1":
+                return self.task_payload(["../J1"])
+            return {"job_id": "../J1"}
+
+        mock_client.get_json.side_effect = response
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        assert result.exit_code == 0
+        with zipfile.ZipFile(tmp_path / "T1_metadata.zip") as archive:
+            assert "metadata/..%2FJ1.json" in archive.namelist()
+        assert not (tmp_path.parent / "J1.json").exists()
+
+    def test_task_id_with_path_separator_rejected(
+        self, runner, mock_client, tmp_path
+    ):
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "../T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        assert result.exit_code == 2
+        assert "path separators" in result.stderr
+        mock_client.get_json.assert_not_called()
+
+    def test_empty_task_fails_without_creating_zip(
+        self, runner, mock_client, tmp_path
+    ):
+        mock_client.get_json.return_value = self.task_payload([])
+        result = invoke(
+            runner,
+            [
+                "--api-key",
+                "k",
+                "metadata",
+                "download",
+                "--task-id",
+                "T1",
+                "-o",
+                str(tmp_path),
+            ],
+        )
+
+        assert result.exit_code == 1
+        assert "download task contains no job ids" in result.stderr
+        assert not (tmp_path / "T1_metadata.zip").exists()

{benchloop_client-0.2.21 → benchloop_client-0.2.22}/uv.lock

@@ -27,7 +27,7 @@ wheels = [
 
 [[package]]
 name = "benchloop-client"
-version = "0.2.21"
+version = "0.2.22"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },