benchloop-client 0.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aliyun/bmcli/__init__.py +3 -0
- aliyun/bmcli/__main__.py +26 -0
- aliyun/bmcli/cli.py +136 -0
- aliyun/bmcli/client.py +145 -0
- aliyun/bmcli/commands/__init__.py +1 -0
- aliyun/bmcli/commands/_common.py +36 -0
- aliyun/bmcli/commands/artifacts.py +138 -0
- aliyun/bmcli/commands/jobs.py +131 -0
- aliyun/bmcli/commands/metrics.py +136 -0
- aliyun/bmcli/commands/overview.py +70 -0
- aliyun/bmcli/commands/pull.py +168 -0
- aliyun/bmcli/commands/report.py +103 -0
- aliyun/bmcli/commands/tasks.py +121 -0
- aliyun/bmcli/commands/trajectory.py +92 -0
- aliyun/bmcli/config.py +120 -0
- aliyun/bmcli/exit_codes.py +15 -0
- aliyun/bmcli/ids.py +87 -0
- aliyun/bmcli/output.py +80 -0
- aliyun/bmcli/runtime.py +147 -0
- benchloop_client-0.1.6.dist-info/METADATA +100 -0
- benchloop_client-0.1.6.dist-info/RECORD +23 -0
- benchloop_client-0.1.6.dist-info/WHEEL +4 -0
- benchloop_client-0.1.6.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""`bmcli metrics` — run metrics for Group (overall/detail) and Job."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from aliyun.bmcli import output
|
|
10
|
+
from aliyun.bmcli.commands._common import (
|
|
11
|
+
GroupOption,
|
|
12
|
+
JobOption,
|
|
13
|
+
OutputDirOption,
|
|
14
|
+
SkipExistingOption,
|
|
15
|
+
expand,
|
|
16
|
+
resolve_output_dir,
|
|
17
|
+
)
|
|
18
|
+
from aliyun.bmcli.ids import enforce_mutually_exclusive, require_any
|
|
19
|
+
from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
|
|
20
|
+
|
|
21
|
+
app = typer.Typer(help="Run metrics (Group overall/detail, Job).", no_args_is_help=True)
|
|
22
|
+
|
|
23
|
+
_SCOPES = ("overall", "detail", "all")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _validate_scope(scope: str) -> str:
|
|
27
|
+
if scope not in _SCOPES:
|
|
28
|
+
raise typer.BadParameter(f"--scope must be one of {', '.join(_SCOPES)}")
|
|
29
|
+
return scope
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _fetch_job(client, job_id: str):
|
|
33
|
+
data = client.get_json(f"/openapi/metrics/{job_id}")
|
|
34
|
+
if isinstance(data, dict):
|
|
35
|
+
return {
|
|
36
|
+
"job_id": data.get("job_id", job_id),
|
|
37
|
+
"metrics": data.get("metrics"),
|
|
38
|
+
"tokenized_metrics": data.get("tokenized_metrics"),
|
|
39
|
+
}
|
|
40
|
+
# Fallback when the backend returns a non-dict (e.g. bare array) body.
|
|
41
|
+
return {"job_id": job_id, "metrics": data, "tokenized_metrics": None}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _fetch_group(client, group_id: str, scope: str):
|
|
45
|
+
result: dict = {"group_id": group_id}
|
|
46
|
+
if scope in ("overall", "all"):
|
|
47
|
+
result["overall"] = client.get_json(
|
|
48
|
+
f"/openapi/groups/{group_id}/metrics/overall"
|
|
49
|
+
)
|
|
50
|
+
if scope in ("detail", "all"):
|
|
51
|
+
result["detail"] = client.get_json(
|
|
52
|
+
f"/openapi/groups/{group_id}/metrics/detail"
|
|
53
|
+
)
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@app.command("get")
|
|
58
|
+
def get(
|
|
59
|
+
ctx: typer.Context,
|
|
60
|
+
group: Optional[list[str]] = GroupOption,
|
|
61
|
+
job: Optional[list[str]] = JobOption,
|
|
62
|
+
scope: str = typer.Option("all", "--scope", help="overall|detail|all (Group only)."),
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Query metrics; prints a JSON array to stdout."""
|
|
65
|
+
rt = get_runtime(ctx)
|
|
66
|
+
groups = expand(group)
|
|
67
|
+
jobs = expand(job)
|
|
68
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
69
|
+
require_any(groups, jobs)
|
|
70
|
+
_validate_scope(scope)
|
|
71
|
+
|
|
72
|
+
if jobs:
|
|
73
|
+
res = safe_run_concurrent(jobs, lambda jid: _fetch_job(rt.client, jid), rt.concurrency)
|
|
74
|
+
else:
|
|
75
|
+
res = safe_run_concurrent(
|
|
76
|
+
groups, lambda gid: _fetch_group(rt.client, gid, scope), rt.concurrency
|
|
77
|
+
)
|
|
78
|
+
report_and_exit(res, emit=True, pretty=rt.pretty)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@app.command("download")
|
|
82
|
+
def download(
|
|
83
|
+
ctx: typer.Context,
|
|
84
|
+
group: Optional[list[str]] = GroupOption,
|
|
85
|
+
job: Optional[list[str]] = JobOption,
|
|
86
|
+
scope: str = typer.Option("all", "--scope", help="overall|detail|all (Group only)."),
|
|
87
|
+
output_dir: Optional[str] = OutputDirOption,
|
|
88
|
+
skip_existing: bool = SkipExistingOption,
|
|
89
|
+
) -> None:
|
|
90
|
+
"""Download metrics to the standard file layout."""
|
|
91
|
+
rt = get_runtime(ctx)
|
|
92
|
+
groups = expand(group)
|
|
93
|
+
jobs = expand(job)
|
|
94
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
95
|
+
require_any(groups, jobs)
|
|
96
|
+
_validate_scope(scope)
|
|
97
|
+
root = resolve_output_dir(rt.output_dir, output_dir)
|
|
98
|
+
|
|
99
|
+
def job_worker(job_id: str):
|
|
100
|
+
dest = root / f"{job_id}_run_metrics.json"
|
|
101
|
+
if skip_existing and dest.exists():
|
|
102
|
+
output.log(f"skip existing {dest}")
|
|
103
|
+
return str(dest)
|
|
104
|
+
data = rt.client.get_json(f"/openapi/metrics/{job_id}")
|
|
105
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
106
|
+
output.log(f"wrote {dest}")
|
|
107
|
+
return str(dest)
|
|
108
|
+
|
|
109
|
+
def group_worker(group_id: str):
|
|
110
|
+
written = []
|
|
111
|
+
gdir = root / group_id
|
|
112
|
+
if scope in ("overall", "all"):
|
|
113
|
+
dest = gdir / "overall_metrics.json"
|
|
114
|
+
if skip_existing and dest.exists():
|
|
115
|
+
output.log(f"skip existing {dest}")
|
|
116
|
+
else:
|
|
117
|
+
data = rt.client.get_json(f"/openapi/groups/{group_id}/metrics/overall")
|
|
118
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
119
|
+
output.log(f"wrote {dest}")
|
|
120
|
+
written.append(str(dest))
|
|
121
|
+
if scope in ("detail", "all"):
|
|
122
|
+
dest = gdir / "detail_metrics.json"
|
|
123
|
+
if skip_existing and dest.exists():
|
|
124
|
+
output.log(f"skip existing {dest}")
|
|
125
|
+
else:
|
|
126
|
+
data = rt.client.get_json(f"/openapi/groups/{group_id}/metrics/detail")
|
|
127
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
128
|
+
output.log(f"wrote {dest}")
|
|
129
|
+
written.append(str(dest))
|
|
130
|
+
return written
|
|
131
|
+
|
|
132
|
+
if jobs:
|
|
133
|
+
res = safe_run_concurrent(jobs, job_worker, rt.concurrency)
|
|
134
|
+
else:
|
|
135
|
+
res = safe_run_concurrent(groups, group_worker, rt.concurrency)
|
|
136
|
+
report_and_exit(res, emit=False)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""`bmcli overview` — run statistics (Group / Task / Job); query only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from aliyun.bmcli.commands._common import GroupOption, JobOption, expand
|
|
10
|
+
from aliyun.bmcli.ids import enforce_mutually_exclusive, require_any
|
|
11
|
+
from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(help="Run statistics overview (query only).", no_args_is_help=True)
|
|
14
|
+
|
|
15
|
+
_SCOPES = ("summary", "tasks", "all")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _validate_scope(scope: str) -> str:
|
|
19
|
+
if scope not in _SCOPES:
|
|
20
|
+
raise typer.BadParameter(f"--scope must be one of {', '.join(_SCOPES)}")
|
|
21
|
+
return scope
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _fetch_job(client, job_id: str):
|
|
25
|
+
job_obj = client.get_json(f"/openapi/jobs/{job_id}")
|
|
26
|
+
group_id = job_obj.get("group_id") if isinstance(job_obj, dict) else None
|
|
27
|
+
stats = client.get_json(
|
|
28
|
+
"/openapi/jobs/stats",
|
|
29
|
+
params={"group_id": group_id} if group_id else None,
|
|
30
|
+
)
|
|
31
|
+
return {"job_id": job_id, "job": job_obj, "stats": stats}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _fetch_group(client, group_id: str, scope: str):
|
|
35
|
+
result: dict = {"group_id": group_id}
|
|
36
|
+
if scope in ("summary", "all"):
|
|
37
|
+
result["summary"] = client.get_json(f"/openapi/groups/{group_id}/eval_summary")
|
|
38
|
+
if scope in ("tasks", "all"):
|
|
39
|
+
result["tasks"] = client.get_json(f"/openapi/groups/{group_id}/eval_tasks")
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@app.command("get")
|
|
44
|
+
def get(
|
|
45
|
+
ctx: typer.Context,
|
|
46
|
+
group: Optional[list[str]] = GroupOption,
|
|
47
|
+
job: Optional[list[str]] = JobOption,
|
|
48
|
+
scope: str = typer.Option(
|
|
49
|
+
"all", "--scope", help="summary|tasks|all (Group only)."
|
|
50
|
+
),
|
|
51
|
+
) -> None:
|
|
52
|
+
"""Query overview; prints a JSON array to stdout.
|
|
53
|
+
|
|
54
|
+
Job (``-j``): jobs/{id} merged with jobs/stats.
|
|
55
|
+
Group (``-g``): eval_summary and/or eval_tasks per ``--scope``.
|
|
56
|
+
"""
|
|
57
|
+
rt = get_runtime(ctx)
|
|
58
|
+
groups = expand(group)
|
|
59
|
+
jobs = expand(job)
|
|
60
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
61
|
+
require_any(groups, jobs)
|
|
62
|
+
_validate_scope(scope)
|
|
63
|
+
|
|
64
|
+
if jobs:
|
|
65
|
+
res = safe_run_concurrent(jobs, lambda jid: _fetch_job(rt.client, jid), rt.concurrency)
|
|
66
|
+
else:
|
|
67
|
+
res = safe_run_concurrent(
|
|
68
|
+
groups, lambda gid: _fetch_group(rt.client, gid, scope), rt.concurrency
|
|
69
|
+
)
|
|
70
|
+
report_and_exit(res, emit=True, pretty=rt.pretty)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""`bmcli pull` — one-shot aggregate download into the standard layout."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from aliyun.bmcli import output
|
|
11
|
+
from aliyun.bmcli.commands._common import (
|
|
12
|
+
GroupOption,
|
|
13
|
+
JobOption,
|
|
14
|
+
OutputDirOption,
|
|
15
|
+
SkipExistingOption,
|
|
16
|
+
expand,
|
|
17
|
+
resolve_output_dir,
|
|
18
|
+
)
|
|
19
|
+
from aliyun.bmcli.ids import enforce_mutually_exclusive, require_any
|
|
20
|
+
from aliyun.bmcli.runtime import get_runtime
|
|
21
|
+
from aliyun.bmcli.exit_codes import OK, PARTIAL
|
|
22
|
+
|
|
23
|
+
_ALL_WITH = ("metrics", "trajectory", "report", "artifacts", "tasks", "jobs")
|
|
24
|
+
_DEFAULT_WITH = "metrics,trajectory"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _parse_with(value: str) -> list[str]:
|
|
28
|
+
parts = [p.strip() for p in value.split(",") if p.strip()]
|
|
29
|
+
bad = [p for p in parts if p not in _ALL_WITH]
|
|
30
|
+
if bad:
|
|
31
|
+
raise typer.BadParameter(
|
|
32
|
+
f"--with has unsupported value(s): {', '.join(bad)}; "
|
|
33
|
+
f"choose from {', '.join(_ALL_WITH)}"
|
|
34
|
+
)
|
|
35
|
+
return parts
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def pull(
|
|
39
|
+
ctx: typer.Context,
|
|
40
|
+
group: Optional[list[str]] = GroupOption,
|
|
41
|
+
job: Optional[list[str]] = JobOption,
|
|
42
|
+
with_: str = typer.Option(
|
|
43
|
+
_DEFAULT_WITH, "--with", help="Comma list: metrics,trajectory,report,artifacts,tasks,jobs."
|
|
44
|
+
),
|
|
45
|
+
output_dir: Optional[str] = OutputDirOption,
|
|
46
|
+
skip_existing: bool = SkipExistingOption,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Aggregate downloads for the given groups/jobs into the layout."""
|
|
49
|
+
rt = get_runtime(ctx)
|
|
50
|
+
groups = expand(group)
|
|
51
|
+
jobs = expand(job)
|
|
52
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
53
|
+
require_any(groups, jobs)
|
|
54
|
+
selected = _parse_with(with_)
|
|
55
|
+
root = resolve_output_dir(rt.output_dir, output_dir)
|
|
56
|
+
client = rt.client
|
|
57
|
+
|
|
58
|
+
failures: list[tuple[str, str]] = []
|
|
59
|
+
|
|
60
|
+
def _safe(label: str, fn) -> None:
|
|
61
|
+
try:
|
|
62
|
+
fn()
|
|
63
|
+
except Exception as exc: # noqa: BLE001 - aggregate, collect failures
|
|
64
|
+
failures.append((label, str(exc)))
|
|
65
|
+
|
|
66
|
+
def _write_json(dest: Path, data: Any) -> None:
|
|
67
|
+
if skip_existing and dest.exists():
|
|
68
|
+
output.log(f"skip existing {dest}")
|
|
69
|
+
return
|
|
70
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
71
|
+
output.log(f"wrote {dest}")
|
|
72
|
+
|
|
73
|
+
for jid in jobs:
|
|
74
|
+
if "metrics" in selected:
|
|
75
|
+
_safe(f"{jid}:metrics", lambda jid=jid: _write_json(
|
|
76
|
+
root / f"{jid}_run_metrics.json",
|
|
77
|
+
client.get_json(f"/openapi/metrics/{jid}")))
|
|
78
|
+
if "trajectory" in selected:
|
|
79
|
+
_safe(f"{jid}:trajectory", lambda jid=jid: _write_json(
|
|
80
|
+
root / f"{jid}_trajectory.json",
|
|
81
|
+
client.get_json(f"/openapi/jobs/{jid}/trajectory")))
|
|
82
|
+
if "report" in selected:
|
|
83
|
+
_safe(f"{jid}:report", lambda jid=jid: _pull_report(client, jid, root, skip_existing))
|
|
84
|
+
if "artifacts" in selected:
|
|
85
|
+
_safe(f"{jid}:artifacts", lambda jid=jid: _pull_job_artifacts(
|
|
86
|
+
rt, jid, root, skip_existing, failures))
|
|
87
|
+
|
|
88
|
+
for gid in groups:
|
|
89
|
+
if "metrics" in selected:
|
|
90
|
+
_safe(f"{gid}:metrics:overall", lambda gid=gid: _write_json(
|
|
91
|
+
root / gid / "overall_metrics.json",
|
|
92
|
+
client.get_json(f"/openapi/groups/{gid}/metrics/overall")))
|
|
93
|
+
_safe(f"{gid}:metrics:detail", lambda gid=gid: _write_json(
|
|
94
|
+
root / gid / "detail_metrics.json",
|
|
95
|
+
client.get_json(f"/openapi/groups/{gid}/metrics/detail")))
|
|
96
|
+
if "trajectory" in selected:
|
|
97
|
+
_safe(f"{gid}:trajectory", lambda gid=gid: _write_json(
|
|
98
|
+
root / gid / "trajectory.json",
|
|
99
|
+
client.get_json(f"/openapi/groups/{gid}/trajectory")))
|
|
100
|
+
if "tasks" in selected:
|
|
101
|
+
_safe(f"{gid}:tasks", lambda gid=gid: _write_json(
|
|
102
|
+
root / gid / "task_list.json",
|
|
103
|
+
client.get_json(f"/openapi/groups/{gid}/eval_tasks", params={"skip": 0, "limit": 1000})))
|
|
104
|
+
if "jobs" in selected:
|
|
105
|
+
_safe(f"{gid}:jobs", lambda gid=gid: _write_json(
|
|
106
|
+
root / gid / "job_list.json",
|
|
107
|
+
client.get_json("/openapi/jobs", params={"group_id": gid, "skip": 0, "limit": 1000})))
|
|
108
|
+
if "artifacts" in selected:
|
|
109
|
+
_safe(f"{gid}:artifacts", lambda gid=gid: _pull_group_artifacts(
|
|
110
|
+
rt, gid, root, skip_existing, failures))
|
|
111
|
+
|
|
112
|
+
if failures:
|
|
113
|
+
output.error(f"{len(failures)} pull step(s) failed:")
|
|
114
|
+
for label, msg in failures:
|
|
115
|
+
output.error(f" - {label}: {msg}")
|
|
116
|
+
raise typer.Exit(code=PARTIAL)
|
|
117
|
+
raise typer.Exit(code=OK)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _pull_report(client, job_id: str, root: Path, skip_existing: bool) -> None:
|
|
121
|
+
data = client.get_json(f"/openapi/jobs/{job_id}/report", params={"agent": "all"})
|
|
122
|
+
reports = data.get("reports", []) if isinstance(data, dict) else []
|
|
123
|
+
for entry in reports:
|
|
124
|
+
a = entry.get("agent", "")
|
|
125
|
+
dest = root / f"{job_id}_report_agent_{a}.md"
|
|
126
|
+
if skip_existing and dest.exists():
|
|
127
|
+
output.log(f"skip existing {dest}")
|
|
128
|
+
continue
|
|
129
|
+
output.write_text_file(dest, entry.get("content", "") or "")
|
|
130
|
+
output.log(f"wrote {dest}")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _filter_arts(arts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
134
|
+
return [a for a in arts if a.get("url")]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _pull_job_artifacts(rt, job_id: str, root: Path, skip_existing: bool,
|
|
138
|
+
failures: list[tuple[str, str]]) -> None:
|
|
139
|
+
data = rt.client.get_json(f"/openapi/jobs/{job_id}/artifacts")
|
|
140
|
+
arts = data.get("artifacts", []) if isinstance(data, dict) else []
|
|
141
|
+
for a in _filter_arts(arts):
|
|
142
|
+
dest = root / job_id / (a.get("name") or "artifact")
|
|
143
|
+
if skip_existing and dest.exists():
|
|
144
|
+
output.log(f"skip existing {dest}")
|
|
145
|
+
continue
|
|
146
|
+
try:
|
|
147
|
+
rt.client.download_file(a["url"], dest)
|
|
148
|
+
output.log(f"downloaded {dest}")
|
|
149
|
+
except Exception as exc: # noqa: BLE001
|
|
150
|
+
failures.append((f"{job_id}:{a.get('name')}", str(exc)))
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _pull_group_artifacts(rt, group_id: str, root: Path, skip_existing: bool,
|
|
154
|
+
failures: list[tuple[str, str]]) -> None:
|
|
155
|
+
data = rt.client.get_json(f"/openapi/groups/{group_id}/artifacts")
|
|
156
|
+
arts = data.get("artifacts", []) if isinstance(data, dict) else []
|
|
157
|
+
for a in _filter_arts(arts):
|
|
158
|
+
# nest under {group_id}/{job_id}/ to avoid same-name (result.tgz) collisions
|
|
159
|
+
jid = str(a.get("job_id") or "")
|
|
160
|
+
dest = root / group_id / jid / (a.get("name") or "artifact")
|
|
161
|
+
if skip_existing and dest.exists():
|
|
162
|
+
output.log(f"skip existing {dest}")
|
|
163
|
+
continue
|
|
164
|
+
try:
|
|
165
|
+
rt.client.download_file(a["url"], dest)
|
|
166
|
+
output.log(f"downloaded {dest}")
|
|
167
|
+
except Exception as exc: # noqa: BLE001
|
|
168
|
+
failures.append((f"{group_id}:{a.get('name')}", str(exc)))
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""`bmcli report` — pre-analysis agent a/b reports (Job only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from aliyun.bmcli import output
|
|
10
|
+
from aliyun.bmcli.commands._common import (
|
|
11
|
+
GroupOption,
|
|
12
|
+
JobOption,
|
|
13
|
+
OutputDirOption,
|
|
14
|
+
SkipExistingOption,
|
|
15
|
+
expand,
|
|
16
|
+
resolve_output_dir,
|
|
17
|
+
)
|
|
18
|
+
from aliyun.bmcli.ids import (
|
|
19
|
+
enforce_mutually_exclusive,
|
|
20
|
+
reject_unsupported_dimension,
|
|
21
|
+
require_any,
|
|
22
|
+
)
|
|
23
|
+
from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
|
|
24
|
+
|
|
25
|
+
app = typer.Typer(help="Pre-analysis agent a/b reports (Job only).", no_args_is_help=True)
|
|
26
|
+
|
|
27
|
+
_AGENTS = ("a", "b", "all")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _validate_agent(agent: str) -> str:
|
|
31
|
+
if agent not in _AGENTS:
|
|
32
|
+
raise typer.BadParameter("--agent must be one of a, b, all")
|
|
33
|
+
return agent
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _guard(groups: list[str], jobs: list[str]) -> None:
|
|
37
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
38
|
+
reject_unsupported_dimension(
|
|
39
|
+
"report", groups=groups, jobs=jobs, allow_group=False, allow_job=True
|
|
40
|
+
)
|
|
41
|
+
require_any(groups, jobs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _fetch(client, job_id: str, agent: str):
|
|
45
|
+
data = client.get_json(f"/openapi/jobs/{job_id}/report", params={"agent": agent})
|
|
46
|
+
return data
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.command("get")
|
|
50
|
+
def get(
|
|
51
|
+
ctx: typer.Context,
|
|
52
|
+
group: Optional[list[str]] = GroupOption,
|
|
53
|
+
job: Optional[list[str]] = JobOption,
|
|
54
|
+
agent: str = typer.Option("all", "--agent", help="a|b|all (default all)."),
|
|
55
|
+
) -> None:
|
|
56
|
+
"""Query reports; prints a JSON array (with job_id, agent, content)."""
|
|
57
|
+
rt = get_runtime(ctx)
|
|
58
|
+
groups = expand(group)
|
|
59
|
+
jobs = expand(job)
|
|
60
|
+
_guard(groups, jobs)
|
|
61
|
+
_validate_agent(agent)
|
|
62
|
+
|
|
63
|
+
res = safe_run_concurrent(
|
|
64
|
+
jobs, lambda jid: _fetch(rt.client, jid, agent), rt.concurrency
|
|
65
|
+
)
|
|
66
|
+
report_and_exit(res, emit=True, pretty=rt.pretty)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@app.command("download")
|
|
70
|
+
def download(
|
|
71
|
+
ctx: typer.Context,
|
|
72
|
+
group: Optional[list[str]] = GroupOption,
|
|
73
|
+
job: Optional[list[str]] = JobOption,
|
|
74
|
+
agent: str = typer.Option("all", "--agent", help="a|b|all (default all)."),
|
|
75
|
+
output_dir: Optional[str] = OutputDirOption,
|
|
76
|
+
skip_existing: bool = SkipExistingOption,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""Download reports as {job_id}_report_agent_<x>.md from the content field."""
|
|
79
|
+
rt = get_runtime(ctx)
|
|
80
|
+
groups = expand(group)
|
|
81
|
+
jobs = expand(job)
|
|
82
|
+
_guard(groups, jobs)
|
|
83
|
+
_validate_agent(agent)
|
|
84
|
+
root = resolve_output_dir(rt.output_dir, output_dir)
|
|
85
|
+
|
|
86
|
+
def worker(job_id: str):
|
|
87
|
+
data = _fetch(rt.client, job_id, agent)
|
|
88
|
+
reports = data.get("reports", []) if isinstance(data, dict) else []
|
|
89
|
+
written = []
|
|
90
|
+
for entry in reports:
|
|
91
|
+
a = entry.get("agent", "")
|
|
92
|
+
content = entry.get("content", "") or ""
|
|
93
|
+
dest = root / f"{job_id}_report_agent_{a}.md"
|
|
94
|
+
if skip_existing and dest.exists():
|
|
95
|
+
output.log(f"skip existing {dest}")
|
|
96
|
+
else:
|
|
97
|
+
output.write_text_file(dest, content)
|
|
98
|
+
output.log(f"wrote {dest}")
|
|
99
|
+
written.append(str(dest))
|
|
100
|
+
return written
|
|
101
|
+
|
|
102
|
+
res = safe_run_concurrent(jobs, worker, rt.concurrency)
|
|
103
|
+
report_and_exit(res, emit=False)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""`bmcli tasks` — Group task list (eval_tasks) with pagination."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from aliyun.bmcli import output
|
|
10
|
+
from aliyun.bmcli.commands._common import (
|
|
11
|
+
GroupOption,
|
|
12
|
+
JobOption,
|
|
13
|
+
OutputDirOption,
|
|
14
|
+
SkipExistingOption,
|
|
15
|
+
expand,
|
|
16
|
+
resolve_output_dir,
|
|
17
|
+
)
|
|
18
|
+
from aliyun.bmcli.ids import (
|
|
19
|
+
enforce_mutually_exclusive,
|
|
20
|
+
reject_unsupported_dimension,
|
|
21
|
+
require_any,
|
|
22
|
+
)
|
|
23
|
+
from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
|
|
24
|
+
|
|
25
|
+
app = typer.Typer(help="Group task list (eval_tasks).", no_args_is_help=True)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _guard(groups: list[str], jobs: list[str]) -> None:
|
|
29
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
30
|
+
reject_unsupported_dimension(
|
|
31
|
+
"tasks", groups=groups, jobs=jobs, allow_group=True, allow_job=False
|
|
32
|
+
)
|
|
33
|
+
require_any(groups, jobs)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _fetch_page(client, group_id: str, skip: int, limit: int, search, status):
|
|
37
|
+
params: dict[str, Any] = {"skip": skip, "limit": limit}
|
|
38
|
+
if search:
|
|
39
|
+
params["search"] = search
|
|
40
|
+
if status:
|
|
41
|
+
params["status"] = status
|
|
42
|
+
return client.get_json(f"/openapi/groups/{group_id}/eval_tasks", params=params)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _fetch_all(client, group_id: str, page_size: int, search, status):
|
|
46
|
+
skip = 0
|
|
47
|
+
all_tasks: list[Any] = []
|
|
48
|
+
total = None
|
|
49
|
+
while True:
|
|
50
|
+
data = _fetch_page(client, group_id, skip, page_size, search, status)
|
|
51
|
+
tasks = data.get("tasks", []) if isinstance(data, dict) else []
|
|
52
|
+
if isinstance(data, dict) and total is None:
|
|
53
|
+
total = data.get("total")
|
|
54
|
+
all_tasks.extend(tasks)
|
|
55
|
+
if not tasks or len(tasks) < page_size:
|
|
56
|
+
break
|
|
57
|
+
skip += page_size
|
|
58
|
+
if total is not None and len(all_tasks) >= total:
|
|
59
|
+
break
|
|
60
|
+
return {"group_id": group_id, "total": total if total is not None else len(all_tasks),
|
|
61
|
+
"tasks": all_tasks}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command("get")
|
|
65
|
+
def get(
|
|
66
|
+
ctx: typer.Context,
|
|
67
|
+
group: Optional[list[str]] = GroupOption,
|
|
68
|
+
job: Optional[list[str]] = JobOption,
|
|
69
|
+
page: int = typer.Option(1, "--page", help="Page number (1-based)."),
|
|
70
|
+
page_size: int = typer.Option(50, "--page-size", help="Page size."),
|
|
71
|
+
all_pages: bool = typer.Option(False, "--all", help="Auto-paginate all tasks."),
|
|
72
|
+
search: Optional[str] = typer.Option(None, "--search", show_default=False),
|
|
73
|
+
status: Optional[str] = typer.Option(None, "--status", show_default=False),
|
|
74
|
+
) -> None:
|
|
75
|
+
"""Query the group task list; prints a JSON array."""
|
|
76
|
+
rt = get_runtime(ctx)
|
|
77
|
+
groups = expand(group)
|
|
78
|
+
jobs = expand(job)
|
|
79
|
+
_guard(groups, jobs)
|
|
80
|
+
|
|
81
|
+
def worker(group_id: str):
|
|
82
|
+
if all_pages:
|
|
83
|
+
return _fetch_all(rt.client, group_id, page_size, search, status)
|
|
84
|
+
skip = max(0, (page - 1) * page_size)
|
|
85
|
+
data = _fetch_page(rt.client, group_id, skip, page_size, search, status)
|
|
86
|
+
return {"group_id": group_id, **(data if isinstance(data, dict) else {"data": data})}
|
|
87
|
+
|
|
88
|
+
res = safe_run_concurrent(groups, worker, rt.concurrency)
|
|
89
|
+
report_and_exit(res, emit=True, pretty=rt.pretty)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@app.command("download")
|
|
93
|
+
def download(
|
|
94
|
+
ctx: typer.Context,
|
|
95
|
+
group: Optional[list[str]] = GroupOption,
|
|
96
|
+
job: Optional[list[str]] = JobOption,
|
|
97
|
+
page_size: int = typer.Option(50, "--page-size", help="Page size for pagination."),
|
|
98
|
+
search: Optional[str] = typer.Option(None, "--search", show_default=False),
|
|
99
|
+
status: Optional[str] = typer.Option(None, "--status", show_default=False),
|
|
100
|
+
output_dir: Optional[str] = OutputDirOption,
|
|
101
|
+
skip_existing: bool = SkipExistingOption,
|
|
102
|
+
) -> None:
|
|
103
|
+
"""Download the full task list to {group_id}/task_list.json."""
|
|
104
|
+
rt = get_runtime(ctx)
|
|
105
|
+
groups = expand(group)
|
|
106
|
+
jobs = expand(job)
|
|
107
|
+
_guard(groups, jobs)
|
|
108
|
+
root = resolve_output_dir(rt.output_dir, output_dir)
|
|
109
|
+
|
|
110
|
+
def worker(group_id: str):
|
|
111
|
+
dest = root / group_id / "task_list.json"
|
|
112
|
+
if skip_existing and dest.exists():
|
|
113
|
+
output.log(f"skip existing {dest}")
|
|
114
|
+
return str(dest)
|
|
115
|
+
data = _fetch_all(rt.client, group_id, page_size, search, status)
|
|
116
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
117
|
+
output.log(f"wrote {dest} ({len(data.get('tasks', []))} tasks)")
|
|
118
|
+
return str(dest)
|
|
119
|
+
|
|
120
|
+
res = safe_run_concurrent(groups, worker, rt.concurrency)
|
|
121
|
+
report_and_exit(res, emit=False)
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""`bmcli trajectory` — trajectory data for Group and Job."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from aliyun.bmcli import output
|
|
10
|
+
from aliyun.bmcli.commands._common import (
|
|
11
|
+
GroupOption,
|
|
12
|
+
JobOption,
|
|
13
|
+
OutputDirOption,
|
|
14
|
+
SkipExistingOption,
|
|
15
|
+
expand,
|
|
16
|
+
resolve_output_dir,
|
|
17
|
+
)
|
|
18
|
+
from aliyun.bmcli.ids import enforce_mutually_exclusive, require_any
|
|
19
|
+
from aliyun.bmcli.runtime import get_runtime, report_and_exit, safe_run_concurrent
|
|
20
|
+
|
|
21
|
+
app = typer.Typer(help="Trajectory data (Group, Job).", no_args_is_help=True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@app.command("get")
|
|
25
|
+
def get(
|
|
26
|
+
ctx: typer.Context,
|
|
27
|
+
group: Optional[list[str]] = GroupOption,
|
|
28
|
+
job: Optional[list[str]] = JobOption,
|
|
29
|
+
) -> None:
|
|
30
|
+
"""Query trajectory; prints a JSON array to stdout."""
|
|
31
|
+
rt = get_runtime(ctx)
|
|
32
|
+
groups = expand(group)
|
|
33
|
+
jobs = expand(job)
|
|
34
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
35
|
+
require_any(groups, jobs)
|
|
36
|
+
|
|
37
|
+
if jobs:
|
|
38
|
+
res = safe_run_concurrent(
|
|
39
|
+
jobs,
|
|
40
|
+
lambda jid: rt.client.get_json(f"/openapi/jobs/{jid}/trajectory"),
|
|
41
|
+
rt.concurrency,
|
|
42
|
+
)
|
|
43
|
+
else:
|
|
44
|
+
res = safe_run_concurrent(
|
|
45
|
+
groups,
|
|
46
|
+
lambda gid: rt.client.get_json(f"/openapi/groups/{gid}/trajectory"),
|
|
47
|
+
rt.concurrency,
|
|
48
|
+
)
|
|
49
|
+
report_and_exit(res, emit=True, pretty=rt.pretty)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@app.command("download")
|
|
53
|
+
def download(
|
|
54
|
+
ctx: typer.Context,
|
|
55
|
+
group: Optional[list[str]] = GroupOption,
|
|
56
|
+
job: Optional[list[str]] = JobOption,
|
|
57
|
+
output_dir: Optional[str] = OutputDirOption,
|
|
58
|
+
skip_existing: bool = SkipExistingOption,
|
|
59
|
+
) -> None:
|
|
60
|
+
"""Download trajectory to the standard file layout."""
|
|
61
|
+
rt = get_runtime(ctx)
|
|
62
|
+
groups = expand(group)
|
|
63
|
+
jobs = expand(job)
|
|
64
|
+
enforce_mutually_exclusive(groups, jobs)
|
|
65
|
+
require_any(groups, jobs)
|
|
66
|
+
root = resolve_output_dir(rt.output_dir, output_dir)
|
|
67
|
+
|
|
68
|
+
def job_worker(job_id: str):
|
|
69
|
+
dest = root / f"{job_id}_trajectory.json"
|
|
70
|
+
if skip_existing and dest.exists():
|
|
71
|
+
output.log(f"skip existing {dest}")
|
|
72
|
+
return str(dest)
|
|
73
|
+
data = rt.client.get_json(f"/openapi/jobs/{job_id}/trajectory")
|
|
74
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
75
|
+
output.log(f"wrote {dest}")
|
|
76
|
+
return str(dest)
|
|
77
|
+
|
|
78
|
+
def group_worker(group_id: str):
|
|
79
|
+
dest = root / group_id / "trajectory.json"
|
|
80
|
+
if skip_existing and dest.exists():
|
|
81
|
+
output.log(f"skip existing {dest}")
|
|
82
|
+
return str(dest)
|
|
83
|
+
data = rt.client.get_json(f"/openapi/groups/{group_id}/trajectory")
|
|
84
|
+
output.write_json_file(dest, data, pretty=rt.pretty)
|
|
85
|
+
output.log(f"wrote {dest}")
|
|
86
|
+
return str(dest)
|
|
87
|
+
|
|
88
|
+
if jobs:
|
|
89
|
+
res = safe_run_concurrent(jobs, job_worker, rt.concurrency)
|
|
90
|
+
else:
|
|
91
|
+
res = safe_run_concurrent(groups, group_worker, rt.concurrency)
|
|
92
|
+
report_and_exit(res, emit=False)
|