benchloop-client 0.2.22__tar.gz → 0.2.24__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchloop_client-0.2.24/PKG-INFO +303 -0
- benchloop_client-0.2.24/README.md +291 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/pyproject.toml +1 -1
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/cli.py +2 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/_common.py +6 -0
- benchloop_client-0.2.24/src/aliyun/bmcli/commands/brm.py +440 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/metrics.py +14 -3
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/pull.py +8 -3
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/trajectory.py +246 -56
- benchloop_client-0.2.24/tests/test_cmd_brm.py +451 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_metrics.py +32 -2
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_pull.py +80 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_trajectory.py +253 -42
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/uv.lock +89 -89
- benchloop_client-0.2.22/PKG-INFO +0 -144
- benchloop_client-0.2.22/README.md +0 -132
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/.gitignore +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/.python-version +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/RELEASE.md +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/benchloop-cli-design.md +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/bmcli_examples.sh +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/__init__.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/__main__.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/client.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/__init__.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/ap_cmd.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/artifacts.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/group_data.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/jobs.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/metadata.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/overview.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/report.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/tasks.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/config.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/exit_codes.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/ids.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/output.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/runtime.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/version_check.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/__init__.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/conftest.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cli_options.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_client.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_ap_cmd.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_artifacts.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_metadata.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_overview.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_report.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_tasks_jobs.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_config.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_exit_code_behavior.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_exit_codes.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_group_data.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_ids.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_runtime.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_transfer_progress.py +0 -0
- {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_version_check.py +0 -0
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: benchloop_client
|
|
3
|
+
Version: 0.2.24
|
|
4
|
+
Summary: Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts).
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: httpx>=0.25.2
|
|
7
|
+
Requires-Dist: tomli>=1.1.0; python_version < '3.13'
|
|
8
|
+
Requires-Dist: typer>=0.9.4
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# benchloop-client (`bl`, `bmcli`)
|
|
14
|
+
|
|
15
|
+
Command line tool for fetching and downloading benchmark analysis data
|
|
16
|
+
(run metrics, trajectory, metadata, pre-analysis reports, task/job lists,
|
|
17
|
+
and artifacts) by `group_id` / `job_id` / download `task_id`, and for
|
|
18
|
+
submitting Behavior Reward Model (`brm`) analysis tasks.
|
|
19
|
+
|
|
20
|
+
> PyPI package name is `benchloop_client`; the installed commands are `bl` and `bmcli`.
|
|
21
|
+
|
|
22
|
+
## Install package
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
pip install benchloop_client
|
|
26
|
+
|
|
27
|
+
bl --api-key $BENCH_LOOP_API_KEY overview get -j $JOB_ID
|
|
28
|
+
bl --api-key $BENCH_LOOP_API_KEY metrics get -j $JOB_ID
|
|
29
|
+
|
|
30
|
+
# Option 2: config ~/.benchloop/config.toml by cli, and run bl without --api-key option:
|
|
31
|
+
bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
|
|
32
|
+
|
|
33
|
+
bl overview get -j $JOB_ID
|
|
34
|
+
bl metrics get -j $JOB_ID
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The CLI checks PyPI for a newer release in the background at most once every
|
|
39
|
+
24 hours. Update notices include the current/latest versions and the upgrade
|
|
40
|
+
command. Notices are written only to stderr, so JSON output remains pipeable.
|
|
41
|
+
|
|
42
|
+
## Install and run from source
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
# with uv (recommended)
|
|
46
|
+
uv sync
|
|
47
|
+
uv run bl --help
|
|
48
|
+
|
|
49
|
+
# or editable install with pip
|
|
50
|
+
python -m venv .venv && source .venv/bin/activate
|
|
51
|
+
pip install -e .
|
|
52
|
+
bl --help
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Auth & config
|
|
56
|
+
|
|
57
|
+
Authentication uses a personal api key sent as the `X-API-Key` header.
|
|
58
|
+
|
|
59
|
+
Precedence: CLI flag > env var > `~/.benchloop/config.toml` > default.
|
|
60
|
+
|
|
61
|
+
- Env: `BENCHLOOP_API_KEY` (or `BENCHMARK_API_KEY`), `BENCHLOOP_HOST`, `BENCHLOOP_OUTPUT_DIR`
|
|
62
|
+
- Default host (prod): `https://benchmark-analysis.alibaba-inc.com`
|
|
63
|
+
- Pre host: `https://pre-benchmark-analysis.alibaba-inc.com`
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Usage
|
|
70
|
+
|
|
71
|
+
Two-segment `bl <resource> <action>` form. `get` prints a JSON array to
|
|
72
|
+
stdout (pipeable to `jq`); `download` writes the standard file layout.
|
|
73
|
+
Global options such as `--api-key`, `--host`, `--timeout`, `--retry`,
|
|
74
|
+
`--concurrency`, `--quiet`, and `--verbose` must be placed before the resource
|
|
75
|
+
name. Use `--concurrency 1` for a per-file byte progress bar, or a higher value
|
|
76
|
+
for concurrent requests/downloads.
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
bl metrics get -j J123 | jq '.[0]'
|
|
80
|
+
bl metrics download -g G1 --scope all -o ./out
|
|
81
|
+
bl --concurrency 10 trajectory download -g G1,G2 -o ./out
|
|
82
|
+
bl --concurrency 10 trajectory download --task-id TASK_ID -o ./out
|
|
83
|
+
bl metadata download --task-id TASK_ID -o ./out
|
|
84
|
+
bl report download -j J123 -o ./out
|
|
85
|
+
bl report get -g G123
|
|
86
|
+
bl report download -g G123 -o ./out
|
|
87
|
+
bl jobs get -g G1 --all # get supports --all (auto-paginate)
|
|
88
|
+
bl jobs download -g G1 # download is always full
|
|
89
|
+
bl artifacts download -g G1 -o ./out
|
|
90
|
+
bl artifacts download --task-id TASK_ID -o ./out
|
|
91
|
+
bl pull -g G1 --with metrics,trajectory,jobs,artifacts -o ./out
|
|
92
|
+
bl brm run -g G1
|
|
93
|
+
bl brm run -g G1 --steps 10,20 --sample-size 512 --scoring-limit 32
|
|
94
|
+
bl brm get --task-id UID
|
|
95
|
+
bl brm get --task-id UID --scope overall
|
|
96
|
+
bl brm get --task-id UID --scope detail
|
|
97
|
+
bl brm get --task-id UID --scope status
|
|
98
|
+
bl brm status --task-id UID
|
|
99
|
+
bl brm list -g G1
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Metrics and Job trajectory requests always explicitly use the Trace source. The
|
|
103
|
+
CLI does not expose a `trace`/`artifacts` source selector.
|
|
104
|
+
|
|
105
|
+
Group trajectory retrieval and downloads use the backend Trace export API. The
|
|
106
|
+
CLI waits for the export URL, downloads the gzip JSON-lines payload, and keeps
|
|
107
|
+
the existing `group_id`/`trajectories` JSON layout. The export wait timeout is
|
|
108
|
+
30 minutes by default; override it in seconds with
|
|
109
|
+
`--download-timeout SECONDS` on trajectory commands or `pull`.
|
|
110
|
+
|
|
111
|
+
Group downloads always auto-paginate and include every Job in the Group. In
|
|
112
|
+
particular, `trajectory`, detailed `metrics`, `report`, and `artifacts` enumerate
|
|
113
|
+
the complete Job list instead of relying on the backend Group endpoints that
|
|
114
|
+
return at most 1000 Jobs. Existing aggregate files keep their original paths:
|
|
115
|
+
|
|
116
|
+
```text
|
|
117
|
+
out/G1/trajectory.json
|
|
118
|
+
out/G1/detail_metrics.json
|
|
119
|
+
out/G1_report.md
|
|
120
|
+
out/G1/J123_report.md
|
|
121
|
+
out/G1/J123/<artifact-name>
|
|
122
|
+
out/TASK_ID_trajectories.zip
|
|
123
|
+
out/TASK_ID_metadata.zip
|
|
124
|
+
out/TASK_ID/J123.tgz
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`report download -g` writes both the Group summary report and each Job report.
|
|
128
|
+
`pull -g` uses the same complete download behavior for every selected resource.
|
|
129
|
+
Use `--skip-existing` to resume file-based downloads after an interruption.
|
|
130
|
+
|
|
131
|
+
IDs accept repeated flags, comma-separated values, and `-` to read from stdin.
|
|
132
|
+
`-g` and `-j` are mutually exclusive in a single command. For trajectory and
|
|
133
|
+
artifact downloads, `--task-id` is a third mutually exclusive input mode.
|
|
134
|
+
|
|
135
|
+
### Download trajectories by task ID
|
|
136
|
+
|
|
137
|
+
The benchmark analysis Jobs page can turn the current filters into a trajectory
|
|
138
|
+
download task. Copy the command shown by the page and run it locally:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
bl trajectory download --task-id TASK_ID -o ./out
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
The CLI fetches the task's complete job ID list, downloads job trajectories with
|
|
145
|
+
fixed concurrency, and prints count/ETA progress to stderr. It creates
|
|
146
|
+
`./out/TASK_ID_trajectories.zip` with this layout:
|
|
147
|
+
|
|
148
|
+
```text
|
|
149
|
+
trajectories/<jobId>_trajectory.json
|
|
150
|
+
metadata/<jobId>.json
|
|
151
|
+
manifest.json
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Metadata is fetched from `/openapi/jobs/{job_id}/meta` and archived alongside
|
|
155
|
+
the trajectory. If trajectory download succeeds but metadata fails, the
|
|
156
|
+
trajectory is still archived, the metadata error is recorded in
|
|
157
|
+
`manifest.metadata_failures`, and the command exits with code `1`. If only some
|
|
158
|
+
jobs fail, the successful files are still archived. If every job fails, no empty
|
|
159
|
+
ZIP is created.
|
|
160
|
+
|
|
161
|
+
### Download metadata by task ID
|
|
162
|
+
|
|
163
|
+
Use `metadata download` when you only need per-job metadata for a download task:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
bl metadata download --task-id TASK_ID -o ./out
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
The CLI creates `./out/TASK_ID_metadata.zip` with this layout:
|
|
170
|
+
|
|
171
|
+
```text
|
|
172
|
+
metadata/<jobId>.json
|
|
173
|
+
manifest.json
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Partial metadata failures are recorded in `manifest.failures`; successful
|
|
177
|
+
metadata files are still archived and the command exits with code `1`.
|
|
178
|
+
|
|
179
|
+
### Download artifacts
|
|
180
|
+
|
|
181
|
+
Artifacts can be downloaded by Job, Group, or download task:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
bl artifacts download -j J123 -o ./out
|
|
185
|
+
bl artifacts download -g G1 -o ./out
|
|
186
|
+
bl artifacts download --task-id TASK_ID -o ./out
|
|
187
|
+
bl artifacts download -g G1 --include "*.tgz" --exclude "debug-*" -o ./out
|
|
188
|
+
bl artifacts download -g G1 --list-only
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
For Job and Group mode, the CLI still lists artifacts first via
|
|
192
|
+
`/openapi/jobs/{job_id}/artifacts`, but binary download is routed through the
|
|
193
|
+
backend proxy endpoint `/openapi/jobs/{job_id}/artifacts/download` instead of
|
|
194
|
+
directly using OSS presigned URLs. In task mode, every job from the task
|
|
195
|
+
snapshot is downloaded as `./out/TASK_ID/<jobId>.tgz`.
|
|
196
|
+
|
|
197
|
+
When an artifact is missing, the backend may return either HTTP 404 or an HTML
|
|
198
|
+
error page. The CLI treats both as a missing artifact and writes a valid empty
|
|
199
|
+
`.tgz` placeholder instead of saving the error page, so downstream batch
|
|
200
|
+
extraction can continue.
|
|
201
|
+
|
|
202
|
+
Artifact downloads show byte-level progress. On a TTY the progress is refreshed
|
|
203
|
+
in place; in CI or redirected stderr it falls back to periodic `[progress]`
|
|
204
|
+
lines. `--quiet` suppresses progress output.
|
|
205
|
+
|
|
206
|
+
### Submit a Behavior Reward Model (`brm`) task
|
|
207
|
+
|
|
208
|
+
`brm run` creates a Behavior analysis task for exactly one Group. The CLI first
|
|
209
|
+
calls `GET /openapi/behavior/steps?group_id=...` to detect whether the Group
|
|
210
|
+
has step sampling, then posts `POST /openapi/behavior/tasks`.
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
# No-step groups (all_jobs sampling): omit --steps
|
|
214
|
+
bl brm run -g G1
|
|
215
|
+
|
|
216
|
+
# Step groups: --steps is required (comma-separated or repeatable)
|
|
217
|
+
bl brm run -g G1 --steps 10,20
|
|
218
|
+
bl brm run -g G1 --steps 10 --steps 20 --sample-size 256 --scoring-limit 64
|
|
219
|
+
|
|
220
|
+
# Pipe the task id
|
|
221
|
+
bl brm run -g G1 --steps 10,20 | jq '.[0].uid'
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Defaults: `--sample-size 512` (step mode only), `--scoring-limit 32` (always
|
|
225
|
+
sent). In step mode, omitting `--steps` is a usage error and the available
|
|
226
|
+
steps are listed on stderr. Passing `--steps` on a no-step Group is also a
|
|
227
|
+
usage error.
|
|
228
|
+
|
|
229
|
+
On success, stderr logs a short confirmation and stdout prints a JSON array
|
|
230
|
+
(one object) that includes at least `uid`, plus `group_id` and
|
|
231
|
+
`samplingMode` (`step` or `all_jobs`). Use `.[0].uid` (not necessarily
|
|
232
|
+
`task_id`) for follow-up `brm status` / `brm get` calls.
|
|
233
|
+
|
|
234
|
+
### Query a Behavior Reward Model (`brm`) task
|
|
235
|
+
|
|
236
|
+
`brm get` queries task status and/or analysis by Behavior task uid (from
|
|
237
|
+
`brm run`). Multiple `--task-id` values are supported (repeatable or
|
|
238
|
+
comma-separated); stdout is always a JSON array, one element per id, matching
|
|
239
|
+
`metrics get -j ...`.
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
# Default --scope all (status + overall + detail); samplingMode only here
|
|
243
|
+
bl brm get --task-id UID
|
|
244
|
+
bl brm get --task-id UID --scope all | jq '.[0] | {status, overall, detail, samplingMode}'
|
|
245
|
+
|
|
246
|
+
# overall: analysis.summary + categoryScores
|
|
247
|
+
bl brm get --task-id UID --scope overall | jq '.[0] | {summary, categoryScores}'
|
|
248
|
+
|
|
249
|
+
# detail: analysis.behaviorPatterns
|
|
250
|
+
bl brm get --task-id UID --scope detail | jq '.[0].behaviorPatterns'
|
|
251
|
+
|
|
252
|
+
# status only; `brm status` is equivalent
|
|
253
|
+
bl brm get --task-id UID --scope status
|
|
254
|
+
bl brm status --task-id UID
|
|
255
|
+
bl brm status --task-id UID1,UID2
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
`--scope` values:
|
|
259
|
+
|
|
260
|
+
| scope | source | payload |
|
|
261
|
+
| --- | --- | --- |
|
|
262
|
+
| `overall` | `GET .../analysis` | `summary`, `categoryScores` |
|
|
263
|
+
| `detail` | `GET .../analysis` | `behaviorPatterns` |
|
|
264
|
+
| `status` | `GET .../tasks/{uid}` | task status / progress |
|
|
265
|
+
| `all` (default) | both endpoints | nested `status`, `overall`, `detail`, plus top-level `samplingMode` |
|
|
266
|
+
|
|
267
|
+
`brm status` always uses the status endpoint (same as `brm get --scope status`).
|
|
268
|
+
|
|
269
|
+
### List Behavior Reward Model (`brm`) tasks by Group
|
|
270
|
+
|
|
271
|
+
`brm list` calls `GET /openapi/behavior/tasks?group_id=...` and prints a JSON
|
|
272
|
+
array (one element per `-g`), matching `jobs get -g ...`:
|
|
273
|
+
|
|
274
|
+
```bash
|
|
275
|
+
bl brm list -g G1
|
|
276
|
+
bl brm list -g G1,G2
|
|
277
|
+
bl brm list -g G1 | jq '.[0].tasks[].uid'
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
Each element looks like `{ "group_id", "total", "tasks": [ ... ] }`. Task
|
|
281
|
+
objects keep the backend fields and also set `task_id` from `uid` when present.
|
|
282
|
+
|
|
283
|
+
## Exit codes
|
|
284
|
+
|
|
285
|
+
| code | meaning |
|
|
286
|
+
| ---- | ------- |
|
|
287
|
+
| 0 | all success |
|
|
288
|
+
| 1 | partial failure (some ids failed) |
|
|
289
|
+
| 2 | usage / parameter error |
|
|
290
|
+
| 3 | auth failure (HTTP 401) |
|
|
291
|
+
| 4 | network / timeout |
|
|
292
|
+
|
|
293
|
+
stdout carries data only; logs and progress go to stderr. `--quiet` suppresses
|
|
294
|
+
non-error logs, `--verbose` adds debug logs.
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
## Release new package
|
|
298
|
+
|
|
299
|
+
```
|
|
300
|
+
uv build
|
|
301
|
+
twine check dist/benchloop_client-*.whl
|
|
302
|
+
twine upload dist/benchloop_client-*.whl --verbose
|
|
303
|
+
```
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# benchloop-client (`bl`, `bmcli`)
|
|
2
|
+
|
|
3
|
+
Command line tool for fetching and downloading benchmark analysis data
|
|
4
|
+
(run metrics, trajectory, metadata, pre-analysis reports, task/job lists,
|
|
5
|
+
and artifacts) by `group_id` / `job_id` / download `task_id`, and for
|
|
6
|
+
submitting Behavior Reward Model (`brm`) analysis tasks.
|
|
7
|
+
|
|
8
|
+
> PyPI package name is `benchloop_client`; the installed commands are `bl` and `bmcli`.
|
|
9
|
+
|
|
10
|
+
## Install package
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
pip install benchloop_client
|
|
14
|
+
|
|
15
|
+
bl --api-key $BENCH_LOOP_API_KEY overview get -j $JOB_ID
|
|
16
|
+
bl --api-key $BENCH_LOOP_API_KEY metrics get -j $JOB_ID
|
|
17
|
+
|
|
18
|
+
# Option 2: config ~/.benchloop/config.toml by cli, and run bl without --api-key option:
|
|
19
|
+
bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
|
|
20
|
+
|
|
21
|
+
bl overview get -j $JOB_ID
|
|
22
|
+
bl metrics get -j $JOB_ID
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The CLI checks PyPI for a newer release in the background at most once every
|
|
27
|
+
24 hours. Update notices include the current/latest versions and the upgrade
|
|
28
|
+
command. Notices are written only to stderr, so JSON output remains pipeable.
|
|
29
|
+
|
|
30
|
+
## Install and run from source
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
# with uv (recommended)
|
|
34
|
+
uv sync
|
|
35
|
+
uv run bl --help
|
|
36
|
+
|
|
37
|
+
# or editable install with pip
|
|
38
|
+
python -m venv .venv && source .venv/bin/activate
|
|
39
|
+
pip install -e .
|
|
40
|
+
bl --help
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Auth & config
|
|
44
|
+
|
|
45
|
+
Authentication uses a personal api key sent as the `X-API-Key` header.
|
|
46
|
+
|
|
47
|
+
Precedence: CLI flag > env var > `~/.benchloop/config.toml` > default.
|
|
48
|
+
|
|
49
|
+
- Env: `BENCHLOOP_API_KEY` (or `BENCHMARK_API_KEY`), `BENCHLOOP_HOST`, `BENCHLOOP_OUTPUT_DIR`
|
|
50
|
+
- Default host (prod): `https://benchmark-analysis.alibaba-inc.com`
|
|
51
|
+
- Pre host: `https://pre-benchmark-analysis.alibaba-inc.com`
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
|
|
59
|
+
Two-segment `bl <resource> <action>` form. `get` prints a JSON array to
|
|
60
|
+
stdout (pipeable to `jq`); `download` writes the standard file layout.
|
|
61
|
+
Global options such as `--api-key`, `--host`, `--timeout`, `--retry`,
|
|
62
|
+
`--concurrency`, `--quiet`, and `--verbose` must be placed before the resource
|
|
63
|
+
name. Use `--concurrency 1` for a per-file byte progress bar, or a higher value
|
|
64
|
+
for concurrent requests/downloads.
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
bl metrics get -j J123 | jq '.[0]'
|
|
68
|
+
bl metrics download -g G1 --scope all -o ./out
|
|
69
|
+
bl --concurrency 10 trajectory download -g G1,G2 -o ./out
|
|
70
|
+
bl --concurrency 10 trajectory download --task-id TASK_ID -o ./out
|
|
71
|
+
bl metadata download --task-id TASK_ID -o ./out
|
|
72
|
+
bl report download -j J123 -o ./out
|
|
73
|
+
bl report get -g G123
|
|
74
|
+
bl report download -g G123 -o ./out
|
|
75
|
+
bl jobs get -g G1 --all # get supports --all (auto-paginate)
|
|
76
|
+
bl jobs download -g G1 # download is always full
|
|
77
|
+
bl artifacts download -g G1 -o ./out
|
|
78
|
+
bl artifacts download --task-id TASK_ID -o ./out
|
|
79
|
+
bl pull -g G1 --with metrics,trajectory,jobs,artifacts -o ./out
|
|
80
|
+
bl brm run -g G1
|
|
81
|
+
bl brm run -g G1 --steps 10,20 --sample-size 512 --scoring-limit 32
|
|
82
|
+
bl brm get --task-id UID
|
|
83
|
+
bl brm get --task-id UID --scope overall
|
|
84
|
+
bl brm get --task-id UID --scope detail
|
|
85
|
+
bl brm get --task-id UID --scope status
|
|
86
|
+
bl brm status --task-id UID
|
|
87
|
+
bl brm list -g G1
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Metrics and Job trajectory requests always explicitly use the Trace source. The
|
|
91
|
+
CLI does not expose a `trace`/`artifacts` source selector.
|
|
92
|
+
|
|
93
|
+
Group trajectory retrieval and downloads use the backend Trace export API. The
|
|
94
|
+
CLI waits for the export URL, downloads the gzip JSON-lines payload, and keeps
|
|
95
|
+
the existing `group_id`/`trajectories` JSON layout. The export wait timeout is
|
|
96
|
+
30 minutes by default; override it in seconds with
|
|
97
|
+
`--download-timeout SECONDS` on trajectory commands or `pull`.
|
|
98
|
+
|
|
99
|
+
Group downloads always auto-paginate and include every Job in the Group. In
|
|
100
|
+
particular, `trajectory`, detailed `metrics`, `report`, and `artifacts` enumerate
|
|
101
|
+
the complete Job list instead of relying on the backend Group endpoints that
|
|
102
|
+
return at most 1000 Jobs. Existing aggregate files keep their original paths:
|
|
103
|
+
|
|
104
|
+
```text
|
|
105
|
+
out/G1/trajectory.json
|
|
106
|
+
out/G1/detail_metrics.json
|
|
107
|
+
out/G1_report.md
|
|
108
|
+
out/G1/J123_report.md
|
|
109
|
+
out/G1/J123/<artifact-name>
|
|
110
|
+
out/TASK_ID_trajectories.zip
|
|
111
|
+
out/TASK_ID_metadata.zip
|
|
112
|
+
out/TASK_ID/J123.tgz
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`report download -g` writes both the Group summary report and each Job report.
|
|
116
|
+
`pull -g` uses the same complete download behavior for every selected resource.
|
|
117
|
+
Use `--skip-existing` to resume file-based downloads after an interruption.
|
|
118
|
+
|
|
119
|
+
IDs accept repeated flags, comma-separated values, and `-` to read from stdin.
|
|
120
|
+
`-g` and `-j` are mutually exclusive in a single command. For trajectory and
|
|
121
|
+
artifact downloads, `--task-id` is a third mutually exclusive input mode.
|
|
122
|
+
|
|
123
|
+
### Download trajectories by task ID
|
|
124
|
+
|
|
125
|
+
The benchmark analysis Jobs page can turn the current filters into a trajectory
|
|
126
|
+
download task. Copy the command shown by the page and run it locally:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
bl trajectory download --task-id TASK_ID -o ./out
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The CLI fetches the task's complete job ID list, downloads job trajectories with
|
|
133
|
+
fixed concurrency, and prints count/ETA progress to stderr. It creates
|
|
134
|
+
`./out/TASK_ID_trajectories.zip` with this layout:
|
|
135
|
+
|
|
136
|
+
```text
|
|
137
|
+
trajectories/<jobId>_trajectory.json
|
|
138
|
+
metadata/<jobId>.json
|
|
139
|
+
manifest.json
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Metadata is fetched from `/openapi/jobs/{job_id}/meta` and archived alongside
|
|
143
|
+
the trajectory. If trajectory download succeeds but metadata fails, the
|
|
144
|
+
trajectory is still archived, the metadata error is recorded in
|
|
145
|
+
`manifest.metadata_failures`, and the command exits with code `1`. If only some
|
|
146
|
+
jobs fail, the successful files are still archived. If every job fails, no empty
|
|
147
|
+
ZIP is created.
|
|
148
|
+
|
|
149
|
+
### Download metadata by task ID
|
|
150
|
+
|
|
151
|
+
Use `metadata download` when you only need per-job metadata for a download task:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
bl metadata download --task-id TASK_ID -o ./out
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The CLI creates `./out/TASK_ID_metadata.zip` with this layout:
|
|
158
|
+
|
|
159
|
+
```text
|
|
160
|
+
metadata/<jobId>.json
|
|
161
|
+
manifest.json
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Partial metadata failures are recorded in `manifest.failures`; successful
|
|
165
|
+
metadata files are still archived and the command exits with code `1`.
|
|
166
|
+
|
|
167
|
+
### Download artifacts
|
|
168
|
+
|
|
169
|
+
Artifacts can be downloaded by Job, Group, or download task:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
bl artifacts download -j J123 -o ./out
|
|
173
|
+
bl artifacts download -g G1 -o ./out
|
|
174
|
+
bl artifacts download --task-id TASK_ID -o ./out
|
|
175
|
+
bl artifacts download -g G1 --include "*.tgz" --exclude "debug-*" -o ./out
|
|
176
|
+
bl artifacts download -g G1 --list-only
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
For Job and Group mode, the CLI still lists artifacts first via
|
|
180
|
+
`/openapi/jobs/{job_id}/artifacts`, but binary download is routed through the
|
|
181
|
+
backend proxy endpoint `/openapi/jobs/{job_id}/artifacts/download` instead of
|
|
182
|
+
directly using OSS presigned URLs. In task mode, every job from the task
|
|
183
|
+
snapshot is downloaded as `./out/TASK_ID/<jobId>.tgz`.
|
|
184
|
+
|
|
185
|
+
When an artifact is missing, the backend may return either HTTP 404 or an HTML
|
|
186
|
+
error page. The CLI treats both as a missing artifact and writes a valid empty
|
|
187
|
+
`.tgz` placeholder instead of saving the error page, so downstream batch
|
|
188
|
+
extraction can continue.
|
|
189
|
+
|
|
190
|
+
Artifact downloads show byte-level progress. On a TTY the progress is refreshed
|
|
191
|
+
in place; in CI or redirected stderr it falls back to periodic `[progress]`
|
|
192
|
+
lines. `--quiet` suppresses progress output.
|
|
193
|
+
|
|
194
|
+
### Submit a Behavior Reward Model (`brm`) task
|
|
195
|
+
|
|
196
|
+
`brm run` creates a Behavior analysis task for exactly one Group. The CLI first
|
|
197
|
+
calls `GET /openapi/behavior/steps?group_id=...` to detect whether the Group
|
|
198
|
+
has step sampling, then posts `POST /openapi/behavior/tasks`.
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
# No-step groups (all_jobs sampling): omit --steps
|
|
202
|
+
bl brm run -g G1
|
|
203
|
+
|
|
204
|
+
# Step groups: --steps is required (comma-separated or repeatable)
|
|
205
|
+
bl brm run -g G1 --steps 10,20
|
|
206
|
+
bl brm run -g G1 --steps 10 --steps 20 --sample-size 256 --scoring-limit 64
|
|
207
|
+
|
|
208
|
+
# Pipe the task id
|
|
209
|
+
bl brm run -g G1 --steps 10,20 | jq '.[0].uid'
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Defaults: `--sample-size 512` (step mode only), `--scoring-limit 32` (always
|
|
213
|
+
sent). In step mode, omitting `--steps` is a usage error and the available
|
|
214
|
+
steps are listed on stderr. Passing `--steps` on a no-step Group is also a
|
|
215
|
+
usage error.
|
|
216
|
+
|
|
217
|
+
On success, stderr logs a short confirmation and stdout prints a JSON array
|
|
218
|
+
(one object) that includes at least `uid`, plus `group_id` and
|
|
219
|
+
`samplingMode` (`step` or `all_jobs`). Use `.[0].uid` (not necessarily
|
|
220
|
+
`task_id`) for follow-up `brm status` / `brm get` calls.
|
|
221
|
+
|
|
222
|
+
### Query a Behavior Reward Model (`brm`) task
|
|
223
|
+
|
|
224
|
+
`brm get` queries task status and/or analysis by Behavior task uid (from
|
|
225
|
+
`brm run`). Multiple `--task-id` values are supported (repeatable or
|
|
226
|
+
comma-separated); stdout is always a JSON array, one element per id, matching
|
|
227
|
+
`metrics get -j ...`.
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
# Default --scope all (status + overall + detail); samplingMode only here
|
|
231
|
+
bl brm get --task-id UID
|
|
232
|
+
bl brm get --task-id UID --scope all | jq '.[0] | {status, overall, detail, samplingMode}'
|
|
233
|
+
|
|
234
|
+
# overall: analysis.summary + categoryScores
|
|
235
|
+
bl brm get --task-id UID --scope overall | jq '.[0] | {summary, categoryScores}'
|
|
236
|
+
|
|
237
|
+
# detail: analysis.behaviorPatterns
|
|
238
|
+
bl brm get --task-id UID --scope detail | jq '.[0].behaviorPatterns'
|
|
239
|
+
|
|
240
|
+
# status only; `brm status` is equivalent
|
|
241
|
+
bl brm get --task-id UID --scope status
|
|
242
|
+
bl brm status --task-id UID
|
|
243
|
+
bl brm status --task-id UID1,UID2
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
`--scope` values:
|
|
247
|
+
|
|
248
|
+
| scope | source | payload |
|
|
249
|
+
| --- | --- | --- |
|
|
250
|
+
| `overall` | `GET .../analysis` | `summary`, `categoryScores` |
|
|
251
|
+
| `detail` | `GET .../analysis` | `behaviorPatterns` |
|
|
252
|
+
| `status` | `GET .../tasks/{uid}` | task status / progress |
|
|
253
|
+
| `all` (default) | both endpoints | nested `status`, `overall`, `detail`, plus top-level `samplingMode` |
|
|
254
|
+
|
|
255
|
+
`brm status` always uses the status endpoint (same as `brm get --scope status`).
|
|
256
|
+
|
|
257
|
+
### List Behavior Reward Model (`brm`) tasks by Group
|
|
258
|
+
|
|
259
|
+
`brm list` calls `GET /openapi/behavior/tasks?group_id=...` and prints a JSON
|
|
260
|
+
array (one element per `-g`), matching `jobs get -g ...`:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
bl brm list -g G1
|
|
264
|
+
bl brm list -g G1,G2
|
|
265
|
+
bl brm list -g G1 | jq '.[0].tasks[].uid'
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Each element looks like `{ "group_id", "total", "tasks": [ ... ] }`. Task
|
|
269
|
+
objects keep the backend fields and also set `task_id` from `uid` when present.
|
|
270
|
+
|
|
271
|
+
## Exit codes
|
|
272
|
+
|
|
273
|
+
| code | meaning |
|
|
274
|
+
| ---- | ------- |
|
|
275
|
+
| 0 | all success |
|
|
276
|
+
| 1 | partial failure (some ids failed) |
|
|
277
|
+
| 2 | usage / parameter error |
|
|
278
|
+
| 3 | auth failure (HTTP 401) |
|
|
279
|
+
| 4 | network / timeout |
|
|
280
|
+
|
|
281
|
+
stdout carries data only; logs and progress go to stderr. `--quiet` suppresses
|
|
282
|
+
non-error logs, `--verbose` adds debug logs.
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
## Release new package
|
|
286
|
+
|
|
287
|
+
```
|
|
288
|
+
uv build
|
|
289
|
+
twine check dist/benchloop_client-*.whl
|
|
290
|
+
twine upload dist/benchloop_client-*.whl --verbose
|
|
291
|
+
```
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "benchloop_client"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.24"
|
|
8
8
|
description = "Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -10,6 +10,7 @@ from aliyun.bmcli import __version__, output, version_check
|
|
|
10
10
|
from aliyun.bmcli.commands import (
|
|
11
11
|
ap_cmd as ap_cmd_mod,
|
|
12
12
|
artifacts as artifacts_cmd,
|
|
13
|
+
brm as brm_cmd,
|
|
13
14
|
jobs as jobs_cmd,
|
|
14
15
|
metadata as metadata_cmd,
|
|
15
16
|
metrics as metrics_cmd,
|
|
@@ -137,6 +138,7 @@ app.add_typer(tasks_cmd.app, name="tasks")
|
|
|
137
138
|
app.add_typer(jobs_cmd.app, name="jobs")
|
|
138
139
|
app.add_typer(artifacts_cmd.app, name="artifacts")
|
|
139
140
|
app.add_typer(metadata_cmd.app, name="metadata")
|
|
141
|
+
app.add_typer(brm_cmd.app, name="brm")
|
|
140
142
|
app.command(
|
|
141
143
|
"ap-cmd",
|
|
142
144
|
context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
|
|
@@ -31,6 +31,12 @@ OutputDirOption = typer.Option(
|
|
|
31
31
|
SkipExistingOption = typer.Option(
|
|
32
32
|
False, "--skip-existing", help="Skip files that already exist on disk."
|
|
33
33
|
)
|
|
34
|
+
DownloadTimeoutOption = typer.Option(
|
|
35
|
+
1800,
|
|
36
|
+
"--download-timeout",
|
|
37
|
+
min=1,
|
|
38
|
+
help="Maximum seconds to wait for a Group Trace trajectory export.",
|
|
39
|
+
)
|
|
34
40
|
|
|
35
41
|
|
|
36
42
|
def expand(values: Optional[list[str]]) -> list[str]:
|