benchloop-client 0.2.22__tar.gz → 0.2.24__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. benchloop_client-0.2.24/PKG-INFO +303 -0
  2. benchloop_client-0.2.24/README.md +291 -0
  3. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/pyproject.toml +1 -1
  4. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/cli.py +2 -0
  5. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/_common.py +6 -0
  6. benchloop_client-0.2.24/src/aliyun/bmcli/commands/brm.py +440 -0
  7. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/metrics.py +14 -3
  8. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/pull.py +8 -3
  9. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/trajectory.py +246 -56
  10. benchloop_client-0.2.24/tests/test_cmd_brm.py +451 -0
  11. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_metrics.py +32 -2
  12. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_pull.py +80 -0
  13. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_trajectory.py +253 -42
  14. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/uv.lock +89 -89
  15. benchloop_client-0.2.22/PKG-INFO +0 -144
  16. benchloop_client-0.2.22/README.md +0 -132
  17. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/.gitignore +0 -0
  18. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/.python-version +0 -0
  19. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/RELEASE.md +0 -0
  20. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/benchloop-cli-design.md +0 -0
  21. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/bmcli_examples.sh +0 -0
  22. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/__init__.py +0 -0
  23. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/__main__.py +0 -0
  24. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/client.py +0 -0
  25. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/__init__.py +0 -0
  26. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/ap_cmd.py +0 -0
  27. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/artifacts.py +0 -0
  28. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/group_data.py +0 -0
  29. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/jobs.py +0 -0
  30. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/metadata.py +0 -0
  31. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/overview.py +0 -0
  32. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/report.py +0 -0
  33. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/commands/tasks.py +0 -0
  34. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/config.py +0 -0
  35. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/exit_codes.py +0 -0
  36. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/ids.py +0 -0
  37. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/output.py +0 -0
  38. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/runtime.py +0 -0
  39. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/src/aliyun/bmcli/version_check.py +0 -0
  40. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/__init__.py +0 -0
  41. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/conftest.py +0 -0
  42. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cli_options.py +0 -0
  43. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_client.py +0 -0
  44. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_ap_cmd.py +0 -0
  45. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_artifacts.py +0 -0
  46. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_metadata.py +0 -0
  47. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_overview.py +0 -0
  48. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_report.py +0 -0
  49. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_cmd_tasks_jobs.py +0 -0
  50. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_config.py +0 -0
  51. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_exit_code_behavior.py +0 -0
  52. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_exit_codes.py +0 -0
  53. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_group_data.py +0 -0
  54. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_ids.py +0 -0
  55. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_runtime.py +0 -0
  56. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_transfer_progress.py +0 -0
  57. {benchloop_client-0.2.22 → benchloop_client-0.2.24}/tests/test_version_check.py +0 -0
@@ -0,0 +1,303 @@
1
+ Metadata-Version: 2.5
2
+ Name: benchloop_client
3
+ Version: 0.2.24
4
+ Summary: Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts).
5
+ Requires-Python: >=3.10
6
+ Requires-Dist: httpx>=0.25.2
7
+ Requires-Dist: tomli>=1.1.0; python_version < '3.13'
8
+ Requires-Dist: typer>=0.9.4
9
+ Provides-Extra: dev
10
+ Requires-Dist: pytest>=8.0; extra == 'dev'
11
+ Description-Content-Type: text/markdown
12
+
13
+ # benchloop-client (`bl`, `bmcli`)
14
+
15
+ Command line tool for fetching and downloading benchmark analysis data
16
+ (run metrics, trajectory, metadata, pre-analysis reports, task/job lists,
17
+ and artifacts) by `group_id` / `job_id` / download `task_id`, and for
18
+ submitting Behavior Reward Model (`brm`) analysis tasks.
19
+
20
+ > PyPI package name is `benchloop_client`; the installed commands are `bl` and `bmcli`.
21
+
22
+ ## Install package
23
+
24
+ ```
25
+ pip install benchloop_client
26
+
27
+ bl --api-key $BENCH_LOOP_API_KEY overview get -j $JOB_ID
28
+ bl --api-key $BENCH_LOOP_API_KEY metrics get -j $JOB_ID
29
+
30
+ # Option 2: config ~/.benchloop/config.toml by cli, and run bl without --api-key option:
31
+ bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
32
+
33
+ bl overview get -j $JOB_ID
34
+ bl metrics get -j $JOB_ID
35
+
36
+ ```
37
+
38
+ The CLI checks PyPI for a newer release in the background at most once every
39
+ 24 hours. Update notices include the current/latest versions and the upgrade
40
+ command. Notices are written only to stderr, so JSON output remains pipeable.
41
+
42
+ ## Install and run from source
43
+
44
+ ```bash
45
+ # with uv (recommended)
46
+ uv sync
47
+ uv run bl --help
48
+
49
+ # or editable install with pip
50
+ python -m venv .venv && source .venv/bin/activate
51
+ pip install -e .
52
+ bl --help
53
+ ```
54
+
55
+ ## Auth & config
56
+
57
+ Authentication uses a personal api key sent as the `X-API-Key` header.
58
+
59
+ Precedence: CLI flag > env var > `~/.benchloop/config.toml` > default.
60
+
61
+ - Env: `BENCHLOOP_API_KEY` (or `BENCHMARK_API_KEY`), `BENCHLOOP_HOST`, `BENCHLOOP_OUTPUT_DIR`
62
+ - Default host (prod): `https://benchmark-analysis.alibaba-inc.com`
63
+ - Pre host: `https://pre-benchmark-analysis.alibaba-inc.com`
64
+
65
+ ```bash
66
+ bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
67
+ ```
68
+
69
+ ## Usage
70
+
71
+ Two-segment `bl <resource> <action>` form. `get` prints a JSON array to
72
+ stdout (pipeable to `jq`); `download` writes the standard file layout.
73
+ Global options such as `--api-key`, `--host`, `--timeout`, `--retry`,
74
+ `--concurrency`, `--quiet`, and `--verbose` must be placed before the resource
75
+ name. Use `--concurrency 1` for a per-file byte progress bar, or a higher value
76
+ for concurrent requests/downloads.
77
+
78
+ ```bash
79
+ bl metrics get -j J123 | jq '.[0]'
80
+ bl metrics download -g G1 --scope all -o ./out
81
+ bl --concurrency 10 trajectory download -g G1,G2 -o ./out
82
+ bl --concurrency 10 trajectory download --task-id TASK_ID -o ./out
83
+ bl metadata download --task-id TASK_ID -o ./out
84
+ bl report download -j J123 -o ./out
85
+ bl report get -g G123
86
+ bl report download -g G123 -o ./out
87
+ bl jobs get -g G1 --all # get supports --all (auto-paginate)
88
+ bl jobs download -g G1 # download is always full
89
+ bl artifacts download -g G1 -o ./out
90
+ bl artifacts download --task-id TASK_ID -o ./out
91
+ bl pull -g G1 --with metrics,trajectory,jobs,artifacts -o ./out
92
+ bl brm run -g G1
93
+ bl brm run -g G1 --steps 10,20 --sample-size 512 --scoring-limit 32
94
+ bl brm get --task-id UID
95
+ bl brm get --task-id UID --scope overall
96
+ bl brm get --task-id UID --scope detail
97
+ bl brm get --task-id UID --scope status
98
+ bl brm status --task-id UID
99
+ bl brm list -g G1
100
+ ```
101
+
102
+ Metrics and Job trajectory requests always explicitly use the Trace source. The
103
+ CLI does not expose a `trace`/`artifacts` source selector.
104
+
105
+ Group trajectory retrieval and downloads use the backend Trace export API. The
106
+ CLI waits for the export URL, downloads the gzip JSON-lines payload, and keeps
107
+ the existing `group_id`/`trajectories` JSON layout. The export wait timeout is
108
+ 30 minutes by default; override it in seconds with
109
+ `--download-timeout SECONDS` on trajectory commands or `pull`.
110
+
111
+ Group downloads always auto-paginate and include every Job in the Group. In
112
+ particular, `trajectory`, detailed `metrics`, `report`, and `artifacts` enumerate
113
+ the complete Job list instead of relying on the backend Group endpoints that
114
+ return at most 1000 Jobs. Existing aggregate files keep their original paths:
115
+
116
+ ```text
117
+ out/G1/trajectory.json
118
+ out/G1/detail_metrics.json
119
+ out/G1_report.md
120
+ out/G1/J123_report.md
121
+ out/G1/J123/<artifact-name>
122
+ out/TASK_ID_trajectories.zip
123
+ out/TASK_ID_metadata.zip
124
+ out/TASK_ID/J123.tgz
125
+ ```
126
+
127
+ `report download -g` writes both the Group summary report and each Job report.
128
+ `pull -g` uses the same complete download behavior for every selected resource.
129
+ Use `--skip-existing` to resume file-based downloads after an interruption.
130
+
131
+ IDs accept repeated flags, comma-separated values, and `-` to read from stdin.
132
+ `-g` and `-j` are mutually exclusive in a single command. For trajectory and
133
+ artifact downloads, `--task-id` is a third mutually exclusive input mode.
134
+
135
+ ### Download trajectories by task ID
136
+
137
+ The benchmark analysis Jobs page can turn the current filters into a trajectory
138
+ download task. Copy the command shown by the page and run it locally:
139
+
140
+ ```bash
141
+ bl trajectory download --task-id TASK_ID -o ./out
142
+ ```
143
+
144
+ The CLI fetches the task's complete job ID list, downloads job trajectories with
145
+ fixed concurrency, and prints count/ETA progress to stderr. It creates
146
+ `./out/TASK_ID_trajectories.zip` with this layout:
147
+
148
+ ```text
149
+ trajectories/<jobId>_trajectory.json
150
+ metadata/<jobId>.json
151
+ manifest.json
152
+ ```
153
+
154
+ Metadata is fetched from `/openapi/jobs/{job_id}/meta` and archived alongside
155
+ the trajectory. If trajectory download succeeds but metadata fails, the
156
+ trajectory is still archived, the metadata error is recorded in
157
+ `manifest.metadata_failures`, and the command exits with code `1`. If only some
158
+ jobs fail, the successful files are still archived. If every job fails, no empty
159
+ ZIP is created.
160
+
161
+ ### Download metadata by task ID
162
+
163
+ Use `metadata download` when you only need per-job metadata for a download task:
164
+
165
+ ```bash
166
+ bl metadata download --task-id TASK_ID -o ./out
167
+ ```
168
+
169
+ The CLI creates `./out/TASK_ID_metadata.zip` with this layout:
170
+
171
+ ```text
172
+ metadata/<jobId>.json
173
+ manifest.json
174
+ ```
175
+
176
+ Partial metadata failures are recorded in `manifest.failures`; successful
177
+ metadata files are still archived and the command exits with code `1`.
178
+
179
+ ### Download artifacts
180
+
181
+ Artifacts can be downloaded by Job, Group, or download task:
182
+
183
+ ```bash
184
+ bl artifacts download -j J123 -o ./out
185
+ bl artifacts download -g G1 -o ./out
186
+ bl artifacts download --task-id TASK_ID -o ./out
187
+ bl artifacts download -g G1 --include "*.tgz" --exclude "debug-*" -o ./out
188
+ bl artifacts download -g G1 --list-only
189
+ ```
190
+
191
+ For Job and Group mode, the CLI still lists artifacts first via
192
+ `/openapi/jobs/{job_id}/artifacts`, but binary download is routed through the
193
+ backend proxy endpoint `/openapi/jobs/{job_id}/artifacts/download` instead of
194
+ directly using OSS presigned URLs. In task mode, every job from the task
195
+ snapshot is downloaded as `./out/TASK_ID/<jobId>.tgz`.
196
+
197
+ When an artifact is missing, the backend may return either HTTP 404 or an HTML
198
+ error page. The CLI treats both as a missing artifact and writes a valid empty
199
+ `.tgz` placeholder instead of saving the error page, so downstream batch
200
+ extraction can continue.
201
+
202
+ Artifact downloads show byte-level progress. On a TTY the progress is refreshed
203
+ in place; in CI or redirected stderr it falls back to periodic `[progress]`
204
+ lines. `--quiet` suppresses progress output.
205
+
206
+ ### Submit a Behavior Reward Model (`brm`) task
207
+
208
+ `brm run` creates a Behavior analysis task for exactly one Group. The CLI first
209
+ calls `GET /openapi/behavior/steps?group_id=...` to detect whether the Group
210
+ has step sampling, then posts `POST /openapi/behavior/tasks`.
211
+
212
+ ```bash
213
+ # No-step groups (all_jobs sampling): omit --steps
214
+ bl brm run -g G1
215
+
216
+ # Step groups: --steps is required (comma-separated or repeatable)
217
+ bl brm run -g G1 --steps 10,20
218
+ bl brm run -g G1 --steps 10 --steps 20 --sample-size 256 --scoring-limit 64
219
+
220
+ # Pipe the task id
221
+ bl brm run -g G1 --steps 10,20 | jq '.[0].uid'
222
+ ```
223
+
224
+ Defaults: `--sample-size 512` (step mode only), `--scoring-limit 32` (always
225
+ sent). In step mode, omitting `--steps` is a usage error and the available
226
+ steps are listed on stderr. Passing `--steps` on a no-step Group is also a
227
+ usage error.
228
+
229
+ On success, stderr logs a short confirmation and stdout prints a JSON array
230
+ (one object) that includes at least `uid`, plus `group_id` and
231
+ `samplingMode` (`step` or `all_jobs`). Use `.[0].uid` (not necessarily
232
+ `task_id`) for follow-up `brm status` / `brm get` calls.
233
+
234
+ ### Query a Behavior Reward Model (`brm`) task
235
+
236
+ `brm get` queries task status and/or analysis by Behavior task uid (from
237
+ `brm run`). Multiple `--task-id` values are supported (repeatable or
238
+ comma-separated); stdout is always a JSON array, one element per id, matching
239
+ `metrics get -j ...`.
240
+
241
+ ```bash
242
+ # Default --scope all (status + overall + detail); samplingMode only here
243
+ bl brm get --task-id UID
244
+ bl brm get --task-id UID --scope all | jq '.[0] | {status, overall, detail, samplingMode}'
245
+
246
+ # overall: analysis.summary + categoryScores
247
+ bl brm get --task-id UID --scope overall | jq '.[0] | {summary, categoryScores}'
248
+
249
+ # detail: analysis.behaviorPatterns
250
+ bl brm get --task-id UID --scope detail | jq '.[0].behaviorPatterns'
251
+
252
+ # status only; `brm status` is equivalent
253
+ bl brm get --task-id UID --scope status
254
+ bl brm status --task-id UID
255
+ bl brm status --task-id UID1,UID2
256
+ ```
257
+
258
+ `--scope` values:
259
+
260
+ | scope | source | payload |
261
+ | --- | --- | --- |
262
+ | `overall` | `GET .../analysis` | `summary`, `categoryScores` |
263
+ | `detail` | `GET .../analysis` | `behaviorPatterns` |
264
+ | `status` | `GET .../tasks/{uid}` | task status / progress |
265
+ | `all` (default) | both endpoints | nested `status`, `overall`, `detail`, plus top-level `samplingMode` |
266
+
267
+ `brm status` always uses the status endpoint (same as `brm get --scope status`).
268
+
269
+ ### List Behavior Reward Model (`brm`) tasks by Group
270
+
271
+ `brm list` calls `GET /openapi/behavior/tasks?group_id=...` and prints a JSON
272
+ array (one element per `-g`), matching `jobs get -g ...`:
273
+
274
+ ```bash
275
+ bl brm list -g G1
276
+ bl brm list -g G1,G2
277
+ bl brm list -g G1 | jq '.[0].tasks[].uid'
278
+ ```
279
+
280
+ Each element looks like `{ "group_id", "total", "tasks": [ ... ] }`. Task
281
+ objects keep the backend fields and also set `task_id` from `uid` when present.
282
+
283
+ ## Exit codes
284
+
285
+ | code | meaning |
286
+ | ---- | ------- |
287
+ | 0 | all success |
288
+ | 1 | partial failure (some ids failed) |
289
+ | 2 | usage / parameter error |
290
+ | 3 | auth failure (HTTP 401) |
291
+ | 4 | network / timeout |
292
+
293
+ stdout carries data only; logs and progress go to stderr. `--quiet` suppresses
294
+ non-error logs, `--verbose` adds debug logs.
295
+
296
+
297
+ ## Release new package
298
+
299
+ ```
300
+ uv build
301
+ twine check dist/benchloop_client-*.whl
302
+ twine upload dist/benchloop_client-*.whl --verbose
303
+ ```
@@ -0,0 +1,291 @@
1
+ # benchloop-client (`bl`, `bmcli`)
2
+
3
+ Command line tool for fetching and downloading benchmark analysis data
4
+ (run metrics, trajectory, metadata, pre-analysis reports, task/job lists,
5
+ and artifacts) by `group_id` / `job_id` / download `task_id`, and for
6
+ submitting Behavior Reward Model (`brm`) analysis tasks.
7
+
8
+ > PyPI package name is `benchloop_client`; the installed commands are `bl` and `bmcli`.
9
+
10
+ ## Install package
11
+
12
+ ```
13
+ pip install benchloop_client
14
+
15
+ bl --api-key $BENCH_LOOP_API_KEY overview get -j $JOB_ID
16
+ bl --api-key $BENCH_LOOP_API_KEY metrics get -j $JOB_ID
17
+
18
+ # Option 2: config ~/.benchloop/config.toml by cli, and run bl without --api-key option:
19
+ bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
20
+
21
+ bl overview get -j $JOB_ID
22
+ bl metrics get -j $JOB_ID
23
+
24
+ ```
25
+
26
+ The CLI checks PyPI for a newer release in the background at most once every
27
+ 24 hours. Update notices include the current/latest versions and the upgrade
28
+ command. Notices are written only to stderr, so JSON output remains pipeable.
29
+
30
+ ## Install and run from source
31
+
32
+ ```bash
33
+ # with uv (recommended)
34
+ uv sync
35
+ uv run bl --help
36
+
37
+ # or editable install with pip
38
+ python -m venv .venv && source .venv/bin/activate
39
+ pip install -e .
40
+ bl --help
41
+ ```
42
+
43
+ ## Auth & config
44
+
45
+ Authentication uses a personal api key sent as the `X-API-Key` header.
46
+
47
+ Precedence: CLI flag > env var > `~/.benchloop/config.toml` > default.
48
+
49
+ - Env: `BENCHLOOP_API_KEY` (or `BENCHMARK_API_KEY`), `BENCHLOOP_HOST`, `BENCHLOOP_OUTPUT_DIR`
50
+ - Default host (prod): `https://benchmark-analysis.alibaba-inc.com`
51
+ - Pre host: `https://pre-benchmark-analysis.alibaba-inc.com`
52
+
53
+ ```bash
54
+ bl config set --api-key $BENCHLOOP_API_KEY --host https://benchmark-analysis.alibaba-inc.com
55
+ ```
56
+
57
+ ## Usage
58
+
59
+ Two-segment `bl <resource> <action>` form. `get` prints a JSON array to
60
+ stdout (pipeable to `jq`); `download` writes the standard file layout.
61
+ Global options such as `--api-key`, `--host`, `--timeout`, `--retry`,
62
+ `--concurrency`, `--quiet`, and `--verbose` must be placed before the resource
63
+ name. Use `--concurrency 1` for a per-file byte progress bar, or a higher value
64
+ for concurrent requests/downloads.
65
+
66
+ ```bash
67
+ bl metrics get -j J123 | jq '.[0]'
68
+ bl metrics download -g G1 --scope all -o ./out
69
+ bl --concurrency 10 trajectory download -g G1,G2 -o ./out
70
+ bl --concurrency 10 trajectory download --task-id TASK_ID -o ./out
71
+ bl metadata download --task-id TASK_ID -o ./out
72
+ bl report download -j J123 -o ./out
73
+ bl report get -g G123
74
+ bl report download -g G123 -o ./out
75
+ bl jobs get -g G1 --all # get supports --all (auto-paginate)
76
+ bl jobs download -g G1 # download is always full
77
+ bl artifacts download -g G1 -o ./out
78
+ bl artifacts download --task-id TASK_ID -o ./out
79
+ bl pull -g G1 --with metrics,trajectory,jobs,artifacts -o ./out
80
+ bl brm run -g G1
81
+ bl brm run -g G1 --steps 10,20 --sample-size 512 --scoring-limit 32
82
+ bl brm get --task-id UID
83
+ bl brm get --task-id UID --scope overall
84
+ bl brm get --task-id UID --scope detail
85
+ bl brm get --task-id UID --scope status
86
+ bl brm status --task-id UID
87
+ bl brm list -g G1
88
+ ```
89
+
90
+ Metrics and Job trajectory requests always explicitly use the Trace source. The
91
+ CLI does not expose a `trace`/`artifacts` source selector.
92
+
93
+ Group trajectory retrieval and downloads use the backend Trace export API. The
94
+ CLI waits for the export URL, downloads the gzip JSON-lines payload, and keeps
95
+ the existing `group_id`/`trajectories` JSON layout. The export wait timeout is
96
+ 30 minutes by default; override it in seconds with
97
+ `--download-timeout SECONDS` on trajectory commands or `pull`.
98
+
99
+ Group downloads always auto-paginate and include every Job in the Group. In
100
+ particular, `trajectory`, detailed `metrics`, `report`, and `artifacts` enumerate
101
+ the complete Job list instead of relying on the backend Group endpoints that
102
+ return at most 1000 Jobs. Existing aggregate files keep their original paths:
103
+
104
+ ```text
105
+ out/G1/trajectory.json
106
+ out/G1/detail_metrics.json
107
+ out/G1_report.md
108
+ out/G1/J123_report.md
109
+ out/G1/J123/<artifact-name>
110
+ out/TASK_ID_trajectories.zip
111
+ out/TASK_ID_metadata.zip
112
+ out/TASK_ID/J123.tgz
113
+ ```
114
+
115
+ `report download -g` writes both the Group summary report and each Job report.
116
+ `pull -g` uses the same complete download behavior for every selected resource.
117
+ Use `--skip-existing` to resume file-based downloads after an interruption.
118
+
119
+ IDs accept repeated flags, comma-separated values, and `-` to read from stdin.
120
+ `-g` and `-j` are mutually exclusive in a single command. For trajectory and
121
+ artifact downloads, `--task-id` is a third mutually exclusive input mode.
122
+
123
+ ### Download trajectories by task ID
124
+
125
+ The benchmark analysis Jobs page can turn the current filters into a trajectory
126
+ download task. Copy the command shown by the page and run it locally:
127
+
128
+ ```bash
129
+ bl trajectory download --task-id TASK_ID -o ./out
130
+ ```
131
+
132
+ The CLI fetches the task's complete job ID list, downloads job trajectories with
133
+ fixed concurrency, and prints count/ETA progress to stderr. It creates
134
+ `./out/TASK_ID_trajectories.zip` with this layout:
135
+
136
+ ```text
137
+ trajectories/<jobId>_trajectory.json
138
+ metadata/<jobId>.json
139
+ manifest.json
140
+ ```
141
+
142
+ Metadata is fetched from `/openapi/jobs/{job_id}/meta` and archived alongside
143
+ the trajectory. If trajectory download succeeds but metadata fails, the
144
+ trajectory is still archived, the metadata error is recorded in
145
+ `manifest.metadata_failures`, and the command exits with code `1`. If only some
146
+ jobs fail, the successful files are still archived. If every job fails, no empty
147
+ ZIP is created.
148
+
149
+ ### Download metadata by task ID
150
+
151
+ Use `metadata download` when you only need per-job metadata for a download task:
152
+
153
+ ```bash
154
+ bl metadata download --task-id TASK_ID -o ./out
155
+ ```
156
+
157
+ The CLI creates `./out/TASK_ID_metadata.zip` with this layout:
158
+
159
+ ```text
160
+ metadata/<jobId>.json
161
+ manifest.json
162
+ ```
163
+
164
+ Partial metadata failures are recorded in `manifest.failures`; successful
165
+ metadata files are still archived and the command exits with code `1`.
166
+
167
+ ### Download artifacts
168
+
169
+ Artifacts can be downloaded by Job, Group, or download task:
170
+
171
+ ```bash
172
+ bl artifacts download -j J123 -o ./out
173
+ bl artifacts download -g G1 -o ./out
174
+ bl artifacts download --task-id TASK_ID -o ./out
175
+ bl artifacts download -g G1 --include "*.tgz" --exclude "debug-*" -o ./out
176
+ bl artifacts download -g G1 --list-only
177
+ ```
178
+
179
+ For Job and Group mode, the CLI still lists artifacts first via
180
+ `/openapi/jobs/{job_id}/artifacts`, but binary download is routed through the
181
+ backend proxy endpoint `/openapi/jobs/{job_id}/artifacts/download` instead of
182
+ directly using OSS presigned URLs. In task mode, every job from the task
183
+ snapshot is downloaded as `./out/TASK_ID/<jobId>.tgz`.
184
+
185
+ When an artifact is missing, the backend may return either HTTP 404 or an HTML
186
+ error page. The CLI treats both as a missing artifact and writes a valid empty
187
+ `.tgz` placeholder instead of saving the error page, so downstream batch
188
+ extraction can continue.
189
+
190
+ Artifact downloads show byte-level progress. On a TTY the progress is refreshed
191
+ in place; in CI or redirected stderr it falls back to periodic `[progress]`
192
+ lines. `--quiet` suppresses progress output.
193
+
194
+ ### Submit a Behavior Reward Model (`brm`) task
195
+
196
+ `brm run` creates a Behavior analysis task for exactly one Group. The CLI first
197
+ calls `GET /openapi/behavior/steps?group_id=...` to detect whether the Group
198
+ has step sampling, then posts `POST /openapi/behavior/tasks`.
199
+
200
+ ```bash
201
+ # No-step groups (all_jobs sampling): omit --steps
202
+ bl brm run -g G1
203
+
204
+ # Step groups: --steps is required (comma-separated or repeatable)
205
+ bl brm run -g G1 --steps 10,20
206
+ bl brm run -g G1 --steps 10 --steps 20 --sample-size 256 --scoring-limit 64
207
+
208
+ # Pipe the task id
209
+ bl brm run -g G1 --steps 10,20 | jq '.[0].uid'
210
+ ```
211
+
212
+ Defaults: `--sample-size 512` (step mode only), `--scoring-limit 32` (always
213
+ sent). In step mode, omitting `--steps` is a usage error and the available
214
+ steps are listed on stderr. Passing `--steps` on a no-step Group is also a
215
+ usage error.
216
+
217
+ On success, stderr logs a short confirmation and stdout prints a JSON array
218
+ (one object) that includes at least `uid`, plus `group_id` and
219
+ `samplingMode` (`step` or `all_jobs`). Use `.[0].uid` (not necessarily
220
+ `task_id`) for follow-up `brm status` / `brm get` calls.
221
+
222
+ ### Query a Behavior Reward Model (`brm`) task
223
+
224
+ `brm get` queries task status and/or analysis by Behavior task uid (from
225
+ `brm run`). Multiple `--task-id` values are supported (repeatable or
226
+ comma-separated); stdout is always a JSON array, one element per id, matching
227
+ `metrics get -j ...`.
228
+
229
+ ```bash
230
+ # Default --scope all (status + overall + detail); samplingMode only here
231
+ bl brm get --task-id UID
232
+ bl brm get --task-id UID --scope all | jq '.[0] | {status, overall, detail, samplingMode}'
233
+
234
+ # overall: analysis.summary + categoryScores
235
+ bl brm get --task-id UID --scope overall | jq '.[0] | {summary, categoryScores}'
236
+
237
+ # detail: analysis.behaviorPatterns
238
+ bl brm get --task-id UID --scope detail | jq '.[0].behaviorPatterns'
239
+
240
+ # status only; `brm status` is equivalent
241
+ bl brm get --task-id UID --scope status
242
+ bl brm status --task-id UID
243
+ bl brm status --task-id UID1,UID2
244
+ ```
245
+
246
+ `--scope` values:
247
+
248
+ | scope | source | payload |
249
+ | --- | --- | --- |
250
+ | `overall` | `GET .../analysis` | `summary`, `categoryScores` |
251
+ | `detail` | `GET .../analysis` | `behaviorPatterns` |
252
+ | `status` | `GET .../tasks/{uid}` | task status / progress |
253
+ | `all` (default) | both endpoints | nested `status`, `overall`, `detail`, plus top-level `samplingMode` |
254
+
255
+ `brm status` always uses the status endpoint (same as `brm get --scope status`).
256
+
257
+ ### List Behavior Reward Model (`brm`) tasks by Group
258
+
259
+ `brm list` calls `GET /openapi/behavior/tasks?group_id=...` and prints a JSON
260
+ array (one element per `-g`), matching `jobs get -g ...`:
261
+
262
+ ```bash
263
+ bl brm list -g G1
264
+ bl brm list -g G1,G2
265
+ bl brm list -g G1 | jq '.[0].tasks[].uid'
266
+ ```
267
+
268
+ Each element looks like `{ "group_id", "total", "tasks": [ ... ] }`. Task
269
+ objects keep the backend fields and also set `task_id` from `uid` when present.
270
+
271
+ ## Exit codes
272
+
273
+ | code | meaning |
274
+ | ---- | ------- |
275
+ | 0 | all success |
276
+ | 1 | partial failure (some ids failed) |
277
+ | 2 | usage / parameter error |
278
+ | 3 | auth failure (HTTP 401) |
279
+ | 4 | network / timeout |
280
+
281
+ stdout carries data only; logs and progress go to stderr. `--quiet` suppresses
282
+ non-error logs, `--verbose` adds debug logs.
283
+
284
+
285
+ ## Release new package
286
+
287
+ ```
288
+ uv build
289
+ twine check dist/benchloop_client-*.whl
290
+ twine upload dist/benchloop_client-*.whl --verbose
291
+ ```
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "benchloop_client"
7
- version = "0.2.22"
7
+ version = "0.2.24"
8
8
  description = "Benchloop CLI — command line tool for benchmark analysis data (metrics, trajectory, reports, artifacts)."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -10,6 +10,7 @@ from aliyun.bmcli import __version__, output, version_check
10
10
  from aliyun.bmcli.commands import (
11
11
  ap_cmd as ap_cmd_mod,
12
12
  artifacts as artifacts_cmd,
13
+ brm as brm_cmd,
13
14
  jobs as jobs_cmd,
14
15
  metadata as metadata_cmd,
15
16
  metrics as metrics_cmd,
@@ -137,6 +138,7 @@ app.add_typer(tasks_cmd.app, name="tasks")
137
138
  app.add_typer(jobs_cmd.app, name="jobs")
138
139
  app.add_typer(artifacts_cmd.app, name="artifacts")
139
140
  app.add_typer(metadata_cmd.app, name="metadata")
141
+ app.add_typer(brm_cmd.app, name="brm")
140
142
  app.command(
141
143
  "ap-cmd",
142
144
  context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
@@ -31,6 +31,12 @@ OutputDirOption = typer.Option(
31
31
  SkipExistingOption = typer.Option(
32
32
  False, "--skip-existing", help="Skip files that already exist on disk."
33
33
  )
34
+ DownloadTimeoutOption = typer.Option(
35
+ 1800,
36
+ "--download-timeout",
37
+ min=1,
38
+ help="Maximum seconds to wait for a Group Trace trajectory export.",
39
+ )
34
40
 
35
41
 
36
42
  def expand(values: Optional[list[str]]) -> list[str]: