nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,763 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Annotated, Any
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from rich.markup import escape
|
|
10
|
+
|
|
11
|
+
from nufi.cli.commands import common
|
|
12
|
+
from nufi.cli.commands.datasets import _is_uuid
|
|
13
|
+
from nufi.cli.commands.evaluations import (
|
|
14
|
+
Limit,
|
|
15
|
+
Output,
|
|
16
|
+
Page,
|
|
17
|
+
Profile,
|
|
18
|
+
Project,
|
|
19
|
+
ProjectID,
|
|
20
|
+
Verbose,
|
|
21
|
+
_client,
|
|
22
|
+
_load_mapping,
|
|
23
|
+
_project,
|
|
24
|
+
_resolve_method_id,
|
|
25
|
+
_resolve_model_artifact_id,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
app = typer.Typer(
|
|
29
|
+
help="Manage Evaluation-backed Leaderboards.",
|
|
30
|
+
invoke_without_command=True,
|
|
31
|
+
context_settings=common.CONTEXT_SETTINGS,
|
|
32
|
+
)
|
|
33
|
+
submission_app = typer.Typer(
|
|
34
|
+
help="Manage Leaderboard Submissions.",
|
|
35
|
+
invoke_without_command=True,
|
|
36
|
+
context_settings=common.CONTEXT_SETTINGS,
|
|
37
|
+
)
|
|
38
|
+
app.add_typer(submission_app, name="submission")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RankSource(str, Enum):
|
|
42
|
+
aggregate_metrics = "aggregateMetrics"
|
|
43
|
+
runtime_metrics = "runtimeMetrics"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class RankDirection(str, Enum):
|
|
47
|
+
asc = "asc"
|
|
48
|
+
desc = "desc"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.callback()
|
|
52
|
+
def leaderboards(ctx: typer.Context):
|
|
53
|
+
common.show_help_if_no_subcommand(ctx)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@submission_app.callback()
|
|
57
|
+
def submissions(ctx: typer.Context):
|
|
58
|
+
common.show_help_if_no_subcommand(ctx)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _resolve_leaderboard_id(
|
|
62
|
+
client: Any, leaderboard: str, *, project: str | int
|
|
63
|
+
) -> str:
|
|
64
|
+
if _is_uuid(leaderboard):
|
|
65
|
+
return leaderboard
|
|
66
|
+
return common.resolve_named_resource(
|
|
67
|
+
client.leaderboards.iter_all(project=project),
|
|
68
|
+
leaderboard,
|
|
69
|
+
resource="Leaderboard",
|
|
70
|
+
).id
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _print_board(item: Any) -> None:
|
|
74
|
+
common.print_table(
|
|
75
|
+
["Field", "Value"],
|
|
76
|
+
[
|
|
77
|
+
["ID", item.id],
|
|
78
|
+
["Name", item.name],
|
|
79
|
+
["Status", item.status],
|
|
80
|
+
["Evaluation Method", item.evaluation_method.name],
|
|
81
|
+
["Description", item.description],
|
|
82
|
+
["Runner Config", json.dumps(item.runner_config)],
|
|
83
|
+
["Gate Policy", json.dumps(common.serialize(item.gate_policy))],
|
|
84
|
+
["Ranking Policy", json.dumps(common.serialize(item.ranking_policy))],
|
|
85
|
+
["Created", item.created_at],
|
|
86
|
+
["Updated", item.updated_at],
|
|
87
|
+
],
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _print_submission(item: Any) -> None:
|
|
92
|
+
latest = item.latest_run
|
|
93
|
+
scoring = item.scoring_run
|
|
94
|
+
common.print_table(
|
|
95
|
+
["Field", "Value"],
|
|
96
|
+
[
|
|
97
|
+
["ID", item.id],
|
|
98
|
+
["Leaderboard ID", item.leaderboard_id],
|
|
99
|
+
[
|
|
100
|
+
"Artifact",
|
|
101
|
+
f"{item.artifact.model_name or ''}/{item.artifact.version or ''}/{item.artifact.name}",
|
|
102
|
+
],
|
|
103
|
+
["Platform", item.artifact.platform],
|
|
104
|
+
["CPU", item.requested_resources.cpu],
|
|
105
|
+
["Memory", item.requested_resources.memory],
|
|
106
|
+
["Accelerators", item.requested_resources.accelerator_count],
|
|
107
|
+
["Qualification", item.qualification],
|
|
108
|
+
["Latest Run", latest.evaluation_run_id if latest else ""],
|
|
109
|
+
["Latest Status", latest.status if latest else ""],
|
|
110
|
+
[
|
|
111
|
+
"Failure",
|
|
112
|
+
common.bounded_error(latest.error_message if latest else None),
|
|
113
|
+
],
|
|
114
|
+
["Scoring Run", scoring.evaluation_run_id if scoring else ""],
|
|
115
|
+
["Created", item.created_at],
|
|
116
|
+
],
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _metric_summary(metrics: dict[str, Any]) -> str:
|
|
121
|
+
rows = [
|
|
122
|
+
(name, f"{value:.6g}" if isinstance(value, float) else str(value))
|
|
123
|
+
for name, value in sorted(metrics.items())
|
|
124
|
+
]
|
|
125
|
+
name_width = max((len(name) for name, _ in rows), default=0)
|
|
126
|
+
value_width = max((len(value) for _, value in rows), default=0)
|
|
127
|
+
return "\n".join(
|
|
128
|
+
f"{name:<{name_width}} {value:>{value_width}}" for name, value in rows
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _print_or_json(item: Any, output: common.OutputChoice, printer: Any) -> None:
|
|
133
|
+
if output == common.OutputChoice.json:
|
|
134
|
+
common.print_json(item)
|
|
135
|
+
else:
|
|
136
|
+
printer(item)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _print_submission_output(item: Any, output: common.OutputChoice) -> None:
|
|
140
|
+
if output == common.OutputChoice.json:
|
|
141
|
+
payload = common.serialize(item)
|
|
142
|
+
for key in ("latestRun", "scoringRun"):
|
|
143
|
+
if (payload.get(key) or {}).get("errorMessage"):
|
|
144
|
+
payload[key]["errorMessage"] = common.bounded_error(
|
|
145
|
+
payload[key]["errorMessage"]
|
|
146
|
+
)
|
|
147
|
+
common.print_json(payload)
|
|
148
|
+
else:
|
|
149
|
+
_print_submission(item)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@app.command("list")
|
|
153
|
+
def list_leaderboards(
|
|
154
|
+
status: Annotated[
|
|
155
|
+
str | None,
|
|
156
|
+
typer.Option("--status", help="active or archived.", show_default=False),
|
|
157
|
+
] = None,
|
|
158
|
+
page: Page = None,
|
|
159
|
+
limit: Limit = None,
|
|
160
|
+
project: Project = None,
|
|
161
|
+
project_id: ProjectID = None,
|
|
162
|
+
profile_name: Profile = None,
|
|
163
|
+
output: Output = common.OutputChoice.table,
|
|
164
|
+
verbose: Verbose = False,
|
|
165
|
+
):
|
|
166
|
+
try:
|
|
167
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
168
|
+
with _client(profile_name) as client:
|
|
169
|
+
result_page = client.leaderboards.list_page(
|
|
170
|
+
project=resolved_project, status=status, page=page, limit=limit
|
|
171
|
+
)
|
|
172
|
+
except Exception as exc:
|
|
173
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
174
|
+
if output == common.OutputChoice.json:
|
|
175
|
+
common.print_json(result_page)
|
|
176
|
+
return
|
|
177
|
+
common.print_page_table(
|
|
178
|
+
result_page,
|
|
179
|
+
["ID", "Name", "Status", "Evaluation Method", "Created"],
|
|
180
|
+
[
|
|
181
|
+
[
|
|
182
|
+
item.id,
|
|
183
|
+
item.name,
|
|
184
|
+
item.status,
|
|
185
|
+
item.evaluation_method.name,
|
|
186
|
+
item.created_at,
|
|
187
|
+
]
|
|
188
|
+
for item in result_page.items
|
|
189
|
+
],
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@app.command("create")
|
|
194
|
+
def create_leaderboard(
|
|
195
|
+
name: Annotated[str, typer.Argument(help="Leaderboard name.")],
|
|
196
|
+
method: Annotated[
|
|
197
|
+
str, typer.Option("--method", help="Evaluation Method name or UUID.")
|
|
198
|
+
],
|
|
199
|
+
ranking_policy: Annotated[
|
|
200
|
+
Path | None,
|
|
201
|
+
typer.Option(
|
|
202
|
+
"--ranking-policy",
|
|
203
|
+
help="JSON or YAML ranking policy file.",
|
|
204
|
+
show_default=False,
|
|
205
|
+
),
|
|
206
|
+
] = None,
|
|
207
|
+
rank_metric: Annotated[
|
|
208
|
+
str | None,
|
|
209
|
+
typer.Option(
|
|
210
|
+
"--rank-metric",
|
|
211
|
+
help="Metric name for a simple one-metric ranking policy.",
|
|
212
|
+
show_default=False,
|
|
213
|
+
),
|
|
214
|
+
] = None,
|
|
215
|
+
rank_source: Annotated[
|
|
216
|
+
RankSource | None,
|
|
217
|
+
typer.Option(
|
|
218
|
+
"--rank-source",
|
|
219
|
+
help="Metric source; defaults to aggregateMetrics.",
|
|
220
|
+
show_default=False,
|
|
221
|
+
),
|
|
222
|
+
] = None,
|
|
223
|
+
rank_direction: Annotated[
|
|
224
|
+
RankDirection | None,
|
|
225
|
+
typer.Option(
|
|
226
|
+
"--rank-direction",
|
|
227
|
+
help="Ranking direction for the simple policy.",
|
|
228
|
+
show_default=False,
|
|
229
|
+
),
|
|
230
|
+
] = None,
|
|
231
|
+
runner_config: Annotated[
|
|
232
|
+
Path | None,
|
|
233
|
+
typer.Option(
|
|
234
|
+
"--runner-config",
|
|
235
|
+
help="JSON or YAML runtime overrides file.",
|
|
236
|
+
show_default=False,
|
|
237
|
+
),
|
|
238
|
+
] = None,
|
|
239
|
+
gate_policy: Annotated[
|
|
240
|
+
Path | None,
|
|
241
|
+
typer.Option(
|
|
242
|
+
"--gate-policy", help="JSON or YAML gate policy file.", show_default=False
|
|
243
|
+
),
|
|
244
|
+
] = None,
|
|
245
|
+
description: Annotated[
|
|
246
|
+
str | None,
|
|
247
|
+
typer.Option("--description", help="Description.", show_default=False),
|
|
248
|
+
] = None,
|
|
249
|
+
project: Project = None,
|
|
250
|
+
project_id: ProjectID = None,
|
|
251
|
+
profile_name: Profile = None,
|
|
252
|
+
output: Output = common.OutputChoice.table,
|
|
253
|
+
verbose: Verbose = False,
|
|
254
|
+
):
|
|
255
|
+
simple = any(
|
|
256
|
+
value is not None for value in (rank_metric, rank_source, rank_direction)
|
|
257
|
+
)
|
|
258
|
+
if ranking_policy is not None and simple:
|
|
259
|
+
raise typer.BadParameter(
|
|
260
|
+
"--ranking-policy and simple ranking flags are mutually exclusive"
|
|
261
|
+
)
|
|
262
|
+
if ranking_policy is None and (rank_metric is None or rank_direction is None):
|
|
263
|
+
raise typer.BadParameter(
|
|
264
|
+
"--rank-metric and --rank-direction are required without --ranking-policy"
|
|
265
|
+
)
|
|
266
|
+
try:
|
|
267
|
+
if ranking_policy is not None:
|
|
268
|
+
ranking = _load_mapping(ranking_policy)
|
|
269
|
+
else:
|
|
270
|
+
ranking = {
|
|
271
|
+
"primary": {
|
|
272
|
+
"metric": {
|
|
273
|
+
"source": (rank_source or RankSource.aggregate_metrics).value,
|
|
274
|
+
"name": rank_metric,
|
|
275
|
+
},
|
|
276
|
+
"direction": rank_direction.value,
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
runner = _load_mapping(runner_config) if runner_config else None
|
|
280
|
+
gate = _load_mapping(gate_policy) if gate_policy else None
|
|
281
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
282
|
+
with _client(profile_name) as client:
|
|
283
|
+
method_id = _resolve_method_id(client, method, project=resolved_project)
|
|
284
|
+
result = client.leaderboards.create(
|
|
285
|
+
name,
|
|
286
|
+
project=resolved_project,
|
|
287
|
+
evaluation_method_id=method_id,
|
|
288
|
+
ranking_policy=ranking,
|
|
289
|
+
runner_config=runner,
|
|
290
|
+
gate_policy=gate,
|
|
291
|
+
description=description,
|
|
292
|
+
)
|
|
293
|
+
except Exception as exc:
|
|
294
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
295
|
+
if output == common.OutputChoice.json:
|
|
296
|
+
common.print_json(result)
|
|
297
|
+
return
|
|
298
|
+
common.console.print(f"[bold green]Leaderboard created:[/] {escape(result.id)}")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
@app.command("get")
|
|
302
|
+
def get_leaderboard(
|
|
303
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
304
|
+
project: Project = None,
|
|
305
|
+
project_id: ProjectID = None,
|
|
306
|
+
profile_name: Profile = None,
|
|
307
|
+
output: Output = common.OutputChoice.table,
|
|
308
|
+
verbose: Verbose = False,
|
|
309
|
+
):
|
|
310
|
+
try:
|
|
311
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
312
|
+
with _client(profile_name) as client:
|
|
313
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
314
|
+
client, leaderboard, project=resolved_project
|
|
315
|
+
)
|
|
316
|
+
result = client.leaderboards.get(leaderboard_id, project=resolved_project)
|
|
317
|
+
except Exception as exc:
|
|
318
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
319
|
+
_print_or_json(result, output, _print_board)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@app.command("update")
|
|
323
|
+
def update_leaderboard(
|
|
324
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
325
|
+
name: Annotated[
|
|
326
|
+
str | None, typer.Option("--name", help="New name.", show_default=False)
|
|
327
|
+
] = None,
|
|
328
|
+
description: Annotated[
|
|
329
|
+
str | None,
|
|
330
|
+
typer.Option("--description", help="New description.", show_default=False),
|
|
331
|
+
] = None,
|
|
332
|
+
project: Project = None,
|
|
333
|
+
project_id: ProjectID = None,
|
|
334
|
+
profile_name: Profile = None,
|
|
335
|
+
output: Output = common.OutputChoice.table,
|
|
336
|
+
verbose: Verbose = False,
|
|
337
|
+
):
|
|
338
|
+
try:
|
|
339
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
340
|
+
with _client(profile_name) as client:
|
|
341
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
342
|
+
client, leaderboard, project=resolved_project
|
|
343
|
+
)
|
|
344
|
+
result = client.leaderboards.update(
|
|
345
|
+
leaderboard_id,
|
|
346
|
+
project=resolved_project,
|
|
347
|
+
name=name,
|
|
348
|
+
description=description,
|
|
349
|
+
)
|
|
350
|
+
except Exception as exc:
|
|
351
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
352
|
+
_print_or_json(result, output, _print_board)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _status_command(
|
|
356
|
+
leaderboard: str,
|
|
357
|
+
*,
|
|
358
|
+
action: str,
|
|
359
|
+
project: str | None,
|
|
360
|
+
project_id: int | None,
|
|
361
|
+
profile_name: str | None,
|
|
362
|
+
output: common.OutputChoice,
|
|
363
|
+
verbose: bool,
|
|
364
|
+
) -> None:
|
|
365
|
+
try:
|
|
366
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
367
|
+
with _client(profile_name) as client:
|
|
368
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
369
|
+
client, leaderboard, project=resolved_project
|
|
370
|
+
)
|
|
371
|
+
result = getattr(client.leaderboards, action)(
|
|
372
|
+
leaderboard_id, project=resolved_project
|
|
373
|
+
)
|
|
374
|
+
except Exception as exc:
|
|
375
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
376
|
+
_print_or_json(result, output, _print_board)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
@app.command("archive")
|
|
380
|
+
def archive_leaderboard(
|
|
381
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
382
|
+
project: Project = None,
|
|
383
|
+
project_id: ProjectID = None,
|
|
384
|
+
profile_name: Profile = None,
|
|
385
|
+
output: Output = common.OutputChoice.table,
|
|
386
|
+
verbose: Verbose = False,
|
|
387
|
+
):
|
|
388
|
+
_status_command(
|
|
389
|
+
leaderboard,
|
|
390
|
+
action="archive",
|
|
391
|
+
project=project,
|
|
392
|
+
project_id=project_id,
|
|
393
|
+
profile_name=profile_name,
|
|
394
|
+
output=output,
|
|
395
|
+
verbose=verbose,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
@app.command("restore")
|
|
400
|
+
def restore_leaderboard(
|
|
401
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
402
|
+
project: Project = None,
|
|
403
|
+
project_id: ProjectID = None,
|
|
404
|
+
profile_name: Profile = None,
|
|
405
|
+
output: Output = common.OutputChoice.table,
|
|
406
|
+
verbose: Verbose = False,
|
|
407
|
+
):
|
|
408
|
+
_status_command(
|
|
409
|
+
leaderboard,
|
|
410
|
+
action="restore",
|
|
411
|
+
project=project,
|
|
412
|
+
project_id=project_id,
|
|
413
|
+
profile_name=profile_name,
|
|
414
|
+
output=output,
|
|
415
|
+
verbose=verbose,
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@app.command("entries")
|
|
420
|
+
def list_entries(
|
|
421
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
422
|
+
platform: Annotated[
|
|
423
|
+
str | None, typer.Option("--platform", help="gpu or rngd.", show_default=False)
|
|
424
|
+
] = None,
|
|
425
|
+
accelerator_count: Annotated[
|
|
426
|
+
int | None,
|
|
427
|
+
typer.Option("--accelerator-count", min=1, show_default=False),
|
|
428
|
+
] = None,
|
|
429
|
+
page: Page = None,
|
|
430
|
+
limit: Limit = None,
|
|
431
|
+
project: Project = None,
|
|
432
|
+
project_id: ProjectID = None,
|
|
433
|
+
profile_name: Profile = None,
|
|
434
|
+
output: Output = common.OutputChoice.table,
|
|
435
|
+
verbose: Verbose = False,
|
|
436
|
+
):
|
|
437
|
+
try:
|
|
438
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
439
|
+
with _client(profile_name) as client:
|
|
440
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
441
|
+
client, leaderboard, project=resolved_project
|
|
442
|
+
)
|
|
443
|
+
result_page = client.leaderboards.entries_page(
|
|
444
|
+
leaderboard_id,
|
|
445
|
+
project=resolved_project,
|
|
446
|
+
platform=platform,
|
|
447
|
+
accelerator_count=accelerator_count,
|
|
448
|
+
page=page,
|
|
449
|
+
limit=limit,
|
|
450
|
+
)
|
|
451
|
+
except Exception as exc:
|
|
452
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
453
|
+
if output == common.OutputChoice.json:
|
|
454
|
+
common.print_json(result_page)
|
|
455
|
+
return
|
|
456
|
+
common.print_page_table(
|
|
457
|
+
result_page,
|
|
458
|
+
[
|
|
459
|
+
"Rank",
|
|
460
|
+
"Submission",
|
|
461
|
+
"Artifact",
|
|
462
|
+
"Platform",
|
|
463
|
+
"Resources",
|
|
464
|
+
"Aggregate Metrics",
|
|
465
|
+
"Runtime Metrics",
|
|
466
|
+
],
|
|
467
|
+
[
|
|
468
|
+
[
|
|
469
|
+
item.rank,
|
|
470
|
+
item.submission_id,
|
|
471
|
+
item.artifact.name,
|
|
472
|
+
item.artifact.platform,
|
|
473
|
+
f"{item.requested_resources.cpu}/{item.requested_resources.memory}/{item.requested_resources.accelerator_count}",
|
|
474
|
+
_metric_summary(item.aggregate_metrics),
|
|
475
|
+
_metric_summary(item.runtime_metrics),
|
|
476
|
+
]
|
|
477
|
+
for item in result_page.items
|
|
478
|
+
],
|
|
479
|
+
show_lines=True,
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
@submission_app.command("create")
|
|
484
|
+
def create_submission(
|
|
485
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
486
|
+
model_artifact_id: Annotated[
|
|
487
|
+
int | None,
|
|
488
|
+
typer.Option("--model-artifact-id", min=1, show_default=False),
|
|
489
|
+
] = None,
|
|
490
|
+
model: Annotated[
|
|
491
|
+
str | None,
|
|
492
|
+
typer.Option("--model", help="Registered Model name.", show_default=False),
|
|
493
|
+
] = None,
|
|
494
|
+
model_version: Annotated[
|
|
495
|
+
str | None,
|
|
496
|
+
typer.Option("--model-version", help="Model Version name.", show_default=False),
|
|
497
|
+
] = None,
|
|
498
|
+
artifact: Annotated[
|
|
499
|
+
str | None,
|
|
500
|
+
typer.Option("--artifact", help="Model Artifact name.", show_default=False),
|
|
501
|
+
] = None,
|
|
502
|
+
cpu: Annotated[
|
|
503
|
+
str | None,
|
|
504
|
+
typer.Option(
|
|
505
|
+
"--cpu", help="CPU quantity; server default 4.", show_default=False
|
|
506
|
+
),
|
|
507
|
+
] = None,
|
|
508
|
+
memory: Annotated[
|
|
509
|
+
str | None,
|
|
510
|
+
typer.Option(
|
|
511
|
+
"--memory", help="Memory quantity; server default 8Gi.", show_default=False
|
|
512
|
+
),
|
|
513
|
+
] = None,
|
|
514
|
+
accelerator_count: Annotated[
|
|
515
|
+
int | None,
|
|
516
|
+
typer.Option(
|
|
517
|
+
"--accelerator-count",
|
|
518
|
+
min=1,
|
|
519
|
+
help="Accelerator count; server default 1.",
|
|
520
|
+
show_default=False,
|
|
521
|
+
),
|
|
522
|
+
] = None,
|
|
523
|
+
wait: Annotated[
|
|
524
|
+
bool, typer.Option("--wait", help="Wait for terminal state.")
|
|
525
|
+
] = False,
|
|
526
|
+
timeout: Annotated[
|
|
527
|
+
float,
|
|
528
|
+
typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
|
|
529
|
+
] = 1800,
|
|
530
|
+
interval: Annotated[
|
|
531
|
+
float,
|
|
532
|
+
typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
|
|
533
|
+
] = 5,
|
|
534
|
+
project: Project = None,
|
|
535
|
+
project_id: ProjectID = None,
|
|
536
|
+
profile_name: Profile = None,
|
|
537
|
+
output: Output = common.OutputChoice.table,
|
|
538
|
+
verbose: Verbose = False,
|
|
539
|
+
):
|
|
540
|
+
common.validate_wait_options(timeout, interval)
|
|
541
|
+
try:
|
|
542
|
+
resources = (cpu, memory, accelerator_count)
|
|
543
|
+
if any(value is not None for value in resources) and not all(
|
|
544
|
+
value is not None for value in resources
|
|
545
|
+
):
|
|
546
|
+
raise ValueError(
|
|
547
|
+
"cpu, memory, and accelerator_count must be provided together"
|
|
548
|
+
)
|
|
549
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
550
|
+
with _client(profile_name) as client:
|
|
551
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
552
|
+
client, leaderboard, project=resolved_project
|
|
553
|
+
)
|
|
554
|
+
artifact_id = _resolve_model_artifact_id(
|
|
555
|
+
client,
|
|
556
|
+
project=resolved_project,
|
|
557
|
+
model_artifact_id=model_artifact_id,
|
|
558
|
+
model=model,
|
|
559
|
+
model_version=model_version,
|
|
560
|
+
artifact=artifact,
|
|
561
|
+
)
|
|
562
|
+
result = client.leaderboards.submit(
|
|
563
|
+
leaderboard_id,
|
|
564
|
+
project=resolved_project,
|
|
565
|
+
model_artifact_id=artifact_id,
|
|
566
|
+
cpu=cpu,
|
|
567
|
+
memory=memory,
|
|
568
|
+
accelerator_count=accelerator_count,
|
|
569
|
+
)
|
|
570
|
+
if wait:
|
|
571
|
+
typer.echo(
|
|
572
|
+
f"Leaderboard Submission created: {result.id}; resume with "
|
|
573
|
+
"'nufi leaderboard submission wait "
|
|
574
|
+
f"{leaderboard_id} {result.id}'",
|
|
575
|
+
err=True,
|
|
576
|
+
)
|
|
577
|
+
result = client.leaderboards.wait(
|
|
578
|
+
leaderboard_id,
|
|
579
|
+
result.id,
|
|
580
|
+
project=resolved_project,
|
|
581
|
+
timeout=timeout,
|
|
582
|
+
interval=interval,
|
|
583
|
+
)
|
|
584
|
+
except Exception as exc:
|
|
585
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
586
|
+
if wait:
|
|
587
|
+
_print_submission_output(result, output)
|
|
588
|
+
elif output == common.OutputChoice.json:
|
|
589
|
+
common.print_json(result)
|
|
590
|
+
else:
|
|
591
|
+
common.console.print(
|
|
592
|
+
f"[bold green]Leaderboard Submission created:[/] {escape(result.id)}"
|
|
593
|
+
)
|
|
594
|
+
if result.latest_run and result.latest_run.status in {"failed", "cancelled"}:
|
|
595
|
+
raise typer.Exit(code=1)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
@submission_app.command("list")
|
|
599
|
+
def list_submissions(
|
|
600
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
601
|
+
page: Page = None,
|
|
602
|
+
limit: Limit = None,
|
|
603
|
+
project: Project = None,
|
|
604
|
+
project_id: ProjectID = None,
|
|
605
|
+
profile_name: Profile = None,
|
|
606
|
+
output: Output = common.OutputChoice.table,
|
|
607
|
+
verbose: Verbose = False,
|
|
608
|
+
):
|
|
609
|
+
try:
|
|
610
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
611
|
+
with _client(profile_name) as client:
|
|
612
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
613
|
+
client, leaderboard, project=resolved_project
|
|
614
|
+
)
|
|
615
|
+
result_page = client.leaderboards.list_submissions_page(
|
|
616
|
+
leaderboard_id, project=resolved_project, page=page, limit=limit
|
|
617
|
+
)
|
|
618
|
+
except Exception as exc:
|
|
619
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
620
|
+
if output == common.OutputChoice.json:
|
|
621
|
+
common.print_json(result_page)
|
|
622
|
+
return
|
|
623
|
+
common.print_page_table(
|
|
624
|
+
result_page,
|
|
625
|
+
["ID", "Artifact", "Platform", "Resources", "Qualification", "Latest Run"],
|
|
626
|
+
[
|
|
627
|
+
[
|
|
628
|
+
item.id,
|
|
629
|
+
item.artifact.name,
|
|
630
|
+
item.artifact.platform,
|
|
631
|
+
f"{item.requested_resources.cpu}/{item.requested_resources.memory}/{item.requested_resources.accelerator_count}",
|
|
632
|
+
item.qualification,
|
|
633
|
+
item.latest_run.status if item.latest_run else "",
|
|
634
|
+
]
|
|
635
|
+
for item in result_page.items
|
|
636
|
+
],
|
|
637
|
+
)
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
@submission_app.command("get")
|
|
641
|
+
def get_submission(
|
|
642
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
643
|
+
submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
|
|
644
|
+
project: Project = None,
|
|
645
|
+
project_id: ProjectID = None,
|
|
646
|
+
profile_name: Profile = None,
|
|
647
|
+
output: Output = common.OutputChoice.table,
|
|
648
|
+
verbose: Verbose = False,
|
|
649
|
+
):
|
|
650
|
+
try:
|
|
651
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
652
|
+
with _client(profile_name) as client:
|
|
653
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
654
|
+
client, leaderboard, project=resolved_project
|
|
655
|
+
)
|
|
656
|
+
result = client.leaderboards.get_submission(
|
|
657
|
+
leaderboard_id, submission_id, project=resolved_project
|
|
658
|
+
)
|
|
659
|
+
except Exception as exc:
|
|
660
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
661
|
+
_print_submission_output(result, output)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
@submission_app.command("wait")
|
|
665
|
+
def wait_submission(
|
|
666
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
667
|
+
submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
|
|
668
|
+
project: Project = None,
|
|
669
|
+
project_id: ProjectID = None,
|
|
670
|
+
timeout: Annotated[
|
|
671
|
+
float,
|
|
672
|
+
typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
|
|
673
|
+
] = 1800,
|
|
674
|
+
interval: Annotated[
|
|
675
|
+
float,
|
|
676
|
+
typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
|
|
677
|
+
] = 5,
|
|
678
|
+
profile_name: Profile = None,
|
|
679
|
+
output: Output = common.OutputChoice.table,
|
|
680
|
+
verbose: Verbose = False,
|
|
681
|
+
):
|
|
682
|
+
common.validate_wait_options(timeout, interval)
|
|
683
|
+
try:
|
|
684
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
685
|
+
with _client(profile_name) as client:
|
|
686
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
687
|
+
client, leaderboard, project=resolved_project
|
|
688
|
+
)
|
|
689
|
+
result = client.leaderboards.wait(
|
|
690
|
+
leaderboard_id,
|
|
691
|
+
submission_id,
|
|
692
|
+
project=resolved_project,
|
|
693
|
+
timeout=timeout,
|
|
694
|
+
interval=interval,
|
|
695
|
+
)
|
|
696
|
+
except Exception as exc:
|
|
697
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
698
|
+
_print_submission_output(result, output)
|
|
699
|
+
if result.latest_run and result.latest_run.status in {"failed", "cancelled"}:
|
|
700
|
+
raise typer.Exit(code=1)
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
@submission_app.command("rerun")
|
|
704
|
+
def rerun_submission(
|
|
705
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
706
|
+
submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
|
|
707
|
+
project: Project = None,
|
|
708
|
+
project_id: ProjectID = None,
|
|
709
|
+
profile_name: Profile = None,
|
|
710
|
+
output: Output = common.OutputChoice.table,
|
|
711
|
+
verbose: Verbose = False,
|
|
712
|
+
):
|
|
713
|
+
try:
|
|
714
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
715
|
+
with _client(profile_name) as client:
|
|
716
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
717
|
+
client, leaderboard, project=resolved_project
|
|
718
|
+
)
|
|
719
|
+
result = client.leaderboards.rerun(
|
|
720
|
+
leaderboard_id, submission_id, project=resolved_project
|
|
721
|
+
)
|
|
722
|
+
except Exception as exc:
|
|
723
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
724
|
+
_print_submission_output(result, output)
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
@submission_app.command("runs")
|
|
728
|
+
def list_submission_runs(
|
|
729
|
+
leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
|
|
730
|
+
submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
|
|
731
|
+
project: Project = None,
|
|
732
|
+
project_id: ProjectID = None,
|
|
733
|
+
profile_name: Profile = None,
|
|
734
|
+
output: Output = common.OutputChoice.table,
|
|
735
|
+
verbose: Verbose = False,
|
|
736
|
+
):
|
|
737
|
+
try:
|
|
738
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
739
|
+
with _client(profile_name) as client:
|
|
740
|
+
leaderboard_id = _resolve_leaderboard_id(
|
|
741
|
+
client, leaderboard, project=resolved_project
|
|
742
|
+
)
|
|
743
|
+
items = client.leaderboards.list_runs(
|
|
744
|
+
leaderboard_id, submission_id, project=resolved_project
|
|
745
|
+
)
|
|
746
|
+
except Exception as exc:
|
|
747
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
748
|
+
if output == common.OutputChoice.json:
|
|
749
|
+
common.print_json(items)
|
|
750
|
+
return
|
|
751
|
+
common.print_table(
|
|
752
|
+
["Attempt", "Evaluation Run", "Status", "Created", "Completed"],
|
|
753
|
+
[
|
|
754
|
+
[
|
|
755
|
+
item.attempt_number,
|
|
756
|
+
item.evaluation_run_id,
|
|
757
|
+
item.status,
|
|
758
|
+
item.created_at,
|
|
759
|
+
item.completed_at or "",
|
|
760
|
+
]
|
|
761
|
+
for item in items
|
|
762
|
+
],
|
|
763
|
+
)
|