nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,763 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from enum import Enum
5
+ from pathlib import Path
6
+ from typing import Annotated, Any
7
+
8
+ import typer
9
+ from rich.markup import escape
10
+
11
+ from nufi.cli.commands import common
12
+ from nufi.cli.commands.datasets import _is_uuid
13
+ from nufi.cli.commands.evaluations import (
14
+ Limit,
15
+ Output,
16
+ Page,
17
+ Profile,
18
+ Project,
19
+ ProjectID,
20
+ Verbose,
21
+ _client,
22
+ _load_mapping,
23
+ _project,
24
+ _resolve_method_id,
25
+ _resolve_model_artifact_id,
26
+ )
27
+
28
+ app = typer.Typer(
29
+ help="Manage Evaluation-backed Leaderboards.",
30
+ invoke_without_command=True,
31
+ context_settings=common.CONTEXT_SETTINGS,
32
+ )
33
+ submission_app = typer.Typer(
34
+ help="Manage Leaderboard Submissions.",
35
+ invoke_without_command=True,
36
+ context_settings=common.CONTEXT_SETTINGS,
37
+ )
38
+ app.add_typer(submission_app, name="submission")
39
+
40
+
41
+ class RankSource(str, Enum):
42
+ aggregate_metrics = "aggregateMetrics"
43
+ runtime_metrics = "runtimeMetrics"
44
+
45
+
46
+ class RankDirection(str, Enum):
47
+ asc = "asc"
48
+ desc = "desc"
49
+
50
+
51
+ @app.callback()
52
+ def leaderboards(ctx: typer.Context):
53
+ common.show_help_if_no_subcommand(ctx)
54
+
55
+
56
+ @submission_app.callback()
57
+ def submissions(ctx: typer.Context):
58
+ common.show_help_if_no_subcommand(ctx)
59
+
60
+
61
+ def _resolve_leaderboard_id(
62
+ client: Any, leaderboard: str, *, project: str | int
63
+ ) -> str:
64
+ if _is_uuid(leaderboard):
65
+ return leaderboard
66
+ return common.resolve_named_resource(
67
+ client.leaderboards.iter_all(project=project),
68
+ leaderboard,
69
+ resource="Leaderboard",
70
+ ).id
71
+
72
+
73
+ def _print_board(item: Any) -> None:
74
+ common.print_table(
75
+ ["Field", "Value"],
76
+ [
77
+ ["ID", item.id],
78
+ ["Name", item.name],
79
+ ["Status", item.status],
80
+ ["Evaluation Method", item.evaluation_method.name],
81
+ ["Description", item.description],
82
+ ["Runner Config", json.dumps(item.runner_config)],
83
+ ["Gate Policy", json.dumps(common.serialize(item.gate_policy))],
84
+ ["Ranking Policy", json.dumps(common.serialize(item.ranking_policy))],
85
+ ["Created", item.created_at],
86
+ ["Updated", item.updated_at],
87
+ ],
88
+ )
89
+
90
+
91
+ def _print_submission(item: Any) -> None:
92
+ latest = item.latest_run
93
+ scoring = item.scoring_run
94
+ common.print_table(
95
+ ["Field", "Value"],
96
+ [
97
+ ["ID", item.id],
98
+ ["Leaderboard ID", item.leaderboard_id],
99
+ [
100
+ "Artifact",
101
+ f"{item.artifact.model_name or ''}/{item.artifact.version or ''}/{item.artifact.name}",
102
+ ],
103
+ ["Platform", item.artifact.platform],
104
+ ["CPU", item.requested_resources.cpu],
105
+ ["Memory", item.requested_resources.memory],
106
+ ["Accelerators", item.requested_resources.accelerator_count],
107
+ ["Qualification", item.qualification],
108
+ ["Latest Run", latest.evaluation_run_id if latest else ""],
109
+ ["Latest Status", latest.status if latest else ""],
110
+ [
111
+ "Failure",
112
+ common.bounded_error(latest.error_message if latest else None),
113
+ ],
114
+ ["Scoring Run", scoring.evaluation_run_id if scoring else ""],
115
+ ["Created", item.created_at],
116
+ ],
117
+ )
118
+
119
+
120
+ def _metric_summary(metrics: dict[str, Any]) -> str:
121
+ rows = [
122
+ (name, f"{value:.6g}" if isinstance(value, float) else str(value))
123
+ for name, value in sorted(metrics.items())
124
+ ]
125
+ name_width = max((len(name) for name, _ in rows), default=0)
126
+ value_width = max((len(value) for _, value in rows), default=0)
127
+ return "\n".join(
128
+ f"{name:<{name_width}} {value:>{value_width}}" for name, value in rows
129
+ )
130
+
131
+
132
+ def _print_or_json(item: Any, output: common.OutputChoice, printer: Any) -> None:
133
+ if output == common.OutputChoice.json:
134
+ common.print_json(item)
135
+ else:
136
+ printer(item)
137
+
138
+
139
+ def _print_submission_output(item: Any, output: common.OutputChoice) -> None:
140
+ if output == common.OutputChoice.json:
141
+ payload = common.serialize(item)
142
+ for key in ("latestRun", "scoringRun"):
143
+ if (payload.get(key) or {}).get("errorMessage"):
144
+ payload[key]["errorMessage"] = common.bounded_error(
145
+ payload[key]["errorMessage"]
146
+ )
147
+ common.print_json(payload)
148
+ else:
149
+ _print_submission(item)
150
+
151
+
152
+ @app.command("list")
153
+ def list_leaderboards(
154
+ status: Annotated[
155
+ str | None,
156
+ typer.Option("--status", help="active or archived.", show_default=False),
157
+ ] = None,
158
+ page: Page = None,
159
+ limit: Limit = None,
160
+ project: Project = None,
161
+ project_id: ProjectID = None,
162
+ profile_name: Profile = None,
163
+ output: Output = common.OutputChoice.table,
164
+ verbose: Verbose = False,
165
+ ):
166
+ try:
167
+ resolved_project = _project(project, project_id, profile_name)
168
+ with _client(profile_name) as client:
169
+ result_page = client.leaderboards.list_page(
170
+ project=resolved_project, status=status, page=page, limit=limit
171
+ )
172
+ except Exception as exc:
173
+ common.handle_cli_error(exc, verbose=verbose)
174
+ if output == common.OutputChoice.json:
175
+ common.print_json(result_page)
176
+ return
177
+ common.print_page_table(
178
+ result_page,
179
+ ["ID", "Name", "Status", "Evaluation Method", "Created"],
180
+ [
181
+ [
182
+ item.id,
183
+ item.name,
184
+ item.status,
185
+ item.evaluation_method.name,
186
+ item.created_at,
187
+ ]
188
+ for item in result_page.items
189
+ ],
190
+ )
191
+
192
+
193
+ @app.command("create")
194
+ def create_leaderboard(
195
+ name: Annotated[str, typer.Argument(help="Leaderboard name.")],
196
+ method: Annotated[
197
+ str, typer.Option("--method", help="Evaluation Method name or UUID.")
198
+ ],
199
+ ranking_policy: Annotated[
200
+ Path | None,
201
+ typer.Option(
202
+ "--ranking-policy",
203
+ help="JSON or YAML ranking policy file.",
204
+ show_default=False,
205
+ ),
206
+ ] = None,
207
+ rank_metric: Annotated[
208
+ str | None,
209
+ typer.Option(
210
+ "--rank-metric",
211
+ help="Metric name for a simple one-metric ranking policy.",
212
+ show_default=False,
213
+ ),
214
+ ] = None,
215
+ rank_source: Annotated[
216
+ RankSource | None,
217
+ typer.Option(
218
+ "--rank-source",
219
+ help="Metric source; defaults to aggregateMetrics.",
220
+ show_default=False,
221
+ ),
222
+ ] = None,
223
+ rank_direction: Annotated[
224
+ RankDirection | None,
225
+ typer.Option(
226
+ "--rank-direction",
227
+ help="Ranking direction for the simple policy.",
228
+ show_default=False,
229
+ ),
230
+ ] = None,
231
+ runner_config: Annotated[
232
+ Path | None,
233
+ typer.Option(
234
+ "--runner-config",
235
+ help="JSON or YAML runtime overrides file.",
236
+ show_default=False,
237
+ ),
238
+ ] = None,
239
+ gate_policy: Annotated[
240
+ Path | None,
241
+ typer.Option(
242
+ "--gate-policy", help="JSON or YAML gate policy file.", show_default=False
243
+ ),
244
+ ] = None,
245
+ description: Annotated[
246
+ str | None,
247
+ typer.Option("--description", help="Description.", show_default=False),
248
+ ] = None,
249
+ project: Project = None,
250
+ project_id: ProjectID = None,
251
+ profile_name: Profile = None,
252
+ output: Output = common.OutputChoice.table,
253
+ verbose: Verbose = False,
254
+ ):
255
+ simple = any(
256
+ value is not None for value in (rank_metric, rank_source, rank_direction)
257
+ )
258
+ if ranking_policy is not None and simple:
259
+ raise typer.BadParameter(
260
+ "--ranking-policy and simple ranking flags are mutually exclusive"
261
+ )
262
+ if ranking_policy is None and (rank_metric is None or rank_direction is None):
263
+ raise typer.BadParameter(
264
+ "--rank-metric and --rank-direction are required without --ranking-policy"
265
+ )
266
+ try:
267
+ if ranking_policy is not None:
268
+ ranking = _load_mapping(ranking_policy)
269
+ else:
270
+ ranking = {
271
+ "primary": {
272
+ "metric": {
273
+ "source": (rank_source or RankSource.aggregate_metrics).value,
274
+ "name": rank_metric,
275
+ },
276
+ "direction": rank_direction.value,
277
+ }
278
+ }
279
+ runner = _load_mapping(runner_config) if runner_config else None
280
+ gate = _load_mapping(gate_policy) if gate_policy else None
281
+ resolved_project = _project(project, project_id, profile_name)
282
+ with _client(profile_name) as client:
283
+ method_id = _resolve_method_id(client, method, project=resolved_project)
284
+ result = client.leaderboards.create(
285
+ name,
286
+ project=resolved_project,
287
+ evaluation_method_id=method_id,
288
+ ranking_policy=ranking,
289
+ runner_config=runner,
290
+ gate_policy=gate,
291
+ description=description,
292
+ )
293
+ except Exception as exc:
294
+ common.handle_cli_error(exc, verbose=verbose)
295
+ if output == common.OutputChoice.json:
296
+ common.print_json(result)
297
+ return
298
+ common.console.print(f"[bold green]Leaderboard created:[/] {escape(result.id)}")
299
+
300
+
301
+ @app.command("get")
302
+ def get_leaderboard(
303
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
304
+ project: Project = None,
305
+ project_id: ProjectID = None,
306
+ profile_name: Profile = None,
307
+ output: Output = common.OutputChoice.table,
308
+ verbose: Verbose = False,
309
+ ):
310
+ try:
311
+ resolved_project = _project(project, project_id, profile_name)
312
+ with _client(profile_name) as client:
313
+ leaderboard_id = _resolve_leaderboard_id(
314
+ client, leaderboard, project=resolved_project
315
+ )
316
+ result = client.leaderboards.get(leaderboard_id, project=resolved_project)
317
+ except Exception as exc:
318
+ common.handle_cli_error(exc, verbose=verbose)
319
+ _print_or_json(result, output, _print_board)
320
+
321
+
322
+ @app.command("update")
323
+ def update_leaderboard(
324
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
325
+ name: Annotated[
326
+ str | None, typer.Option("--name", help="New name.", show_default=False)
327
+ ] = None,
328
+ description: Annotated[
329
+ str | None,
330
+ typer.Option("--description", help="New description.", show_default=False),
331
+ ] = None,
332
+ project: Project = None,
333
+ project_id: ProjectID = None,
334
+ profile_name: Profile = None,
335
+ output: Output = common.OutputChoice.table,
336
+ verbose: Verbose = False,
337
+ ):
338
+ try:
339
+ resolved_project = _project(project, project_id, profile_name)
340
+ with _client(profile_name) as client:
341
+ leaderboard_id = _resolve_leaderboard_id(
342
+ client, leaderboard, project=resolved_project
343
+ )
344
+ result = client.leaderboards.update(
345
+ leaderboard_id,
346
+ project=resolved_project,
347
+ name=name,
348
+ description=description,
349
+ )
350
+ except Exception as exc:
351
+ common.handle_cli_error(exc, verbose=verbose)
352
+ _print_or_json(result, output, _print_board)
353
+
354
+
355
+ def _status_command(
356
+ leaderboard: str,
357
+ *,
358
+ action: str,
359
+ project: str | None,
360
+ project_id: int | None,
361
+ profile_name: str | None,
362
+ output: common.OutputChoice,
363
+ verbose: bool,
364
+ ) -> None:
365
+ try:
366
+ resolved_project = _project(project, project_id, profile_name)
367
+ with _client(profile_name) as client:
368
+ leaderboard_id = _resolve_leaderboard_id(
369
+ client, leaderboard, project=resolved_project
370
+ )
371
+ result = getattr(client.leaderboards, action)(
372
+ leaderboard_id, project=resolved_project
373
+ )
374
+ except Exception as exc:
375
+ common.handle_cli_error(exc, verbose=verbose)
376
+ _print_or_json(result, output, _print_board)
377
+
378
+
379
+ @app.command("archive")
380
+ def archive_leaderboard(
381
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
382
+ project: Project = None,
383
+ project_id: ProjectID = None,
384
+ profile_name: Profile = None,
385
+ output: Output = common.OutputChoice.table,
386
+ verbose: Verbose = False,
387
+ ):
388
+ _status_command(
389
+ leaderboard,
390
+ action="archive",
391
+ project=project,
392
+ project_id=project_id,
393
+ profile_name=profile_name,
394
+ output=output,
395
+ verbose=verbose,
396
+ )
397
+
398
+
399
+ @app.command("restore")
400
+ def restore_leaderboard(
401
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
402
+ project: Project = None,
403
+ project_id: ProjectID = None,
404
+ profile_name: Profile = None,
405
+ output: Output = common.OutputChoice.table,
406
+ verbose: Verbose = False,
407
+ ):
408
+ _status_command(
409
+ leaderboard,
410
+ action="restore",
411
+ project=project,
412
+ project_id=project_id,
413
+ profile_name=profile_name,
414
+ output=output,
415
+ verbose=verbose,
416
+ )
417
+
418
+
419
+ @app.command("entries")
420
+ def list_entries(
421
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
422
+ platform: Annotated[
423
+ str | None, typer.Option("--platform", help="gpu or rngd.", show_default=False)
424
+ ] = None,
425
+ accelerator_count: Annotated[
426
+ int | None,
427
+ typer.Option("--accelerator-count", min=1, show_default=False),
428
+ ] = None,
429
+ page: Page = None,
430
+ limit: Limit = None,
431
+ project: Project = None,
432
+ project_id: ProjectID = None,
433
+ profile_name: Profile = None,
434
+ output: Output = common.OutputChoice.table,
435
+ verbose: Verbose = False,
436
+ ):
437
+ try:
438
+ resolved_project = _project(project, project_id, profile_name)
439
+ with _client(profile_name) as client:
440
+ leaderboard_id = _resolve_leaderboard_id(
441
+ client, leaderboard, project=resolved_project
442
+ )
443
+ result_page = client.leaderboards.entries_page(
444
+ leaderboard_id,
445
+ project=resolved_project,
446
+ platform=platform,
447
+ accelerator_count=accelerator_count,
448
+ page=page,
449
+ limit=limit,
450
+ )
451
+ except Exception as exc:
452
+ common.handle_cli_error(exc, verbose=verbose)
453
+ if output == common.OutputChoice.json:
454
+ common.print_json(result_page)
455
+ return
456
+ common.print_page_table(
457
+ result_page,
458
+ [
459
+ "Rank",
460
+ "Submission",
461
+ "Artifact",
462
+ "Platform",
463
+ "Resources",
464
+ "Aggregate Metrics",
465
+ "Runtime Metrics",
466
+ ],
467
+ [
468
+ [
469
+ item.rank,
470
+ item.submission_id,
471
+ item.artifact.name,
472
+ item.artifact.platform,
473
+ f"{item.requested_resources.cpu}/{item.requested_resources.memory}/{item.requested_resources.accelerator_count}",
474
+ _metric_summary(item.aggregate_metrics),
475
+ _metric_summary(item.runtime_metrics),
476
+ ]
477
+ for item in result_page.items
478
+ ],
479
+ show_lines=True,
480
+ )
481
+
482
+
483
+ @submission_app.command("create")
484
+ def create_submission(
485
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
486
+ model_artifact_id: Annotated[
487
+ int | None,
488
+ typer.Option("--model-artifact-id", min=1, show_default=False),
489
+ ] = None,
490
+ model: Annotated[
491
+ str | None,
492
+ typer.Option("--model", help="Registered Model name.", show_default=False),
493
+ ] = None,
494
+ model_version: Annotated[
495
+ str | None,
496
+ typer.Option("--model-version", help="Model Version name.", show_default=False),
497
+ ] = None,
498
+ artifact: Annotated[
499
+ str | None,
500
+ typer.Option("--artifact", help="Model Artifact name.", show_default=False),
501
+ ] = None,
502
+ cpu: Annotated[
503
+ str | None,
504
+ typer.Option(
505
+ "--cpu", help="CPU quantity; server default 4.", show_default=False
506
+ ),
507
+ ] = None,
508
+ memory: Annotated[
509
+ str | None,
510
+ typer.Option(
511
+ "--memory", help="Memory quantity; server default 8Gi.", show_default=False
512
+ ),
513
+ ] = None,
514
+ accelerator_count: Annotated[
515
+ int | None,
516
+ typer.Option(
517
+ "--accelerator-count",
518
+ min=1,
519
+ help="Accelerator count; server default 1.",
520
+ show_default=False,
521
+ ),
522
+ ] = None,
523
+ wait: Annotated[
524
+ bool, typer.Option("--wait", help="Wait for terminal state.")
525
+ ] = False,
526
+ timeout: Annotated[
527
+ float,
528
+ typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
529
+ ] = 1800,
530
+ interval: Annotated[
531
+ float,
532
+ typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
533
+ ] = 5,
534
+ project: Project = None,
535
+ project_id: ProjectID = None,
536
+ profile_name: Profile = None,
537
+ output: Output = common.OutputChoice.table,
538
+ verbose: Verbose = False,
539
+ ):
540
+ common.validate_wait_options(timeout, interval)
541
+ try:
542
+ resources = (cpu, memory, accelerator_count)
543
+ if any(value is not None for value in resources) and not all(
544
+ value is not None for value in resources
545
+ ):
546
+ raise ValueError(
547
+ "cpu, memory, and accelerator_count must be provided together"
548
+ )
549
+ resolved_project = _project(project, project_id, profile_name)
550
+ with _client(profile_name) as client:
551
+ leaderboard_id = _resolve_leaderboard_id(
552
+ client, leaderboard, project=resolved_project
553
+ )
554
+ artifact_id = _resolve_model_artifact_id(
555
+ client,
556
+ project=resolved_project,
557
+ model_artifact_id=model_artifact_id,
558
+ model=model,
559
+ model_version=model_version,
560
+ artifact=artifact,
561
+ )
562
+ result = client.leaderboards.submit(
563
+ leaderboard_id,
564
+ project=resolved_project,
565
+ model_artifact_id=artifact_id,
566
+ cpu=cpu,
567
+ memory=memory,
568
+ accelerator_count=accelerator_count,
569
+ )
570
+ if wait:
571
+ typer.echo(
572
+ f"Leaderboard Submission created: {result.id}; resume with "
573
+ "'nufi leaderboard submission wait "
574
+ f"{leaderboard_id} {result.id}'",
575
+ err=True,
576
+ )
577
+ result = client.leaderboards.wait(
578
+ leaderboard_id,
579
+ result.id,
580
+ project=resolved_project,
581
+ timeout=timeout,
582
+ interval=interval,
583
+ )
584
+ except Exception as exc:
585
+ common.handle_cli_error(exc, verbose=verbose)
586
+ if wait:
587
+ _print_submission_output(result, output)
588
+ elif output == common.OutputChoice.json:
589
+ common.print_json(result)
590
+ else:
591
+ common.console.print(
592
+ f"[bold green]Leaderboard Submission created:[/] {escape(result.id)}"
593
+ )
594
+ if result.latest_run and result.latest_run.status in {"failed", "cancelled"}:
595
+ raise typer.Exit(code=1)
596
+
597
+
598
+ @submission_app.command("list")
599
+ def list_submissions(
600
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
601
+ page: Page = None,
602
+ limit: Limit = None,
603
+ project: Project = None,
604
+ project_id: ProjectID = None,
605
+ profile_name: Profile = None,
606
+ output: Output = common.OutputChoice.table,
607
+ verbose: Verbose = False,
608
+ ):
609
+ try:
610
+ resolved_project = _project(project, project_id, profile_name)
611
+ with _client(profile_name) as client:
612
+ leaderboard_id = _resolve_leaderboard_id(
613
+ client, leaderboard, project=resolved_project
614
+ )
615
+ result_page = client.leaderboards.list_submissions_page(
616
+ leaderboard_id, project=resolved_project, page=page, limit=limit
617
+ )
618
+ except Exception as exc:
619
+ common.handle_cli_error(exc, verbose=verbose)
620
+ if output == common.OutputChoice.json:
621
+ common.print_json(result_page)
622
+ return
623
+ common.print_page_table(
624
+ result_page,
625
+ ["ID", "Artifact", "Platform", "Resources", "Qualification", "Latest Run"],
626
+ [
627
+ [
628
+ item.id,
629
+ item.artifact.name,
630
+ item.artifact.platform,
631
+ f"{item.requested_resources.cpu}/{item.requested_resources.memory}/{item.requested_resources.accelerator_count}",
632
+ item.qualification,
633
+ item.latest_run.status if item.latest_run else "",
634
+ ]
635
+ for item in result_page.items
636
+ ],
637
+ )
638
+
639
+
640
+ @submission_app.command("get")
641
+ def get_submission(
642
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
643
+ submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
644
+ project: Project = None,
645
+ project_id: ProjectID = None,
646
+ profile_name: Profile = None,
647
+ output: Output = common.OutputChoice.table,
648
+ verbose: Verbose = False,
649
+ ):
650
+ try:
651
+ resolved_project = _project(project, project_id, profile_name)
652
+ with _client(profile_name) as client:
653
+ leaderboard_id = _resolve_leaderboard_id(
654
+ client, leaderboard, project=resolved_project
655
+ )
656
+ result = client.leaderboards.get_submission(
657
+ leaderboard_id, submission_id, project=resolved_project
658
+ )
659
+ except Exception as exc:
660
+ common.handle_cli_error(exc, verbose=verbose)
661
+ _print_submission_output(result, output)
662
+
663
+
664
+ @submission_app.command("wait")
665
+ def wait_submission(
666
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
667
+ submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
668
+ project: Project = None,
669
+ project_id: ProjectID = None,
670
+ timeout: Annotated[
671
+ float,
672
+ typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
673
+ ] = 1800,
674
+ interval: Annotated[
675
+ float,
676
+ typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
677
+ ] = 5,
678
+ profile_name: Profile = None,
679
+ output: Output = common.OutputChoice.table,
680
+ verbose: Verbose = False,
681
+ ):
682
+ common.validate_wait_options(timeout, interval)
683
+ try:
684
+ resolved_project = _project(project, project_id, profile_name)
685
+ with _client(profile_name) as client:
686
+ leaderboard_id = _resolve_leaderboard_id(
687
+ client, leaderboard, project=resolved_project
688
+ )
689
+ result = client.leaderboards.wait(
690
+ leaderboard_id,
691
+ submission_id,
692
+ project=resolved_project,
693
+ timeout=timeout,
694
+ interval=interval,
695
+ )
696
+ except Exception as exc:
697
+ common.handle_cli_error(exc, verbose=verbose)
698
+ _print_submission_output(result, output)
699
+ if result.latest_run and result.latest_run.status in {"failed", "cancelled"}:
700
+ raise typer.Exit(code=1)
701
+
702
+
703
+ @submission_app.command("rerun")
704
+ def rerun_submission(
705
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
706
+ submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
707
+ project: Project = None,
708
+ project_id: ProjectID = None,
709
+ profile_name: Profile = None,
710
+ output: Output = common.OutputChoice.table,
711
+ verbose: Verbose = False,
712
+ ):
713
+ try:
714
+ resolved_project = _project(project, project_id, profile_name)
715
+ with _client(profile_name) as client:
716
+ leaderboard_id = _resolve_leaderboard_id(
717
+ client, leaderboard, project=resolved_project
718
+ )
719
+ result = client.leaderboards.rerun(
720
+ leaderboard_id, submission_id, project=resolved_project
721
+ )
722
+ except Exception as exc:
723
+ common.handle_cli_error(exc, verbose=verbose)
724
+ _print_submission_output(result, output)
725
+
726
+
727
+ @submission_app.command("runs")
728
+ def list_submission_runs(
729
+ leaderboard: Annotated[str, typer.Argument(help="Leaderboard name or UUID.")],
730
+ submission_id: Annotated[str, typer.Argument(help="Submission UUID.")],
731
+ project: Project = None,
732
+ project_id: ProjectID = None,
733
+ profile_name: Profile = None,
734
+ output: Output = common.OutputChoice.table,
735
+ verbose: Verbose = False,
736
+ ):
737
+ try:
738
+ resolved_project = _project(project, project_id, profile_name)
739
+ with _client(profile_name) as client:
740
+ leaderboard_id = _resolve_leaderboard_id(
741
+ client, leaderboard, project=resolved_project
742
+ )
743
+ items = client.leaderboards.list_runs(
744
+ leaderboard_id, submission_id, project=resolved_project
745
+ )
746
+ except Exception as exc:
747
+ common.handle_cli_error(exc, verbose=verbose)
748
+ if output == common.OutputChoice.json:
749
+ common.print_json(items)
750
+ return
751
+ common.print_table(
752
+ ["Attempt", "Evaluation Run", "Status", "Created", "Completed"],
753
+ [
754
+ [
755
+ item.attempt_number,
756
+ item.evaluation_run_id,
757
+ item.status,
758
+ item.created_at,
759
+ item.completed_at or "",
760
+ ]
761
+ for item in items
762
+ ],
763
+ )