nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,854 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from collections.abc import Iterator
5
+ from contextlib import contextmanager
6
+ from pathlib import Path
7
+ from typing import Annotated, Any
8
+
9
+ import typer
10
+ import yaml
11
+ from rich.markup import escape
12
+
13
+ from nufi.cli.commands import common
14
+ from nufi.cli.commands.datasets import (
15
+ _is_uuid,
16
+ _resolve_dataset_id,
17
+ _resolve_version_id,
18
+ )
19
+ from nufi.exceptions import NotFoundError, ValidationError
20
+ from nufi.resources.custom_evaluation_templates import load_strict_mapping
21
+
22
+ app = typer.Typer(
23
+ help="Manage NuFi evaluations.",
24
+ invoke_without_command=True,
25
+ context_settings=common.CONTEXT_SETTINGS,
26
+ )
27
+ method_app = typer.Typer(
28
+ help="Manage Evaluation Methods.",
29
+ invoke_without_command=True,
30
+ context_settings=common.CONTEXT_SETTINGS,
31
+ )
32
+ run_app = typer.Typer(
33
+ help="Manage Evaluation Runs.",
34
+ invoke_without_command=True,
35
+ context_settings=common.CONTEXT_SETTINGS,
36
+ )
37
+ app.add_typer(method_app, name="method")
38
+ app.add_typer(run_app, name="run")
39
+
40
+ Project = Annotated[
41
+ str | None,
42
+ typer.Option("--project", help="Project name.", show_default=False),
43
+ ]
44
+ ProjectID = Annotated[
45
+ int | None,
46
+ typer.Option("--project-id", min=1, help="Project ID.", show_default=False),
47
+ ]
48
+ Profile = Annotated[str | None, typer.Option("--profile", help="Profile name.")]
49
+ Page = Annotated[
50
+ int | None, typer.Option("--page", min=1, help="Page number.", show_default=False)
51
+ ]
52
+ Limit = Annotated[
53
+ int | None,
54
+ typer.Option("--limit", min=1, max=100, help="Items per page.", show_default=False),
55
+ ]
56
+ Output = Annotated[
57
+ common.OutputChoice,
58
+ typer.Option("--output", help="Output format."),
59
+ ]
60
+ Verbose = Annotated[bool, typer.Option("--verbose", help="Show detailed errors.")]
61
+
62
+
63
+ @app.callback()
64
+ def evaluations(ctx: typer.Context):
65
+ common.show_help_if_no_subcommand(ctx)
66
+
67
+
68
+ @method_app.callback()
69
+ def evaluation_methods(ctx: typer.Context):
70
+ common.show_help_if_no_subcommand(ctx)
71
+
72
+
73
+ @run_app.callback()
74
+ def evaluation_runs(ctx: typer.Context):
75
+ common.show_help_if_no_subcommand(ctx)
76
+
77
+
78
+ @contextmanager
79
+ def _client(profile_name: str | None) -> Iterator[Any]:
80
+ client = common.make_client(profile_name)
81
+ try:
82
+ yield client
83
+ finally:
84
+ client.close()
85
+
86
+
87
+ def _project(
88
+ project: str | None, project_id: int | None, profile_name: str | None
89
+ ) -> str | int:
90
+ return common.resolve_cli_project(project, project_id, profile_name=profile_name)
91
+
92
+
93
+ def _load_mapping(path: Path, *, strict: bool = False) -> dict[str, Any]:
94
+ if strict:
95
+ return load_strict_mapping(path)
96
+ try:
97
+ payload = yaml.safe_load(path.read_text(encoding="utf-8"))
98
+ except (OSError, UnicodeError, yaml.YAMLError) as exc:
99
+ raise ValidationError(f"Could not read mapping file '{path}': {exc}") from exc
100
+ if not isinstance(payload, dict):
101
+ raise ValidationError(f"Mapping file '{path}' must contain a mapping.")
102
+ return payload
103
+
104
+
105
+ def _dataset_version_ids(
106
+ client: Any,
107
+ dataset: str | None,
108
+ version: str | None,
109
+ *,
110
+ project: str | int,
111
+ ) -> tuple[str | None, str | None]:
112
+ if version is not None and dataset is None:
113
+ raise ValidationError("--version requires --dataset.")
114
+ if dataset is None:
115
+ return None, None
116
+ dataset_id = _resolve_dataset_id(client, dataset, project=project)
117
+ if version is None:
118
+ return dataset_id, None
119
+ return dataset_id, _resolve_version_id(client, dataset_id, version, project=project)
120
+
121
+
122
+ def _resolve_method_id(
123
+ client: Any, method: str, *, project: str | int, archived: bool = False
124
+ ) -> str:
125
+ if _is_uuid(method):
126
+ return method
127
+ return common.resolve_named_resource(
128
+ client.evaluations.methods.iter_all(project=project, archived=archived),
129
+ method,
130
+ resource="Evaluation Method",
131
+ ).id
132
+
133
+
134
+ def _resolve_template_id(
135
+ client: Any,
136
+ template: str,
137
+ version: str | None,
138
+ *,
139
+ project: str | int,
140
+ ) -> str:
141
+ if template.startswith("nufi:") or _is_uuid(template):
142
+ if version is not None:
143
+ raise ValidationError(
144
+ "--template-version cannot be used with a template ID."
145
+ )
146
+ return template
147
+ candidates = [
148
+ item
149
+ for item in client.evaluations.methods.templates(project=project).templates
150
+ if item.name == template and (version is None or item.version == version)
151
+ ]
152
+ if len(candidates) == 1:
153
+ return candidates[0].id
154
+ if candidates:
155
+ ids = ", ".join(item.id for item in candidates)
156
+ raise ValidationError(
157
+ f"Multiple templates named '{template}'; use an ID or --template-version. "
158
+ f"Candidates: {ids}"
159
+ )
160
+ raise NotFoundError(f"Evaluation template not found: {template}")
161
+
162
+
163
+ def _resolve_model_artifact_id(
164
+ client: Any,
165
+ *,
166
+ project: str | int,
167
+ model_artifact_id: int | None,
168
+ model: str | None,
169
+ model_version: str | None,
170
+ artifact: str | None,
171
+ ) -> int:
172
+ names = (model, model_version, artifact)
173
+ if model_artifact_id is not None:
174
+ if any(names):
175
+ raise ValidationError(
176
+ "Use either --model-artifact-id or --model/--model-version/--artifact."
177
+ )
178
+ return model_artifact_id
179
+ if not all(names):
180
+ raise ValidationError(
181
+ "Provide --model, --model-version, and --artifact together."
182
+ )
183
+ return client.models.get_artifact(
184
+ model, model_version, artifact, project=project
185
+ ).id
186
+
187
+
188
+ def _method_rows(methods: list[Any]) -> list[list[str]]:
189
+ return [
190
+ [
191
+ item.id,
192
+ item.name,
193
+ item.type,
194
+ item.status,
195
+ item.dataset_id,
196
+ item.dataset_version_id or "",
197
+ ]
198
+ for item in methods
199
+ ]
200
+
201
+
202
+ def _print_method(method: Any) -> None:
203
+ common.print_table(
204
+ ["Field", "Value"],
205
+ [
206
+ ["ID", method.id],
207
+ ["Name", method.name],
208
+ ["Type", method.type],
209
+ ["Status", method.status],
210
+ ["Dataset ID", method.dataset_id],
211
+ ["Dataset Version ID", method.dataset_version_id or ""],
212
+ ["Description", method.description],
213
+ ["Config", json.dumps(method.config, ensure_ascii=False)],
214
+ ["Created", method.created_at],
215
+ ["Updated", method.updated_at],
216
+ ],
217
+ )
218
+
219
+
220
+ def _diagnostic_summary(run: Any) -> str:
221
+ diagnostics = run.diagnostic_samples
222
+ if diagnostics is None:
223
+ return ""
224
+ return (
225
+ f"{diagnostics.status}; items={len(diagnostics.items)}; "
226
+ f"omitted={diagnostics.omitted_row_count}; "
227
+ f"truncated={str(diagnostics.values_truncated).lower()}"
228
+ )
229
+
230
+
231
+ def _aggregate_summary(run: Any) -> str:
232
+ return ", ".join(
233
+ f"{metric.key}={metric.value}" for metric in (run.aggregate_metrics or [])
234
+ )
235
+
236
+
237
+ def _run_error_guidance(error: str | None) -> str:
238
+ if not error:
239
+ return ""
240
+ guidance = {
241
+ "credential could not be retrieved": (
242
+ "Ask a Kubernetes operator to provision the referenced pull Secret in "
243
+ "this project namespace."
244
+ ),
245
+ "image pull is backing off": (
246
+ "Check the image reference and project-namespace Registry credentials."
247
+ ),
248
+ "image pull failed": (
249
+ "Check that the image exists and the Registry credentials can pull it."
250
+ ),
251
+ "configuration is invalid": "Fix the Method evaluatorConfig and create a new Run.",
252
+ "invalid result": "Check that the evaluator returns every declared metric within bounds.",
253
+ "result submission failed": "Check the evaluator result submission and retry the Run.",
254
+ "result submission marker not found": (
255
+ "The evaluator exited without confirming result submission; inspect the image."
256
+ ),
257
+ }
258
+ lowered = error.lower()
259
+ for fragment, action in guidance.items():
260
+ if fragment in lowered:
261
+ return action
262
+ return error
263
+
264
+
265
+ def _print_run(run: Any) -> None:
266
+ resources = run.target_resources
267
+ common.print_table(
268
+ ["Field", "Value"],
269
+ [
270
+ ["ID", run.id],
271
+ ["Method ID", run.evaluation_method_id or ""],
272
+ ["Dataset ID", run.dataset_id or ""],
273
+ ["Dataset Version ID", run.dataset_version_id or ""],
274
+ ["Model Artifact ID", run.model_artifact_id],
275
+ ["Status", run.status],
276
+ ["Runner Type", run.runner_type],
277
+ ["Runner Schema", run.runner_config_schema_version],
278
+ *(
279
+ [
280
+ ["Template Source", run.template_source],
281
+ ["Requested Image", run.runner_image or ""],
282
+ ["Image Policy", run.image_policy or ""],
283
+ ["Resolved Image ID", run.resolved_image_id or ""],
284
+ ]
285
+ if run.template_source == "custom"
286
+ else []
287
+ ),
288
+ ["CPU", resources.cpu if resources else ""],
289
+ ["Memory", resources.memory if resources else ""],
290
+ ["Accelerators", resources.accelerator_count if resources else ""],
291
+ ["Aggregate Metrics", _aggregate_summary(run)],
292
+ ["Diagnostics", _diagnostic_summary(run)],
293
+ *(
294
+ [
295
+ [
296
+ "Failure",
297
+ common.bounded_error(_run_error_guidance(run.error_message)),
298
+ ]
299
+ ]
300
+ if run.error_message
301
+ else []
302
+ ),
303
+ ["Created", run.created_at],
304
+ ["Completed", run.completed_at or ""],
305
+ ],
306
+ )
307
+
308
+
309
+ def _print_run_output(run: Any, output: common.OutputChoice) -> None:
310
+ if output == common.OutputChoice.json:
311
+ payload = common.serialize(run)
312
+ if payload.get("errorMessage"):
313
+ payload["errorMessage"] = common.bounded_error(payload["errorMessage"])
314
+ common.print_json(payload)
315
+ else:
316
+ _print_run(run)
317
+
318
+
319
+ @method_app.command("templates", help="List available Evaluation Method templates.")
320
+ def method_templates(
321
+ project: Project = None,
322
+ project_id: ProjectID = None,
323
+ profile_name: Profile = None,
324
+ output: Output = common.OutputChoice.table,
325
+ verbose: Verbose = False,
326
+ ):
327
+ try:
328
+ resolved_project = _project(project, project_id, profile_name)
329
+ with _client(profile_name) as client:
330
+ templates = client.evaluations.methods.templates(project=resolved_project)
331
+ except Exception as exc:
332
+ common.handle_cli_error(exc, verbose=verbose)
333
+
334
+ if output == common.OutputChoice.json:
335
+ common.print_json(templates)
336
+ return
337
+ if not templates.templates:
338
+ common.print_table(
339
+ ["Name", "Display Name", "Output Type", "Metrics"],
340
+ [
341
+ [
342
+ item.name,
343
+ item.display_name,
344
+ item.output_type,
345
+ ", ".join(item.metrics),
346
+ ]
347
+ for item in templates.method_v2.templates
348
+ ],
349
+ )
350
+ return
351
+ common.print_table(
352
+ ["ID", "Source", "Name", "Version", "Image Policy", "Metrics"],
353
+ [
354
+ [
355
+ item.id,
356
+ item.source,
357
+ item.name,
358
+ item.version or "",
359
+ item.image_policy or "",
360
+ ", ".join(metric.key for metric in item.metrics),
361
+ ]
362
+ for item in templates.templates
363
+ ],
364
+ )
365
+
366
+
367
+ @method_app.command("list")
368
+ def list_methods(
369
+ dataset: Annotated[
370
+ str | None,
371
+ typer.Option("--dataset", help="Dataset name or UUID.", show_default=False),
372
+ ] = None,
373
+ version: Annotated[
374
+ str | None,
375
+ typer.Option(
376
+ "--version", help="Dataset version name or UUID.", show_default=False
377
+ ),
378
+ ] = None,
379
+ page: Page = None,
380
+ limit: Limit = None,
381
+ project: Project = None,
382
+ project_id: ProjectID = None,
383
+ profile_name: Profile = None,
384
+ output: Output = common.OutputChoice.table,
385
+ verbose: Verbose = False,
386
+ ):
387
+ try:
388
+ resolved_project = _project(project, project_id, profile_name)
389
+ with _client(profile_name) as client:
390
+ dataset_id, version_id = _dataset_version_ids(
391
+ client, dataset, version, project=resolved_project
392
+ )
393
+ result_page = client.evaluations.methods.list_page(
394
+ project=resolved_project,
395
+ dataset_id=dataset_id,
396
+ dataset_version_id=version_id,
397
+ page=page,
398
+ limit=limit,
399
+ )
400
+ except Exception as exc:
401
+ common.handle_cli_error(exc, verbose=verbose)
402
+
403
+ if output == common.OutputChoice.json:
404
+ common.print_json(result_page)
405
+ return
406
+ common.print_page_table(
407
+ result_page,
408
+ ["ID", "Name", "Type", "Status", "Dataset ID", "Version ID"],
409
+ _method_rows(result_page.items),
410
+ )
411
+
412
+
413
+ @method_app.command("get")
414
+ def get_method(
415
+ method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
416
+ project: Project = None,
417
+ project_id: ProjectID = None,
418
+ profile_name: Profile = None,
419
+ output: Output = common.OutputChoice.table,
420
+ verbose: Verbose = False,
421
+ ):
422
+ try:
423
+ resolved_project = _project(project, project_id, profile_name)
424
+ with _client(profile_name) as client:
425
+ method_id = _resolve_method_id(client, method, project=resolved_project)
426
+ result = client.evaluations.methods.get(method_id, project=resolved_project)
427
+ except Exception as exc:
428
+ common.handle_cli_error(exc, verbose=verbose)
429
+
430
+ if output == common.OutputChoice.json:
431
+ common.print_json(result)
432
+ return
433
+ _print_method(result)
434
+
435
+
436
+ @method_app.command("validate")
437
+ def validate_template_method(
438
+ dataset: Annotated[str, typer.Argument(help="Dataset name or UUID.")],
439
+ version: Annotated[str, typer.Argument(help="Dataset version name or UUID.")],
440
+ config: Annotated[Path, typer.Option("--config", help="JSON or YAML config file.")],
441
+ template: Annotated[str, typer.Option("--template", help="Template name or ID.")],
442
+ template_version: Annotated[
443
+ str | None,
444
+ typer.Option(
445
+ "--template-version", help="Template version.", show_default=False
446
+ ),
447
+ ] = None,
448
+ project: Project = None,
449
+ project_id: ProjectID = None,
450
+ profile_name: Profile = None,
451
+ output: Output = common.OutputChoice.table,
452
+ verbose: Verbose = False,
453
+ ):
454
+ try:
455
+ loaded_config = _load_mapping(config, strict=True)
456
+ resolved_project = _project(project, project_id, profile_name)
457
+ with _client(profile_name) as client:
458
+ template_id = _resolve_template_id(
459
+ client, template, template_version, project=resolved_project
460
+ )
461
+ dataset_id, version_id = _dataset_version_ids(
462
+ client, dataset, version, project=resolved_project
463
+ )
464
+ result = client.evaluations.methods.preview(
465
+ dataset_id,
466
+ version_id,
467
+ project=resolved_project,
468
+ config=loaded_config,
469
+ template_id=template_id,
470
+ )
471
+ except Exception as exc:
472
+ common.handle_cli_error(exc, verbose=verbose)
473
+
474
+ if output == common.OutputChoice.json:
475
+ common.print_json(result)
476
+ return
477
+ common.print_table(
478
+ ["Field", "Value"],
479
+ [
480
+ ["Source Schema Hash", result.source_schema_hash],
481
+ ["Config", json.dumps(result.config, ensure_ascii=False)],
482
+ ],
483
+ )
484
+
485
+
486
+ @method_app.command("create")
487
+ def create_template_method(
488
+ dataset: Annotated[str, typer.Argument(help="Dataset name or UUID.")],
489
+ version: Annotated[str, typer.Argument(help="Dataset version name or UUID.")],
490
+ name: Annotated[str, typer.Argument(help="Evaluation Method name.")],
491
+ config: Annotated[Path, typer.Option("--config", help="JSON or YAML config file.")],
492
+ template: Annotated[str, typer.Option("--template", help="Template name or ID.")],
493
+ template_version: Annotated[
494
+ str | None,
495
+ typer.Option(
496
+ "--template-version", help="Template version.", show_default=False
497
+ ),
498
+ ] = None,
499
+ default_runtime: Annotated[
500
+ Path | None,
501
+ typer.Option(
502
+ "--default-runtime",
503
+ help="JSON or YAML default runtime file.",
504
+ show_default=False,
505
+ ),
506
+ ] = None,
507
+ description: Annotated[
508
+ str | None,
509
+ typer.Option("--description", help="Description.", show_default=False),
510
+ ] = None,
511
+ project: Project = None,
512
+ project_id: ProjectID = None,
513
+ profile_name: Profile = None,
514
+ verbose: Verbose = False,
515
+ ):
516
+ try:
517
+ loaded_config = _load_mapping(config, strict=True)
518
+ loaded_runtime = _load_mapping(default_runtime) if default_runtime else None
519
+ resolved_project = _project(project, project_id, profile_name)
520
+ with _client(profile_name) as client:
521
+ template_id = _resolve_template_id(
522
+ client, template, template_version, project=resolved_project
523
+ )
524
+ dataset_id, version_id = _dataset_version_ids(
525
+ client, dataset, version, project=resolved_project
526
+ )
527
+ result = client.evaluations.methods.create(
528
+ dataset_id,
529
+ version_id,
530
+ name,
531
+ project=resolved_project,
532
+ config=loaded_config,
533
+ template_id=template_id,
534
+ default_runtime=loaded_runtime,
535
+ description=description,
536
+ )
537
+ except Exception as exc:
538
+ common.handle_cli_error(exc, verbose=verbose)
539
+
540
+ common.console.print(
541
+ f"[bold green]Evaluation Method created:[/] {escape(result.id)}"
542
+ )
543
+
544
+
545
+ @method_app.command("archive")
546
+ def archive_method(
547
+ method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
548
+ reason: Annotated[
549
+ str | None,
550
+ typer.Option("--reason", help="Archive reason.", show_default=False),
551
+ ] = None,
552
+ project: Project = None,
553
+ project_id: ProjectID = None,
554
+ profile_name: Profile = None,
555
+ yes: Annotated[
556
+ bool, typer.Option("--yes", help="Skip archive confirmation.")
557
+ ] = False,
558
+ verbose: Verbose = False,
559
+ ):
560
+ try:
561
+ resolved_project = _project(project, project_id, profile_name)
562
+ with _client(profile_name) as client:
563
+ method_id = _resolve_method_id(client, method, project=resolved_project)
564
+ common.confirm_delete(
565
+ resource="Evaluation Method",
566
+ target=method,
567
+ profile_name=profile_name,
568
+ yes=yes,
569
+ )
570
+ client.evaluations.methods.archive(
571
+ method_id, project=resolved_project, reason=reason
572
+ )
573
+ except typer.Exit:
574
+ raise
575
+ except Exception as exc:
576
+ common.handle_cli_error(exc, verbose=verbose)
577
+ common.console.print(
578
+ f"[bold green]Evaluation Method archived:[/] {escape(method_id)}"
579
+ )
580
+
581
+
582
+ @method_app.command("restore")
583
+ def restore_method(
584
+ method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
585
+ project: Project = None,
586
+ project_id: ProjectID = None,
587
+ profile_name: Profile = None,
588
+ verbose: Verbose = False,
589
+ ):
590
+ try:
591
+ resolved_project = _project(project, project_id, profile_name)
592
+ with _client(profile_name) as client:
593
+ method_id = _resolve_method_id(
594
+ client, method, project=resolved_project, archived=True
595
+ )
596
+ client.evaluations.methods.restore(method_id, project=resolved_project)
597
+ except Exception as exc:
598
+ common.handle_cli_error(exc, verbose=verbose)
599
+ common.console.print(
600
+ f"[bold green]Evaluation Method restored:[/] {escape(method_id)}"
601
+ )
602
+
603
+
604
+ @run_app.command("list")
605
+ def list_runs(
606
+ dataset: Annotated[
607
+ str | None,
608
+ typer.Option("--dataset", help="Dataset name or UUID.", show_default=False),
609
+ ] = None,
610
+ version: Annotated[
611
+ str | None,
612
+ typer.Option(
613
+ "--version", help="Dataset version name or UUID.", show_default=False
614
+ ),
615
+ ] = None,
616
+ model_artifact_id: Annotated[
617
+ int | None,
618
+ typer.Option(
619
+ "--model-artifact-id",
620
+ min=1,
621
+ help="Model artifact ID.",
622
+ show_default=False,
623
+ ),
624
+ ] = None,
625
+ status: Annotated[
626
+ str | None,
627
+ typer.Option("--status", help="Run status.", show_default=False),
628
+ ] = None,
629
+ page: Page = None,
630
+ limit: Limit = None,
631
+ project: Project = None,
632
+ project_id: ProjectID = None,
633
+ profile_name: Profile = None,
634
+ output: Output = common.OutputChoice.table,
635
+ verbose: Verbose = False,
636
+ ):
637
+ try:
638
+ resolved_project = _project(project, project_id, profile_name)
639
+ with _client(profile_name) as client:
640
+ dataset_id, version_id = _dataset_version_ids(
641
+ client, dataset, version, project=resolved_project
642
+ )
643
+ result_page = client.evaluations.list_page(
644
+ project=resolved_project,
645
+ dataset_id=dataset_id,
646
+ dataset_version_id=version_id,
647
+ model_artifact_id=model_artifact_id,
648
+ status=status,
649
+ page=page,
650
+ limit=limit,
651
+ )
652
+ except Exception as exc:
653
+ common.handle_cli_error(exc, verbose=verbose)
654
+
655
+ if output == common.OutputChoice.json:
656
+ common.print_json(result_page)
657
+ return
658
+ common.print_page_table(
659
+ result_page,
660
+ ["ID", "Method ID", "Model Artifact ID", "Status", "Runner", "Created"],
661
+ [
662
+ [
663
+ item.id,
664
+ item.evaluation_method_id or "",
665
+ item.model_artifact_id,
666
+ item.status,
667
+ item.runner_type,
668
+ item.created_at,
669
+ ]
670
+ for item in result_page.items
671
+ ],
672
+ )
673
+
674
+
675
+ @run_app.command("get")
676
+ def get_run(
677
+ run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
678
+ project: Project = None,
679
+ project_id: ProjectID = None,
680
+ profile_name: Profile = None,
681
+ output: Output = common.OutputChoice.table,
682
+ verbose: Verbose = False,
683
+ ):
684
+ try:
685
+ resolved_project = _project(project, project_id, profile_name)
686
+ with _client(profile_name) as client:
687
+ result = client.evaluations.get(run_id, project=resolved_project)
688
+ except Exception as exc:
689
+ common.handle_cli_error(exc, verbose=verbose)
690
+ _print_run_output(result, output)
691
+
692
+
693
+ @run_app.command("wait")
694
+ def wait_run(
695
+ run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
696
+ project: Project = None,
697
+ project_id: ProjectID = None,
698
+ timeout: Annotated[
699
+ float,
700
+ typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
701
+ ] = 1800,
702
+ interval: Annotated[
703
+ float,
704
+ typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
705
+ ] = 5,
706
+ profile_name: Profile = None,
707
+ output: Output = common.OutputChoice.table,
708
+ verbose: Verbose = False,
709
+ ):
710
+ common.validate_wait_options(timeout, interval)
711
+ try:
712
+ resolved_project = _project(project, project_id, profile_name)
713
+ with _client(profile_name) as client:
714
+ result = client.evaluations.wait(
715
+ run_id,
716
+ project=resolved_project,
717
+ timeout=timeout,
718
+ interval=interval,
719
+ )
720
+ except Exception as exc:
721
+ common.handle_cli_error(exc, verbose=verbose)
722
+ _print_run_output(result, output)
723
+ if result.status in {"failed", "cancelled"}:
724
+ raise typer.Exit(code=1)
725
+
726
+
727
+ @run_app.command("create")
728
+ def create_run(
729
+ method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
730
+ accelerator_count: Annotated[
731
+ int,
732
+ typer.Option("--accelerator-count", min=0, help="Accelerator count."),
733
+ ] = 1,
734
+ cpu: Annotated[str, typer.Option("--cpu", help="CPU quantity.")] = "4",
735
+ memory: Annotated[str, typer.Option("--memory", help="Memory quantity.")] = "8Gi",
736
+ model_artifact_id: Annotated[
737
+ int | None,
738
+ typer.Option(
739
+ "--model-artifact-id", min=1, help="Model artifact ID.", show_default=False
740
+ ),
741
+ ] = None,
742
+ model: Annotated[
743
+ str | None,
744
+ typer.Option("--model", help="Registered Model name.", show_default=False),
745
+ ] = None,
746
+ model_version: Annotated[
747
+ str | None,
748
+ typer.Option("--model-version", help="Model Version name.", show_default=False),
749
+ ] = None,
750
+ artifact: Annotated[
751
+ str | None,
752
+ typer.Option("--artifact", help="Model Artifact name.", show_default=False),
753
+ ] = None,
754
+ runner_config: Annotated[
755
+ Path | None,
756
+ typer.Option(
757
+ "--runner-config",
758
+ help="JSON or YAML runner config file.",
759
+ show_default=False,
760
+ ),
761
+ ] = None,
762
+ wait: Annotated[
763
+ bool, typer.Option("--wait", help="Wait for terminal state.")
764
+ ] = False,
765
+ timeout: Annotated[
766
+ float,
767
+ typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
768
+ ] = 1800,
769
+ interval: Annotated[
770
+ float,
771
+ typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
772
+ ] = 5,
773
+ project: Project = None,
774
+ project_id: ProjectID = None,
775
+ profile_name: Profile = None,
776
+ output: Output = common.OutputChoice.table,
777
+ verbose: Verbose = False,
778
+ ):
779
+ common.validate_wait_options(timeout, interval)
780
+ try:
781
+ loaded_config = _load_mapping(runner_config) if runner_config else None
782
+ resolved_project = _project(project, project_id, profile_name)
783
+ with _client(profile_name) as client:
784
+ method_id = _resolve_method_id(client, method, project=resolved_project)
785
+ resolved_artifact_id = _resolve_model_artifact_id(
786
+ client,
787
+ project=resolved_project,
788
+ model_artifact_id=model_artifact_id,
789
+ model=model,
790
+ model_version=model_version,
791
+ artifact=artifact,
792
+ )
793
+ result = client.evaluations.create(
794
+ project=resolved_project,
795
+ evaluation_method_id=method_id,
796
+ model_artifact_id=resolved_artifact_id,
797
+ cpu=cpu,
798
+ memory=memory,
799
+ accelerator_count=accelerator_count,
800
+ runner_config=loaded_config,
801
+ )
802
+ if wait:
803
+ typer.echo(
804
+ f"Evaluation Run created: {result.id}; resume with "
805
+ f"'nufi evaluation run wait {result.id}'",
806
+ err=True,
807
+ )
808
+ result = client.evaluations.wait(
809
+ result.id,
810
+ project=resolved_project,
811
+ timeout=timeout,
812
+ interval=interval,
813
+ )
814
+ except Exception as exc:
815
+ common.handle_cli_error(exc, verbose=verbose)
816
+ if wait or output == common.OutputChoice.json:
817
+ _print_run_output(result, output)
818
+ else:
819
+ common.console.print(
820
+ f"[bold green]Evaluation Run created:[/] {escape(result.id)}"
821
+ )
822
+ if result.status in {"failed", "cancelled"}:
823
+ raise typer.Exit(code=1)
824
+
825
+
826
+ @run_app.command("cancel")
827
+ def cancel_run(
828
+ run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
829
+ project: Project = None,
830
+ project_id: ProjectID = None,
831
+ profile_name: Profile = None,
832
+ yes: Annotated[
833
+ bool, typer.Option("--yes", help="Skip cancel confirmation.")
834
+ ] = False,
835
+ verbose: Verbose = False,
836
+ ):
837
+ try:
838
+ resolved_project = _project(project, project_id, profile_name)
839
+ except Exception as exc:
840
+ common.handle_cli_error(exc, verbose=verbose)
841
+ common.confirm_delete(
842
+ resource="Evaluation Run",
843
+ target=run_id,
844
+ profile_name=profile_name,
845
+ yes=yes,
846
+ )
847
+ try:
848
+ with _client(profile_name) as client:
849
+ result = client.evaluations.cancel(run_id, project=resolved_project)
850
+ except Exception as exc:
851
+ common.handle_cli_error(exc, verbose=verbose)
852
+ common.console.print(
853
+ f"[bold green]Evaluation Run cancelled:[/] {escape(result.id)}"
854
+ )