nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,854 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated, Any
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
import yaml
|
|
11
|
+
from rich.markup import escape
|
|
12
|
+
|
|
13
|
+
from nufi.cli.commands import common
|
|
14
|
+
from nufi.cli.commands.datasets import (
|
|
15
|
+
_is_uuid,
|
|
16
|
+
_resolve_dataset_id,
|
|
17
|
+
_resolve_version_id,
|
|
18
|
+
)
|
|
19
|
+
from nufi.exceptions import NotFoundError, ValidationError
|
|
20
|
+
from nufi.resources.custom_evaluation_templates import load_strict_mapping
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(
|
|
23
|
+
help="Manage NuFi evaluations.",
|
|
24
|
+
invoke_without_command=True,
|
|
25
|
+
context_settings=common.CONTEXT_SETTINGS,
|
|
26
|
+
)
|
|
27
|
+
method_app = typer.Typer(
|
|
28
|
+
help="Manage Evaluation Methods.",
|
|
29
|
+
invoke_without_command=True,
|
|
30
|
+
context_settings=common.CONTEXT_SETTINGS,
|
|
31
|
+
)
|
|
32
|
+
run_app = typer.Typer(
|
|
33
|
+
help="Manage Evaluation Runs.",
|
|
34
|
+
invoke_without_command=True,
|
|
35
|
+
context_settings=common.CONTEXT_SETTINGS,
|
|
36
|
+
)
|
|
37
|
+
app.add_typer(method_app, name="method")
|
|
38
|
+
app.add_typer(run_app, name="run")
|
|
39
|
+
|
|
40
|
+
Project = Annotated[
|
|
41
|
+
str | None,
|
|
42
|
+
typer.Option("--project", help="Project name.", show_default=False),
|
|
43
|
+
]
|
|
44
|
+
ProjectID = Annotated[
|
|
45
|
+
int | None,
|
|
46
|
+
typer.Option("--project-id", min=1, help="Project ID.", show_default=False),
|
|
47
|
+
]
|
|
48
|
+
Profile = Annotated[str | None, typer.Option("--profile", help="Profile name.")]
|
|
49
|
+
Page = Annotated[
|
|
50
|
+
int | None, typer.Option("--page", min=1, help="Page number.", show_default=False)
|
|
51
|
+
]
|
|
52
|
+
Limit = Annotated[
|
|
53
|
+
int | None,
|
|
54
|
+
typer.Option("--limit", min=1, max=100, help="Items per page.", show_default=False),
|
|
55
|
+
]
|
|
56
|
+
Output = Annotated[
|
|
57
|
+
common.OutputChoice,
|
|
58
|
+
typer.Option("--output", help="Output format."),
|
|
59
|
+
]
|
|
60
|
+
Verbose = Annotated[bool, typer.Option("--verbose", help="Show detailed errors.")]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@app.callback()
|
|
64
|
+
def evaluations(ctx: typer.Context):
|
|
65
|
+
common.show_help_if_no_subcommand(ctx)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@method_app.callback()
|
|
69
|
+
def evaluation_methods(ctx: typer.Context):
|
|
70
|
+
common.show_help_if_no_subcommand(ctx)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@run_app.callback()
|
|
74
|
+
def evaluation_runs(ctx: typer.Context):
|
|
75
|
+
common.show_help_if_no_subcommand(ctx)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@contextmanager
|
|
79
|
+
def _client(profile_name: str | None) -> Iterator[Any]:
|
|
80
|
+
client = common.make_client(profile_name)
|
|
81
|
+
try:
|
|
82
|
+
yield client
|
|
83
|
+
finally:
|
|
84
|
+
client.close()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _project(
|
|
88
|
+
project: str | None, project_id: int | None, profile_name: str | None
|
|
89
|
+
) -> str | int:
|
|
90
|
+
return common.resolve_cli_project(project, project_id, profile_name=profile_name)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _load_mapping(path: Path, *, strict: bool = False) -> dict[str, Any]:
|
|
94
|
+
if strict:
|
|
95
|
+
return load_strict_mapping(path)
|
|
96
|
+
try:
|
|
97
|
+
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
98
|
+
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
|
99
|
+
raise ValidationError(f"Could not read mapping file '{path}': {exc}") from exc
|
|
100
|
+
if not isinstance(payload, dict):
|
|
101
|
+
raise ValidationError(f"Mapping file '{path}' must contain a mapping.")
|
|
102
|
+
return payload
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _dataset_version_ids(
|
|
106
|
+
client: Any,
|
|
107
|
+
dataset: str | None,
|
|
108
|
+
version: str | None,
|
|
109
|
+
*,
|
|
110
|
+
project: str | int,
|
|
111
|
+
) -> tuple[str | None, str | None]:
|
|
112
|
+
if version is not None and dataset is None:
|
|
113
|
+
raise ValidationError("--version requires --dataset.")
|
|
114
|
+
if dataset is None:
|
|
115
|
+
return None, None
|
|
116
|
+
dataset_id = _resolve_dataset_id(client, dataset, project=project)
|
|
117
|
+
if version is None:
|
|
118
|
+
return dataset_id, None
|
|
119
|
+
return dataset_id, _resolve_version_id(client, dataset_id, version, project=project)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _resolve_method_id(
|
|
123
|
+
client: Any, method: str, *, project: str | int, archived: bool = False
|
|
124
|
+
) -> str:
|
|
125
|
+
if _is_uuid(method):
|
|
126
|
+
return method
|
|
127
|
+
return common.resolve_named_resource(
|
|
128
|
+
client.evaluations.methods.iter_all(project=project, archived=archived),
|
|
129
|
+
method,
|
|
130
|
+
resource="Evaluation Method",
|
|
131
|
+
).id
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _resolve_template_id(
|
|
135
|
+
client: Any,
|
|
136
|
+
template: str,
|
|
137
|
+
version: str | None,
|
|
138
|
+
*,
|
|
139
|
+
project: str | int,
|
|
140
|
+
) -> str:
|
|
141
|
+
if template.startswith("nufi:") or _is_uuid(template):
|
|
142
|
+
if version is not None:
|
|
143
|
+
raise ValidationError(
|
|
144
|
+
"--template-version cannot be used with a template ID."
|
|
145
|
+
)
|
|
146
|
+
return template
|
|
147
|
+
candidates = [
|
|
148
|
+
item
|
|
149
|
+
for item in client.evaluations.methods.templates(project=project).templates
|
|
150
|
+
if item.name == template and (version is None or item.version == version)
|
|
151
|
+
]
|
|
152
|
+
if len(candidates) == 1:
|
|
153
|
+
return candidates[0].id
|
|
154
|
+
if candidates:
|
|
155
|
+
ids = ", ".join(item.id for item in candidates)
|
|
156
|
+
raise ValidationError(
|
|
157
|
+
f"Multiple templates named '{template}'; use an ID or --template-version. "
|
|
158
|
+
f"Candidates: {ids}"
|
|
159
|
+
)
|
|
160
|
+
raise NotFoundError(f"Evaluation template not found: {template}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _resolve_model_artifact_id(
|
|
164
|
+
client: Any,
|
|
165
|
+
*,
|
|
166
|
+
project: str | int,
|
|
167
|
+
model_artifact_id: int | None,
|
|
168
|
+
model: str | None,
|
|
169
|
+
model_version: str | None,
|
|
170
|
+
artifact: str | None,
|
|
171
|
+
) -> int:
|
|
172
|
+
names = (model, model_version, artifact)
|
|
173
|
+
if model_artifact_id is not None:
|
|
174
|
+
if any(names):
|
|
175
|
+
raise ValidationError(
|
|
176
|
+
"Use either --model-artifact-id or --model/--model-version/--artifact."
|
|
177
|
+
)
|
|
178
|
+
return model_artifact_id
|
|
179
|
+
if not all(names):
|
|
180
|
+
raise ValidationError(
|
|
181
|
+
"Provide --model, --model-version, and --artifact together."
|
|
182
|
+
)
|
|
183
|
+
return client.models.get_artifact(
|
|
184
|
+
model, model_version, artifact, project=project
|
|
185
|
+
).id
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _method_rows(methods: list[Any]) -> list[list[str]]:
|
|
189
|
+
return [
|
|
190
|
+
[
|
|
191
|
+
item.id,
|
|
192
|
+
item.name,
|
|
193
|
+
item.type,
|
|
194
|
+
item.status,
|
|
195
|
+
item.dataset_id,
|
|
196
|
+
item.dataset_version_id or "",
|
|
197
|
+
]
|
|
198
|
+
for item in methods
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _print_method(method: Any) -> None:
|
|
203
|
+
common.print_table(
|
|
204
|
+
["Field", "Value"],
|
|
205
|
+
[
|
|
206
|
+
["ID", method.id],
|
|
207
|
+
["Name", method.name],
|
|
208
|
+
["Type", method.type],
|
|
209
|
+
["Status", method.status],
|
|
210
|
+
["Dataset ID", method.dataset_id],
|
|
211
|
+
["Dataset Version ID", method.dataset_version_id or ""],
|
|
212
|
+
["Description", method.description],
|
|
213
|
+
["Config", json.dumps(method.config, ensure_ascii=False)],
|
|
214
|
+
["Created", method.created_at],
|
|
215
|
+
["Updated", method.updated_at],
|
|
216
|
+
],
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _diagnostic_summary(run: Any) -> str:
|
|
221
|
+
diagnostics = run.diagnostic_samples
|
|
222
|
+
if diagnostics is None:
|
|
223
|
+
return ""
|
|
224
|
+
return (
|
|
225
|
+
f"{diagnostics.status}; items={len(diagnostics.items)}; "
|
|
226
|
+
f"omitted={diagnostics.omitted_row_count}; "
|
|
227
|
+
f"truncated={str(diagnostics.values_truncated).lower()}"
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _aggregate_summary(run: Any) -> str:
|
|
232
|
+
return ", ".join(
|
|
233
|
+
f"{metric.key}={metric.value}" for metric in (run.aggregate_metrics or [])
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _run_error_guidance(error: str | None) -> str:
|
|
238
|
+
if not error:
|
|
239
|
+
return ""
|
|
240
|
+
guidance = {
|
|
241
|
+
"credential could not be retrieved": (
|
|
242
|
+
"Ask a Kubernetes operator to provision the referenced pull Secret in "
|
|
243
|
+
"this project namespace."
|
|
244
|
+
),
|
|
245
|
+
"image pull is backing off": (
|
|
246
|
+
"Check the image reference and project-namespace Registry credentials."
|
|
247
|
+
),
|
|
248
|
+
"image pull failed": (
|
|
249
|
+
"Check that the image exists and the Registry credentials can pull it."
|
|
250
|
+
),
|
|
251
|
+
"configuration is invalid": "Fix the Method evaluatorConfig and create a new Run.",
|
|
252
|
+
"invalid result": "Check that the evaluator returns every declared metric within bounds.",
|
|
253
|
+
"result submission failed": "Check the evaluator result submission and retry the Run.",
|
|
254
|
+
"result submission marker not found": (
|
|
255
|
+
"The evaluator exited without confirming result submission; inspect the image."
|
|
256
|
+
),
|
|
257
|
+
}
|
|
258
|
+
lowered = error.lower()
|
|
259
|
+
for fragment, action in guidance.items():
|
|
260
|
+
if fragment in lowered:
|
|
261
|
+
return action
|
|
262
|
+
return error
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _print_run(run: Any) -> None:
|
|
266
|
+
resources = run.target_resources
|
|
267
|
+
common.print_table(
|
|
268
|
+
["Field", "Value"],
|
|
269
|
+
[
|
|
270
|
+
["ID", run.id],
|
|
271
|
+
["Method ID", run.evaluation_method_id or ""],
|
|
272
|
+
["Dataset ID", run.dataset_id or ""],
|
|
273
|
+
["Dataset Version ID", run.dataset_version_id or ""],
|
|
274
|
+
["Model Artifact ID", run.model_artifact_id],
|
|
275
|
+
["Status", run.status],
|
|
276
|
+
["Runner Type", run.runner_type],
|
|
277
|
+
["Runner Schema", run.runner_config_schema_version],
|
|
278
|
+
*(
|
|
279
|
+
[
|
|
280
|
+
["Template Source", run.template_source],
|
|
281
|
+
["Requested Image", run.runner_image or ""],
|
|
282
|
+
["Image Policy", run.image_policy or ""],
|
|
283
|
+
["Resolved Image ID", run.resolved_image_id or ""],
|
|
284
|
+
]
|
|
285
|
+
if run.template_source == "custom"
|
|
286
|
+
else []
|
|
287
|
+
),
|
|
288
|
+
["CPU", resources.cpu if resources else ""],
|
|
289
|
+
["Memory", resources.memory if resources else ""],
|
|
290
|
+
["Accelerators", resources.accelerator_count if resources else ""],
|
|
291
|
+
["Aggregate Metrics", _aggregate_summary(run)],
|
|
292
|
+
["Diagnostics", _diagnostic_summary(run)],
|
|
293
|
+
*(
|
|
294
|
+
[
|
|
295
|
+
[
|
|
296
|
+
"Failure",
|
|
297
|
+
common.bounded_error(_run_error_guidance(run.error_message)),
|
|
298
|
+
]
|
|
299
|
+
]
|
|
300
|
+
if run.error_message
|
|
301
|
+
else []
|
|
302
|
+
),
|
|
303
|
+
["Created", run.created_at],
|
|
304
|
+
["Completed", run.completed_at or ""],
|
|
305
|
+
],
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _print_run_output(run: Any, output: common.OutputChoice) -> None:
|
|
310
|
+
if output == common.OutputChoice.json:
|
|
311
|
+
payload = common.serialize(run)
|
|
312
|
+
if payload.get("errorMessage"):
|
|
313
|
+
payload["errorMessage"] = common.bounded_error(payload["errorMessage"])
|
|
314
|
+
common.print_json(payload)
|
|
315
|
+
else:
|
|
316
|
+
_print_run(run)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@method_app.command("templates", help="List available Evaluation Method templates.")
|
|
320
|
+
def method_templates(
|
|
321
|
+
project: Project = None,
|
|
322
|
+
project_id: ProjectID = None,
|
|
323
|
+
profile_name: Profile = None,
|
|
324
|
+
output: Output = common.OutputChoice.table,
|
|
325
|
+
verbose: Verbose = False,
|
|
326
|
+
):
|
|
327
|
+
try:
|
|
328
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
329
|
+
with _client(profile_name) as client:
|
|
330
|
+
templates = client.evaluations.methods.templates(project=resolved_project)
|
|
331
|
+
except Exception as exc:
|
|
332
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
333
|
+
|
|
334
|
+
if output == common.OutputChoice.json:
|
|
335
|
+
common.print_json(templates)
|
|
336
|
+
return
|
|
337
|
+
if not templates.templates:
|
|
338
|
+
common.print_table(
|
|
339
|
+
["Name", "Display Name", "Output Type", "Metrics"],
|
|
340
|
+
[
|
|
341
|
+
[
|
|
342
|
+
item.name,
|
|
343
|
+
item.display_name,
|
|
344
|
+
item.output_type,
|
|
345
|
+
", ".join(item.metrics),
|
|
346
|
+
]
|
|
347
|
+
for item in templates.method_v2.templates
|
|
348
|
+
],
|
|
349
|
+
)
|
|
350
|
+
return
|
|
351
|
+
common.print_table(
|
|
352
|
+
["ID", "Source", "Name", "Version", "Image Policy", "Metrics"],
|
|
353
|
+
[
|
|
354
|
+
[
|
|
355
|
+
item.id,
|
|
356
|
+
item.source,
|
|
357
|
+
item.name,
|
|
358
|
+
item.version or "",
|
|
359
|
+
item.image_policy or "",
|
|
360
|
+
", ".join(metric.key for metric in item.metrics),
|
|
361
|
+
]
|
|
362
|
+
for item in templates.templates
|
|
363
|
+
],
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
@method_app.command("list")
|
|
368
|
+
def list_methods(
|
|
369
|
+
dataset: Annotated[
|
|
370
|
+
str | None,
|
|
371
|
+
typer.Option("--dataset", help="Dataset name or UUID.", show_default=False),
|
|
372
|
+
] = None,
|
|
373
|
+
version: Annotated[
|
|
374
|
+
str | None,
|
|
375
|
+
typer.Option(
|
|
376
|
+
"--version", help="Dataset version name or UUID.", show_default=False
|
|
377
|
+
),
|
|
378
|
+
] = None,
|
|
379
|
+
page: Page = None,
|
|
380
|
+
limit: Limit = None,
|
|
381
|
+
project: Project = None,
|
|
382
|
+
project_id: ProjectID = None,
|
|
383
|
+
profile_name: Profile = None,
|
|
384
|
+
output: Output = common.OutputChoice.table,
|
|
385
|
+
verbose: Verbose = False,
|
|
386
|
+
):
|
|
387
|
+
try:
|
|
388
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
389
|
+
with _client(profile_name) as client:
|
|
390
|
+
dataset_id, version_id = _dataset_version_ids(
|
|
391
|
+
client, dataset, version, project=resolved_project
|
|
392
|
+
)
|
|
393
|
+
result_page = client.evaluations.methods.list_page(
|
|
394
|
+
project=resolved_project,
|
|
395
|
+
dataset_id=dataset_id,
|
|
396
|
+
dataset_version_id=version_id,
|
|
397
|
+
page=page,
|
|
398
|
+
limit=limit,
|
|
399
|
+
)
|
|
400
|
+
except Exception as exc:
|
|
401
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
402
|
+
|
|
403
|
+
if output == common.OutputChoice.json:
|
|
404
|
+
common.print_json(result_page)
|
|
405
|
+
return
|
|
406
|
+
common.print_page_table(
|
|
407
|
+
result_page,
|
|
408
|
+
["ID", "Name", "Type", "Status", "Dataset ID", "Version ID"],
|
|
409
|
+
_method_rows(result_page.items),
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
@method_app.command("get")
|
|
414
|
+
def get_method(
|
|
415
|
+
method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
|
|
416
|
+
project: Project = None,
|
|
417
|
+
project_id: ProjectID = None,
|
|
418
|
+
profile_name: Profile = None,
|
|
419
|
+
output: Output = common.OutputChoice.table,
|
|
420
|
+
verbose: Verbose = False,
|
|
421
|
+
):
|
|
422
|
+
try:
|
|
423
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
424
|
+
with _client(profile_name) as client:
|
|
425
|
+
method_id = _resolve_method_id(client, method, project=resolved_project)
|
|
426
|
+
result = client.evaluations.methods.get(method_id, project=resolved_project)
|
|
427
|
+
except Exception as exc:
|
|
428
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
429
|
+
|
|
430
|
+
if output == common.OutputChoice.json:
|
|
431
|
+
common.print_json(result)
|
|
432
|
+
return
|
|
433
|
+
_print_method(result)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
@method_app.command("validate")
|
|
437
|
+
def validate_template_method(
|
|
438
|
+
dataset: Annotated[str, typer.Argument(help="Dataset name or UUID.")],
|
|
439
|
+
version: Annotated[str, typer.Argument(help="Dataset version name or UUID.")],
|
|
440
|
+
config: Annotated[Path, typer.Option("--config", help="JSON or YAML config file.")],
|
|
441
|
+
template: Annotated[str, typer.Option("--template", help="Template name or ID.")],
|
|
442
|
+
template_version: Annotated[
|
|
443
|
+
str | None,
|
|
444
|
+
typer.Option(
|
|
445
|
+
"--template-version", help="Template version.", show_default=False
|
|
446
|
+
),
|
|
447
|
+
] = None,
|
|
448
|
+
project: Project = None,
|
|
449
|
+
project_id: ProjectID = None,
|
|
450
|
+
profile_name: Profile = None,
|
|
451
|
+
output: Output = common.OutputChoice.table,
|
|
452
|
+
verbose: Verbose = False,
|
|
453
|
+
):
|
|
454
|
+
try:
|
|
455
|
+
loaded_config = _load_mapping(config, strict=True)
|
|
456
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
457
|
+
with _client(profile_name) as client:
|
|
458
|
+
template_id = _resolve_template_id(
|
|
459
|
+
client, template, template_version, project=resolved_project
|
|
460
|
+
)
|
|
461
|
+
dataset_id, version_id = _dataset_version_ids(
|
|
462
|
+
client, dataset, version, project=resolved_project
|
|
463
|
+
)
|
|
464
|
+
result = client.evaluations.methods.preview(
|
|
465
|
+
dataset_id,
|
|
466
|
+
version_id,
|
|
467
|
+
project=resolved_project,
|
|
468
|
+
config=loaded_config,
|
|
469
|
+
template_id=template_id,
|
|
470
|
+
)
|
|
471
|
+
except Exception as exc:
|
|
472
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
473
|
+
|
|
474
|
+
if output == common.OutputChoice.json:
|
|
475
|
+
common.print_json(result)
|
|
476
|
+
return
|
|
477
|
+
common.print_table(
|
|
478
|
+
["Field", "Value"],
|
|
479
|
+
[
|
|
480
|
+
["Source Schema Hash", result.source_schema_hash],
|
|
481
|
+
["Config", json.dumps(result.config, ensure_ascii=False)],
|
|
482
|
+
],
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
@method_app.command("create")
|
|
487
|
+
def create_template_method(
|
|
488
|
+
dataset: Annotated[str, typer.Argument(help="Dataset name or UUID.")],
|
|
489
|
+
version: Annotated[str, typer.Argument(help="Dataset version name or UUID.")],
|
|
490
|
+
name: Annotated[str, typer.Argument(help="Evaluation Method name.")],
|
|
491
|
+
config: Annotated[Path, typer.Option("--config", help="JSON or YAML config file.")],
|
|
492
|
+
template: Annotated[str, typer.Option("--template", help="Template name or ID.")],
|
|
493
|
+
template_version: Annotated[
|
|
494
|
+
str | None,
|
|
495
|
+
typer.Option(
|
|
496
|
+
"--template-version", help="Template version.", show_default=False
|
|
497
|
+
),
|
|
498
|
+
] = None,
|
|
499
|
+
default_runtime: Annotated[
|
|
500
|
+
Path | None,
|
|
501
|
+
typer.Option(
|
|
502
|
+
"--default-runtime",
|
|
503
|
+
help="JSON or YAML default runtime file.",
|
|
504
|
+
show_default=False,
|
|
505
|
+
),
|
|
506
|
+
] = None,
|
|
507
|
+
description: Annotated[
|
|
508
|
+
str | None,
|
|
509
|
+
typer.Option("--description", help="Description.", show_default=False),
|
|
510
|
+
] = None,
|
|
511
|
+
project: Project = None,
|
|
512
|
+
project_id: ProjectID = None,
|
|
513
|
+
profile_name: Profile = None,
|
|
514
|
+
verbose: Verbose = False,
|
|
515
|
+
):
|
|
516
|
+
try:
|
|
517
|
+
loaded_config = _load_mapping(config, strict=True)
|
|
518
|
+
loaded_runtime = _load_mapping(default_runtime) if default_runtime else None
|
|
519
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
520
|
+
with _client(profile_name) as client:
|
|
521
|
+
template_id = _resolve_template_id(
|
|
522
|
+
client, template, template_version, project=resolved_project
|
|
523
|
+
)
|
|
524
|
+
dataset_id, version_id = _dataset_version_ids(
|
|
525
|
+
client, dataset, version, project=resolved_project
|
|
526
|
+
)
|
|
527
|
+
result = client.evaluations.methods.create(
|
|
528
|
+
dataset_id,
|
|
529
|
+
version_id,
|
|
530
|
+
name,
|
|
531
|
+
project=resolved_project,
|
|
532
|
+
config=loaded_config,
|
|
533
|
+
template_id=template_id,
|
|
534
|
+
default_runtime=loaded_runtime,
|
|
535
|
+
description=description,
|
|
536
|
+
)
|
|
537
|
+
except Exception as exc:
|
|
538
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
539
|
+
|
|
540
|
+
common.console.print(
|
|
541
|
+
f"[bold green]Evaluation Method created:[/] {escape(result.id)}"
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
@method_app.command("archive")
|
|
546
|
+
def archive_method(
|
|
547
|
+
method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
|
|
548
|
+
reason: Annotated[
|
|
549
|
+
str | None,
|
|
550
|
+
typer.Option("--reason", help="Archive reason.", show_default=False),
|
|
551
|
+
] = None,
|
|
552
|
+
project: Project = None,
|
|
553
|
+
project_id: ProjectID = None,
|
|
554
|
+
profile_name: Profile = None,
|
|
555
|
+
yes: Annotated[
|
|
556
|
+
bool, typer.Option("--yes", help="Skip archive confirmation.")
|
|
557
|
+
] = False,
|
|
558
|
+
verbose: Verbose = False,
|
|
559
|
+
):
|
|
560
|
+
try:
|
|
561
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
562
|
+
with _client(profile_name) as client:
|
|
563
|
+
method_id = _resolve_method_id(client, method, project=resolved_project)
|
|
564
|
+
common.confirm_delete(
|
|
565
|
+
resource="Evaluation Method",
|
|
566
|
+
target=method,
|
|
567
|
+
profile_name=profile_name,
|
|
568
|
+
yes=yes,
|
|
569
|
+
)
|
|
570
|
+
client.evaluations.methods.archive(
|
|
571
|
+
method_id, project=resolved_project, reason=reason
|
|
572
|
+
)
|
|
573
|
+
except typer.Exit:
|
|
574
|
+
raise
|
|
575
|
+
except Exception as exc:
|
|
576
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
577
|
+
common.console.print(
|
|
578
|
+
f"[bold green]Evaluation Method archived:[/] {escape(method_id)}"
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
@method_app.command("restore")
|
|
583
|
+
def restore_method(
|
|
584
|
+
method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
|
|
585
|
+
project: Project = None,
|
|
586
|
+
project_id: ProjectID = None,
|
|
587
|
+
profile_name: Profile = None,
|
|
588
|
+
verbose: Verbose = False,
|
|
589
|
+
):
|
|
590
|
+
try:
|
|
591
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
592
|
+
with _client(profile_name) as client:
|
|
593
|
+
method_id = _resolve_method_id(
|
|
594
|
+
client, method, project=resolved_project, archived=True
|
|
595
|
+
)
|
|
596
|
+
client.evaluations.methods.restore(method_id, project=resolved_project)
|
|
597
|
+
except Exception as exc:
|
|
598
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
599
|
+
common.console.print(
|
|
600
|
+
f"[bold green]Evaluation Method restored:[/] {escape(method_id)}"
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
@run_app.command("list")
|
|
605
|
+
def list_runs(
|
|
606
|
+
dataset: Annotated[
|
|
607
|
+
str | None,
|
|
608
|
+
typer.Option("--dataset", help="Dataset name or UUID.", show_default=False),
|
|
609
|
+
] = None,
|
|
610
|
+
version: Annotated[
|
|
611
|
+
str | None,
|
|
612
|
+
typer.Option(
|
|
613
|
+
"--version", help="Dataset version name or UUID.", show_default=False
|
|
614
|
+
),
|
|
615
|
+
] = None,
|
|
616
|
+
model_artifact_id: Annotated[
|
|
617
|
+
int | None,
|
|
618
|
+
typer.Option(
|
|
619
|
+
"--model-artifact-id",
|
|
620
|
+
min=1,
|
|
621
|
+
help="Model artifact ID.",
|
|
622
|
+
show_default=False,
|
|
623
|
+
),
|
|
624
|
+
] = None,
|
|
625
|
+
status: Annotated[
|
|
626
|
+
str | None,
|
|
627
|
+
typer.Option("--status", help="Run status.", show_default=False),
|
|
628
|
+
] = None,
|
|
629
|
+
page: Page = None,
|
|
630
|
+
limit: Limit = None,
|
|
631
|
+
project: Project = None,
|
|
632
|
+
project_id: ProjectID = None,
|
|
633
|
+
profile_name: Profile = None,
|
|
634
|
+
output: Output = common.OutputChoice.table,
|
|
635
|
+
verbose: Verbose = False,
|
|
636
|
+
):
|
|
637
|
+
try:
|
|
638
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
639
|
+
with _client(profile_name) as client:
|
|
640
|
+
dataset_id, version_id = _dataset_version_ids(
|
|
641
|
+
client, dataset, version, project=resolved_project
|
|
642
|
+
)
|
|
643
|
+
result_page = client.evaluations.list_page(
|
|
644
|
+
project=resolved_project,
|
|
645
|
+
dataset_id=dataset_id,
|
|
646
|
+
dataset_version_id=version_id,
|
|
647
|
+
model_artifact_id=model_artifact_id,
|
|
648
|
+
status=status,
|
|
649
|
+
page=page,
|
|
650
|
+
limit=limit,
|
|
651
|
+
)
|
|
652
|
+
except Exception as exc:
|
|
653
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
654
|
+
|
|
655
|
+
if output == common.OutputChoice.json:
|
|
656
|
+
common.print_json(result_page)
|
|
657
|
+
return
|
|
658
|
+
common.print_page_table(
|
|
659
|
+
result_page,
|
|
660
|
+
["ID", "Method ID", "Model Artifact ID", "Status", "Runner", "Created"],
|
|
661
|
+
[
|
|
662
|
+
[
|
|
663
|
+
item.id,
|
|
664
|
+
item.evaluation_method_id or "",
|
|
665
|
+
item.model_artifact_id,
|
|
666
|
+
item.status,
|
|
667
|
+
item.runner_type,
|
|
668
|
+
item.created_at,
|
|
669
|
+
]
|
|
670
|
+
for item in result_page.items
|
|
671
|
+
],
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
@run_app.command("get")
|
|
676
|
+
def get_run(
|
|
677
|
+
run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
|
|
678
|
+
project: Project = None,
|
|
679
|
+
project_id: ProjectID = None,
|
|
680
|
+
profile_name: Profile = None,
|
|
681
|
+
output: Output = common.OutputChoice.table,
|
|
682
|
+
verbose: Verbose = False,
|
|
683
|
+
):
|
|
684
|
+
try:
|
|
685
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
686
|
+
with _client(profile_name) as client:
|
|
687
|
+
result = client.evaluations.get(run_id, project=resolved_project)
|
|
688
|
+
except Exception as exc:
|
|
689
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
690
|
+
_print_run_output(result, output)
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
@run_app.command("wait")
|
|
694
|
+
def wait_run(
|
|
695
|
+
run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
|
|
696
|
+
project: Project = None,
|
|
697
|
+
project_id: ProjectID = None,
|
|
698
|
+
timeout: Annotated[
|
|
699
|
+
float,
|
|
700
|
+
typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
|
|
701
|
+
] = 1800,
|
|
702
|
+
interval: Annotated[
|
|
703
|
+
float,
|
|
704
|
+
typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
|
|
705
|
+
] = 5,
|
|
706
|
+
profile_name: Profile = None,
|
|
707
|
+
output: Output = common.OutputChoice.table,
|
|
708
|
+
verbose: Verbose = False,
|
|
709
|
+
):
|
|
710
|
+
common.validate_wait_options(timeout, interval)
|
|
711
|
+
try:
|
|
712
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
713
|
+
with _client(profile_name) as client:
|
|
714
|
+
result = client.evaluations.wait(
|
|
715
|
+
run_id,
|
|
716
|
+
project=resolved_project,
|
|
717
|
+
timeout=timeout,
|
|
718
|
+
interval=interval,
|
|
719
|
+
)
|
|
720
|
+
except Exception as exc:
|
|
721
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
722
|
+
_print_run_output(result, output)
|
|
723
|
+
if result.status in {"failed", "cancelled"}:
|
|
724
|
+
raise typer.Exit(code=1)
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
@run_app.command("create")
|
|
728
|
+
def create_run(
|
|
729
|
+
method: Annotated[str, typer.Argument(help="Evaluation Method name or UUID.")],
|
|
730
|
+
accelerator_count: Annotated[
|
|
731
|
+
int,
|
|
732
|
+
typer.Option("--accelerator-count", min=0, help="Accelerator count."),
|
|
733
|
+
] = 1,
|
|
734
|
+
cpu: Annotated[str, typer.Option("--cpu", help="CPU quantity.")] = "4",
|
|
735
|
+
memory: Annotated[str, typer.Option("--memory", help="Memory quantity.")] = "8Gi",
|
|
736
|
+
model_artifact_id: Annotated[
|
|
737
|
+
int | None,
|
|
738
|
+
typer.Option(
|
|
739
|
+
"--model-artifact-id", min=1, help="Model artifact ID.", show_default=False
|
|
740
|
+
),
|
|
741
|
+
] = None,
|
|
742
|
+
model: Annotated[
|
|
743
|
+
str | None,
|
|
744
|
+
typer.Option("--model", help="Registered Model name.", show_default=False),
|
|
745
|
+
] = None,
|
|
746
|
+
model_version: Annotated[
|
|
747
|
+
str | None,
|
|
748
|
+
typer.Option("--model-version", help="Model Version name.", show_default=False),
|
|
749
|
+
] = None,
|
|
750
|
+
artifact: Annotated[
|
|
751
|
+
str | None,
|
|
752
|
+
typer.Option("--artifact", help="Model Artifact name.", show_default=False),
|
|
753
|
+
] = None,
|
|
754
|
+
runner_config: Annotated[
|
|
755
|
+
Path | None,
|
|
756
|
+
typer.Option(
|
|
757
|
+
"--runner-config",
|
|
758
|
+
help="JSON or YAML runner config file.",
|
|
759
|
+
show_default=False,
|
|
760
|
+
),
|
|
761
|
+
] = None,
|
|
762
|
+
wait: Annotated[
|
|
763
|
+
bool, typer.Option("--wait", help="Wait for terminal state.")
|
|
764
|
+
] = False,
|
|
765
|
+
timeout: Annotated[
|
|
766
|
+
float,
|
|
767
|
+
typer.Option("--timeout", min=0.001, help="Wait timeout in seconds."),
|
|
768
|
+
] = 1800,
|
|
769
|
+
interval: Annotated[
|
|
770
|
+
float,
|
|
771
|
+
typer.Option("--interval", min=0.001, help="Polling interval in seconds."),
|
|
772
|
+
] = 5,
|
|
773
|
+
project: Project = None,
|
|
774
|
+
project_id: ProjectID = None,
|
|
775
|
+
profile_name: Profile = None,
|
|
776
|
+
output: Output = common.OutputChoice.table,
|
|
777
|
+
verbose: Verbose = False,
|
|
778
|
+
):
|
|
779
|
+
common.validate_wait_options(timeout, interval)
|
|
780
|
+
try:
|
|
781
|
+
loaded_config = _load_mapping(runner_config) if runner_config else None
|
|
782
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
783
|
+
with _client(profile_name) as client:
|
|
784
|
+
method_id = _resolve_method_id(client, method, project=resolved_project)
|
|
785
|
+
resolved_artifact_id = _resolve_model_artifact_id(
|
|
786
|
+
client,
|
|
787
|
+
project=resolved_project,
|
|
788
|
+
model_artifact_id=model_artifact_id,
|
|
789
|
+
model=model,
|
|
790
|
+
model_version=model_version,
|
|
791
|
+
artifact=artifact,
|
|
792
|
+
)
|
|
793
|
+
result = client.evaluations.create(
|
|
794
|
+
project=resolved_project,
|
|
795
|
+
evaluation_method_id=method_id,
|
|
796
|
+
model_artifact_id=resolved_artifact_id,
|
|
797
|
+
cpu=cpu,
|
|
798
|
+
memory=memory,
|
|
799
|
+
accelerator_count=accelerator_count,
|
|
800
|
+
runner_config=loaded_config,
|
|
801
|
+
)
|
|
802
|
+
if wait:
|
|
803
|
+
typer.echo(
|
|
804
|
+
f"Evaluation Run created: {result.id}; resume with "
|
|
805
|
+
f"'nufi evaluation run wait {result.id}'",
|
|
806
|
+
err=True,
|
|
807
|
+
)
|
|
808
|
+
result = client.evaluations.wait(
|
|
809
|
+
result.id,
|
|
810
|
+
project=resolved_project,
|
|
811
|
+
timeout=timeout,
|
|
812
|
+
interval=interval,
|
|
813
|
+
)
|
|
814
|
+
except Exception as exc:
|
|
815
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
816
|
+
if wait or output == common.OutputChoice.json:
|
|
817
|
+
_print_run_output(result, output)
|
|
818
|
+
else:
|
|
819
|
+
common.console.print(
|
|
820
|
+
f"[bold green]Evaluation Run created:[/] {escape(result.id)}"
|
|
821
|
+
)
|
|
822
|
+
if result.status in {"failed", "cancelled"}:
|
|
823
|
+
raise typer.Exit(code=1)
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
@run_app.command("cancel")
|
|
827
|
+
def cancel_run(
|
|
828
|
+
run_id: Annotated[str, typer.Argument(help="Evaluation Run UUID.")],
|
|
829
|
+
project: Project = None,
|
|
830
|
+
project_id: ProjectID = None,
|
|
831
|
+
profile_name: Profile = None,
|
|
832
|
+
yes: Annotated[
|
|
833
|
+
bool, typer.Option("--yes", help="Skip cancel confirmation.")
|
|
834
|
+
] = False,
|
|
835
|
+
verbose: Verbose = False,
|
|
836
|
+
):
|
|
837
|
+
try:
|
|
838
|
+
resolved_project = _project(project, project_id, profile_name)
|
|
839
|
+
except Exception as exc:
|
|
840
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
841
|
+
common.confirm_delete(
|
|
842
|
+
resource="Evaluation Run",
|
|
843
|
+
target=run_id,
|
|
844
|
+
profile_name=profile_name,
|
|
845
|
+
yes=yes,
|
|
846
|
+
)
|
|
847
|
+
try:
|
|
848
|
+
with _client(profile_name) as client:
|
|
849
|
+
result = client.evaluations.cancel(run_id, project=resolved_project)
|
|
850
|
+
except Exception as exc:
|
|
851
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
852
|
+
common.console.print(
|
|
853
|
+
f"[bold green]Evaluation Run cancelled:[/] {escape(result.id)}"
|
|
854
|
+
)
|