nufi-python 1.1.1__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nufi_python-1.1.1/src/nufi_python.egg-info → nufi_python-1.1.2}/PKG-INFO +59 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/README.md +58 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/pyproject.toml +1 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/__init__.py +8 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/compiles.py +1 -39
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/datasets.py +91 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/evaluations.py +28 -2
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/leaderboards.py +15 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/profile.py +2 -4
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/servings.py +126 -15
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/evaluator.py +12 -4
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/__init__.py +8 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/compiles.py +28 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/datasets.py +18 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/evaluations.py +70 -4
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/leaderboards.py +3 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/servings.py +29 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/compiles.py +6 -20
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/datasets.py +62 -5
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/evaluations.py +19 -14
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/leaderboards.py +3 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/servings.py +55 -10
- {nufi_python-1.1.1 → nufi_python-1.1.2/src/nufi_python.egg-info}/PKG-INFO +59 -1
- {nufi_python-1.1.1 → nufi_python-1.1.2}/LICENSE +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/setup.cfg +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/auth.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/__init__.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/__init__.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/auth.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/common.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/models.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/monitoring.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/commands/projects.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/cli/main.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/client.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/config.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/exceptions.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/http_utils.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/monitoring.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/pagination.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/projects.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/models/registry.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/py.typed +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/__init__.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/base.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/models.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/monitoring.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/resources/projects.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi/transport.py +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi_python.egg-info/SOURCES.txt +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi_python.egg-info/dependency_links.txt +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi_python.egg-info/entry_points.txt +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi_python.egg-info/requires.txt +0 -0
- {nufi_python-1.1.1 → nufi_python-1.1.2}/src/nufi_python.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nufi-python
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: Python SDK and CLI for the NuFi platform.
|
|
5
5
|
Author: Dudaji Inc.
|
|
6
6
|
License-Expression: MIT
|
|
@@ -116,6 +116,64 @@ with NuFi.from_profile() as client:
|
|
|
116
116
|
|
|
117
117
|
## Evaluations
|
|
118
118
|
|
|
119
|
+
Dataset import completion and Dataset profile completion are separate asynchronous
|
|
120
|
+
steps. Wait for the profile before previewing or creating an Evaluation Method:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
124
|
+
--repository cais/mmlu --subset abstract_algebra --split test --wait
|
|
125
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The SDK uses the import result's `dataset_version_id` for the same workflow. A failed
|
|
129
|
+
profile is returned as a terminal object so the caller can report its server error
|
|
130
|
+
without sending an Evaluation Method request:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from nufi import NuFi
|
|
134
|
+
|
|
135
|
+
client = NuFi.from_profile()
|
|
136
|
+
dataset_id = "DATASET_ID"
|
|
137
|
+
imported = client.datasets.versions.create_from_huggingface(
|
|
138
|
+
dataset_id,
|
|
139
|
+
"mmlu-aa-test",
|
|
140
|
+
repository="cais/mmlu",
|
|
141
|
+
subset="abstract_algebra",
|
|
142
|
+
split="test",
|
|
143
|
+
wait=True,
|
|
144
|
+
)
|
|
145
|
+
if not imported.dataset_version_id:
|
|
146
|
+
raise RuntimeError("Dataset import completed without a Dataset Version")
|
|
147
|
+
|
|
148
|
+
version_id = imported.dataset_version_id
|
|
149
|
+
profile = client.datasets.versions.wait_for_profile(dataset_id, version_id)
|
|
150
|
+
if profile.status != "ready":
|
|
151
|
+
raise RuntimeError(profile.error or "Dataset profile failed")
|
|
152
|
+
|
|
153
|
+
config = {
|
|
154
|
+
"schemaVersion": "nufi.evaluation.method.v2",
|
|
155
|
+
"runnerType": "lm_eval",
|
|
156
|
+
"template": "multiple_choice",
|
|
157
|
+
"datasetMapping": {
|
|
158
|
+
"input": "question",
|
|
159
|
+
"choices": "choices",
|
|
160
|
+
"reference": "answer",
|
|
161
|
+
},
|
|
162
|
+
"nativeConfig": {
|
|
163
|
+
"output_type": "multiple_choice",
|
|
164
|
+
"doc_to_text": "{{input}}",
|
|
165
|
+
"metric_list": [{"metric": "acc"}],
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
preview = client.evaluations.methods.preview(
|
|
169
|
+
dataset_id, version_id, config=config
|
|
170
|
+
)
|
|
171
|
+
method = client.evaluations.methods.create(
|
|
172
|
+
dataset_id, version_id, "mmlu-aa", config=config
|
|
173
|
+
)
|
|
174
|
+
client.close()
|
|
175
|
+
```
|
|
176
|
+
|
|
119
177
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
120
178
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
121
179
|
without a client-side allowlist.
|
|
@@ -94,6 +94,64 @@ with NuFi.from_profile() as client:
|
|
|
94
94
|
|
|
95
95
|
## Evaluations
|
|
96
96
|
|
|
97
|
+
Dataset import completion and Dataset profile completion are separate asynchronous
|
|
98
|
+
steps. Wait for the profile before previewing or creating an Evaluation Method:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
102
|
+
--repository cais/mmlu --subset abstract_algebra --split test --wait
|
|
103
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The SDK uses the import result's `dataset_version_id` for the same workflow. A failed
|
|
107
|
+
profile is returned as a terminal object so the caller can report its server error
|
|
108
|
+
without sending an Evaluation Method request:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from nufi import NuFi
|
|
112
|
+
|
|
113
|
+
client = NuFi.from_profile()
|
|
114
|
+
dataset_id = "DATASET_ID"
|
|
115
|
+
imported = client.datasets.versions.create_from_huggingface(
|
|
116
|
+
dataset_id,
|
|
117
|
+
"mmlu-aa-test",
|
|
118
|
+
repository="cais/mmlu",
|
|
119
|
+
subset="abstract_algebra",
|
|
120
|
+
split="test",
|
|
121
|
+
wait=True,
|
|
122
|
+
)
|
|
123
|
+
if not imported.dataset_version_id:
|
|
124
|
+
raise RuntimeError("Dataset import completed without a Dataset Version")
|
|
125
|
+
|
|
126
|
+
version_id = imported.dataset_version_id
|
|
127
|
+
profile = client.datasets.versions.wait_for_profile(dataset_id, version_id)
|
|
128
|
+
if profile.status != "ready":
|
|
129
|
+
raise RuntimeError(profile.error or "Dataset profile failed")
|
|
130
|
+
|
|
131
|
+
config = {
|
|
132
|
+
"schemaVersion": "nufi.evaluation.method.v2",
|
|
133
|
+
"runnerType": "lm_eval",
|
|
134
|
+
"template": "multiple_choice",
|
|
135
|
+
"datasetMapping": {
|
|
136
|
+
"input": "question",
|
|
137
|
+
"choices": "choices",
|
|
138
|
+
"reference": "answer",
|
|
139
|
+
},
|
|
140
|
+
"nativeConfig": {
|
|
141
|
+
"output_type": "multiple_choice",
|
|
142
|
+
"doc_to_text": "{{input}}",
|
|
143
|
+
"metric_list": [{"metric": "acc"}],
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
preview = client.evaluations.methods.preview(
|
|
147
|
+
dataset_id, version_id, config=config
|
|
148
|
+
)
|
|
149
|
+
method = client.evaluations.methods.create(
|
|
150
|
+
dataset_id, version_id, "mmlu-aa", config=config
|
|
151
|
+
)
|
|
152
|
+
client.close()
|
|
153
|
+
```
|
|
154
|
+
|
|
97
155
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
98
156
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
99
157
|
without a client-side allowlist.
|
|
@@ -4,9 +4,12 @@ from nufi.client import NuFi
|
|
|
4
4
|
from nufi.models import (
|
|
5
5
|
ArtifactValidation,
|
|
6
6
|
CompileRun,
|
|
7
|
+
CompileRunSummary,
|
|
8
|
+
DatasetVersionProfile,
|
|
7
9
|
EvaluationMethod,
|
|
8
10
|
EvaluationMethodTemplates,
|
|
9
11
|
EvaluationRun,
|
|
12
|
+
EvaluationRunSummary,
|
|
10
13
|
HuggingFaceMapping,
|
|
11
14
|
Leaderboard,
|
|
12
15
|
LeaderboardEntry,
|
|
@@ -22,6 +25,7 @@ from nufi.models import (
|
|
|
22
25
|
ProjectAccessTokenCreateResponse,
|
|
23
26
|
RegisteredModel,
|
|
24
27
|
Serving,
|
|
28
|
+
ServingSummary,
|
|
25
29
|
)
|
|
26
30
|
|
|
27
31
|
__version__ = version("nufi-python")
|
|
@@ -30,9 +34,12 @@ __all__ = [
|
|
|
30
34
|
"__version__",
|
|
31
35
|
"ArtifactValidation",
|
|
32
36
|
"CompileRun",
|
|
37
|
+
"CompileRunSummary",
|
|
38
|
+
"DatasetVersionProfile",
|
|
33
39
|
"EvaluationMethod",
|
|
34
40
|
"EvaluationMethodTemplates",
|
|
35
41
|
"EvaluationRun",
|
|
42
|
+
"EvaluationRunSummary",
|
|
36
43
|
"HuggingFaceMapping",
|
|
37
44
|
"Leaderboard",
|
|
38
45
|
"LeaderboardEntry",
|
|
@@ -49,4 +56,5 @@ __all__ = [
|
|
|
49
56
|
"ProjectAccessTokenCreateResponse",
|
|
50
57
|
"RegisteredModel",
|
|
51
58
|
"Serving",
|
|
59
|
+
"ServingSummary",
|
|
52
60
|
]
|
|
@@ -323,7 +323,7 @@ def get_compile(
|
|
|
323
323
|
),
|
|
324
324
|
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
325
325
|
output: common.OutputChoice = typer.Option(
|
|
326
|
-
common.OutputChoice.
|
|
326
|
+
common.OutputChoice.table,
|
|
327
327
|
"--output",
|
|
328
328
|
help="Output format.",
|
|
329
329
|
),
|
|
@@ -409,44 +409,6 @@ def wait_compile(
|
|
|
409
409
|
raise typer.Exit(code=1)
|
|
410
410
|
|
|
411
411
|
|
|
412
|
-
@app.command("retry")
|
|
413
|
-
def retry_compile(
|
|
414
|
-
name: str = typer.Argument(..., help="Compile CR name."),
|
|
415
|
-
step_name: str = typer.Argument(..., help="Step name."),
|
|
416
|
-
project: str | None = typer.Option(
|
|
417
|
-
None,
|
|
418
|
-
"--project",
|
|
419
|
-
help="Project name.",
|
|
420
|
-
show_default=False,
|
|
421
|
-
),
|
|
422
|
-
project_id: int | None = typer.Option(
|
|
423
|
-
None,
|
|
424
|
-
"--project-id",
|
|
425
|
-
min=1,
|
|
426
|
-
help="Project ID.",
|
|
427
|
-
show_default=False,
|
|
428
|
-
),
|
|
429
|
-
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
430
|
-
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
431
|
-
):
|
|
432
|
-
client = None
|
|
433
|
-
try:
|
|
434
|
-
resolved_project = common.resolve_cli_project(
|
|
435
|
-
project,
|
|
436
|
-
project_id,
|
|
437
|
-
profile_name=profile_name,
|
|
438
|
-
)
|
|
439
|
-
client = common.make_client(profile_name)
|
|
440
|
-
client.compiles.retry(name, step_name, project=resolved_project)
|
|
441
|
-
except Exception as exc:
|
|
442
|
-
common.handle_cli_error(exc, verbose=verbose)
|
|
443
|
-
finally:
|
|
444
|
-
if client is not None:
|
|
445
|
-
client.close()
|
|
446
|
-
|
|
447
|
-
common.console.print(f"[bold green]Compile step retried:[/] {name}/{step_name}")
|
|
448
|
-
|
|
449
|
-
|
|
450
412
|
@app.command("delete")
|
|
451
413
|
def delete_compile(
|
|
452
414
|
name: str = typer.Argument(..., help="Compile CR name."),
|
|
@@ -126,6 +126,27 @@ def _print_version_detail(version) -> None:
|
|
|
126
126
|
)
|
|
127
127
|
|
|
128
128
|
|
|
129
|
+
def _print_profile(profile) -> None:
|
|
130
|
+
common.print_table(
|
|
131
|
+
["Field", "Value"],
|
|
132
|
+
[
|
|
133
|
+
["Version ID", profile.version_id],
|
|
134
|
+
["Status", profile.status],
|
|
135
|
+
["Row Count", profile.row_count],
|
|
136
|
+
["Byte Size", profile.byte_size],
|
|
137
|
+
["Split Profiles", json.dumps(profile.split_profiles, ensure_ascii=False)],
|
|
138
|
+
[
|
|
139
|
+
"Column Profiles",
|
|
140
|
+
json.dumps(profile.column_profiles, ensure_ascii=False),
|
|
141
|
+
],
|
|
142
|
+
["Warnings", json.dumps(profile.warnings, ensure_ascii=False)],
|
|
143
|
+
["Schema Version", profile.profile_schema_version],
|
|
144
|
+
["Error", profile.error or ""],
|
|
145
|
+
["Computed At", profile.computed_at or ""],
|
|
146
|
+
],
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
129
150
|
@app.command("list")
|
|
130
151
|
def list_datasets(
|
|
131
152
|
project: str | None = typer.Option(
|
|
@@ -477,6 +498,73 @@ def get_version(
|
|
|
477
498
|
_print_version_detail(result)
|
|
478
499
|
|
|
479
500
|
|
|
501
|
+
@version_app.command("profile")
|
|
502
|
+
def get_version_profile(
|
|
503
|
+
dataset: str = typer.Argument(..., help="Dataset name or UUID."),
|
|
504
|
+
version: str = typer.Argument(..., help="Dataset version name or UUID."),
|
|
505
|
+
wait: bool = typer.Option(False, "--wait", help="Wait for profile completion."),
|
|
506
|
+
timeout: float = typer.Option(
|
|
507
|
+
1800, "--timeout", min=0.001, help="Wait timeout in seconds."
|
|
508
|
+
),
|
|
509
|
+
interval: float = typer.Option(
|
|
510
|
+
5, "--interval", min=0.001, help="Polling interval in seconds."
|
|
511
|
+
),
|
|
512
|
+
project: str | None = typer.Option(
|
|
513
|
+
None,
|
|
514
|
+
"--project",
|
|
515
|
+
help="Project name.",
|
|
516
|
+
show_default=False,
|
|
517
|
+
),
|
|
518
|
+
project_id: int | None = typer.Option(
|
|
519
|
+
None,
|
|
520
|
+
"--project-id",
|
|
521
|
+
min=1,
|
|
522
|
+
help="Project ID.",
|
|
523
|
+
show_default=False,
|
|
524
|
+
),
|
|
525
|
+
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
526
|
+
output: common.OutputChoice = typer.Option(
|
|
527
|
+
common.OutputChoice.table,
|
|
528
|
+
"--output",
|
|
529
|
+
help="Output format.",
|
|
530
|
+
),
|
|
531
|
+
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
532
|
+
):
|
|
533
|
+
common.validate_wait_options(timeout, interval)
|
|
534
|
+
client = None
|
|
535
|
+
try:
|
|
536
|
+
resolved_project = _project_ref(project, project_id, profile_name)
|
|
537
|
+
client = common.make_client(profile_name)
|
|
538
|
+
dataset_id = _resolve_dataset_id(client, dataset, project=resolved_project)
|
|
539
|
+
version_id = _resolve_version_id(
|
|
540
|
+
client, dataset_id, version, project=resolved_project
|
|
541
|
+
)
|
|
542
|
+
if wait:
|
|
543
|
+
result = client.datasets.versions.wait_for_profile(
|
|
544
|
+
dataset_id,
|
|
545
|
+
version_id,
|
|
546
|
+
project=resolved_project,
|
|
547
|
+
timeout=timeout,
|
|
548
|
+
interval=interval,
|
|
549
|
+
)
|
|
550
|
+
else:
|
|
551
|
+
result = client.datasets.versions.profile(
|
|
552
|
+
dataset_id, version_id, project=resolved_project
|
|
553
|
+
)
|
|
554
|
+
except Exception as exc:
|
|
555
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
556
|
+
finally:
|
|
557
|
+
if client is not None:
|
|
558
|
+
client.close()
|
|
559
|
+
|
|
560
|
+
if output == common.OutputChoice.json:
|
|
561
|
+
common.print_json(result)
|
|
562
|
+
else:
|
|
563
|
+
_print_profile(result)
|
|
564
|
+
if wait and result.status == "failed":
|
|
565
|
+
raise typer.Exit(code=1)
|
|
566
|
+
|
|
567
|
+
|
|
480
568
|
@version_app.command("create")
|
|
481
569
|
def create_version(
|
|
482
570
|
dataset: str = typer.Argument(..., help="Dataset name or UUID."),
|
|
@@ -496,7 +584,9 @@ def create_version(
|
|
|
496
584
|
),
|
|
497
585
|
source: str = typer.Option(..., "--source", help="pvc, file, or huggingface."),
|
|
498
586
|
data_uri: str | None = typer.Option(None, "--data-uri", help="PVC data URI."),
|
|
499
|
-
file: Path | None = typer.Option(
|
|
587
|
+
file: Path | None = typer.Option(
|
|
588
|
+
None, "--file", help="JSONL, CSV, TSV, or Parquet file to upload."
|
|
589
|
+
),
|
|
500
590
|
repository: str | None = typer.Option(
|
|
501
591
|
None, "--repository", help="Hugging Face repository ID."
|
|
502
592
|
),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import shlex
|
|
4
5
|
from collections.abc import Iterator
|
|
5
6
|
from contextlib import contextmanager
|
|
6
7
|
from pathlib import Path
|
|
@@ -102,6 +103,15 @@ def _load_mapping(path: Path, *, strict: bool = False) -> dict[str, Any]:
|
|
|
102
103
|
return payload
|
|
103
104
|
|
|
104
105
|
|
|
106
|
+
def _parse_target_args(value: str | None) -> list[str] | None:
|
|
107
|
+
if value is None:
|
|
108
|
+
return None
|
|
109
|
+
try:
|
|
110
|
+
return shlex.split(value)
|
|
111
|
+
except ValueError as exc:
|
|
112
|
+
raise ValidationError(f"Invalid --target-args: {exc}") from exc
|
|
113
|
+
|
|
114
|
+
|
|
105
115
|
def _dataset_version_ids(
|
|
106
116
|
client: Any,
|
|
107
117
|
dataset: str | None,
|
|
@@ -308,7 +318,7 @@ def _print_run(run: Any) -> None:
|
|
|
308
318
|
|
|
309
319
|
def _print_run_output(run: Any, output: common.OutputChoice) -> None:
|
|
310
320
|
if output == common.OutputChoice.json:
|
|
311
|
-
payload =
|
|
321
|
+
payload = run.model_dump(mode="json", by_alias=True, exclude_unset=True)
|
|
312
322
|
if payload.get("errorMessage"):
|
|
313
323
|
payload["errorMessage"] = common.bounded_error(payload["errorMessage"])
|
|
314
324
|
common.print_json(payload)
|
|
@@ -653,7 +663,9 @@ def list_runs(
|
|
|
653
663
|
common.handle_cli_error(exc, verbose=verbose)
|
|
654
664
|
|
|
655
665
|
if output == common.OutputChoice.json:
|
|
656
|
-
common.print_json(
|
|
666
|
+
common.print_json(
|
|
667
|
+
result_page.model_dump(mode="json", by_alias=True, exclude_unset=True)
|
|
668
|
+
)
|
|
657
669
|
return
|
|
658
670
|
common.print_page_table(
|
|
659
671
|
result_page,
|
|
@@ -759,6 +771,14 @@ def create_run(
|
|
|
759
771
|
show_default=False,
|
|
760
772
|
),
|
|
761
773
|
] = None,
|
|
774
|
+
target_args: Annotated[
|
|
775
|
+
str | None,
|
|
776
|
+
typer.Option(
|
|
777
|
+
"--target-args",
|
|
778
|
+
help="Quoted target-serving arguments.",
|
|
779
|
+
show_default=False,
|
|
780
|
+
),
|
|
781
|
+
] = None,
|
|
762
782
|
wait: Annotated[
|
|
763
783
|
bool, typer.Option("--wait", help="Wait for terminal state.")
|
|
764
784
|
] = False,
|
|
@@ -779,6 +799,7 @@ def create_run(
|
|
|
779
799
|
common.validate_wait_options(timeout, interval)
|
|
780
800
|
try:
|
|
781
801
|
loaded_config = _load_mapping(runner_config) if runner_config else None
|
|
802
|
+
parsed_target_args = _parse_target_args(target_args)
|
|
782
803
|
resolved_project = _project(project, project_id, profile_name)
|
|
783
804
|
with _client(profile_name) as client:
|
|
784
805
|
method_id = _resolve_method_id(client, method, project=resolved_project)
|
|
@@ -798,6 +819,11 @@ def create_run(
|
|
|
798
819
|
memory=memory,
|
|
799
820
|
accelerator_count=accelerator_count,
|
|
800
821
|
runner_config=loaded_config,
|
|
822
|
+
**(
|
|
823
|
+
{"target_args": parsed_target_args}
|
|
824
|
+
if parsed_target_args is not None
|
|
825
|
+
else {}
|
|
826
|
+
),
|
|
801
827
|
)
|
|
802
828
|
if wait:
|
|
803
829
|
typer.echo(
|
|
@@ -20,6 +20,7 @@ from nufi.cli.commands.evaluations import (
|
|
|
20
20
|
Verbose,
|
|
21
21
|
_client,
|
|
22
22
|
_load_mapping,
|
|
23
|
+
_parse_target_args,
|
|
23
24
|
_project,
|
|
24
25
|
_resolve_method_id,
|
|
25
26
|
_resolve_model_artifact_id,
|
|
@@ -520,6 +521,14 @@ def create_submission(
|
|
|
520
521
|
show_default=False,
|
|
521
522
|
),
|
|
522
523
|
] = None,
|
|
524
|
+
target_args: Annotated[
|
|
525
|
+
str | None,
|
|
526
|
+
typer.Option(
|
|
527
|
+
"--target-args",
|
|
528
|
+
help="Quoted target-serving arguments.",
|
|
529
|
+
show_default=False,
|
|
530
|
+
),
|
|
531
|
+
] = None,
|
|
523
532
|
wait: Annotated[
|
|
524
533
|
bool, typer.Option("--wait", help="Wait for terminal state.")
|
|
525
534
|
] = False,
|
|
@@ -539,6 +548,7 @@ def create_submission(
|
|
|
539
548
|
):
|
|
540
549
|
common.validate_wait_options(timeout, interval)
|
|
541
550
|
try:
|
|
551
|
+
parsed_target_args = _parse_target_args(target_args)
|
|
542
552
|
resources = (cpu, memory, accelerator_count)
|
|
543
553
|
if any(value is not None for value in resources) and not all(
|
|
544
554
|
value is not None for value in resources
|
|
@@ -566,6 +576,11 @@ def create_submission(
|
|
|
566
576
|
cpu=cpu,
|
|
567
577
|
memory=memory,
|
|
568
578
|
accelerator_count=accelerator_count,
|
|
579
|
+
**(
|
|
580
|
+
{"target_args": parsed_target_args}
|
|
581
|
+
if parsed_target_args is not None
|
|
582
|
+
else {}
|
|
583
|
+
),
|
|
569
584
|
)
|
|
570
585
|
if wait:
|
|
571
586
|
typer.echo(
|
|
@@ -30,7 +30,7 @@ def create_profile(
|
|
|
30
30
|
tls_enabled: bool | None = typer.Option(
|
|
31
31
|
None,
|
|
32
32
|
"--tls-enabled/--no-tls-enabled",
|
|
33
|
-
help="Use HTTPS for derived URLs (default: HTTP
|
|
33
|
+
help="Use HTTPS for derived URLs (default: HTTP).",
|
|
34
34
|
),
|
|
35
35
|
api_url: str | None = typer.Option(None, "--api-url", help="API URL override."),
|
|
36
36
|
keycloak_issuer: str | None = typer.Option(
|
|
@@ -88,9 +88,7 @@ def _resolve_profile_create_defaults(
|
|
|
88
88
|
|
|
89
89
|
if tls_enabled is not None:
|
|
90
90
|
return base_domain, "https" if tls_enabled else "http"
|
|
91
|
-
|
|
92
|
-
return base_domain, "http"
|
|
93
|
-
return base_domain, "https"
|
|
91
|
+
return base_domain, "http"
|
|
94
92
|
|
|
95
93
|
|
|
96
94
|
@app.command("use")
|
|
@@ -20,6 +20,33 @@ def servings(ctx: typer.Context):
|
|
|
20
20
|
common.show_help_if_no_subcommand(ctx)
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
def _serving_rows(servings: list[Any]) -> list[list[str]]:
|
|
24
|
+
return [
|
|
25
|
+
[
|
|
26
|
+
serving.name,
|
|
27
|
+
serving.deploy_status or "",
|
|
28
|
+
"" if serving.ready_replicas is None else str(serving.ready_replicas),
|
|
29
|
+
"" if serving.desired_replicas is None else str(serving.desired_replicas),
|
|
30
|
+
serving.endpoint or "",
|
|
31
|
+
]
|
|
32
|
+
for serving in servings
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _print_servings(servings: list[Any]) -> None:
|
|
37
|
+
common.print_table(
|
|
38
|
+
["Name", "Status", "Ready", "Desired", "Endpoint"],
|
|
39
|
+
_serving_rows(servings),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _print_serving(serving: Any, output: common.OutputChoice) -> None:
|
|
44
|
+
if output == common.OutputChoice.json:
|
|
45
|
+
common.print_json(serving)
|
|
46
|
+
else:
|
|
47
|
+
_print_servings([serving])
|
|
48
|
+
|
|
49
|
+
|
|
23
50
|
@app.command("list")
|
|
24
51
|
def list_servings(
|
|
25
52
|
project: str | None = typer.Option(
|
|
@@ -62,21 +89,105 @@ def list_servings(
|
|
|
62
89
|
common.print_json(servings)
|
|
63
90
|
return
|
|
64
91
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
92
|
+
_print_servings(servings)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.command("get")
|
|
96
|
+
def get_serving(
|
|
97
|
+
name: str = typer.Argument(..., help="Serving name."),
|
|
98
|
+
project: str | None = typer.Option(
|
|
99
|
+
None,
|
|
100
|
+
"--project",
|
|
101
|
+
help="Project name.",
|
|
102
|
+
show_default=False,
|
|
103
|
+
),
|
|
104
|
+
project_id: int | None = typer.Option(
|
|
105
|
+
None,
|
|
106
|
+
"--project-id",
|
|
107
|
+
min=1,
|
|
108
|
+
help="Project ID.",
|
|
109
|
+
show_default=False,
|
|
110
|
+
),
|
|
111
|
+
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
112
|
+
output: common.OutputChoice = typer.Option(
|
|
113
|
+
common.OutputChoice.table,
|
|
114
|
+
"--output",
|
|
115
|
+
help="Output format.",
|
|
116
|
+
),
|
|
117
|
+
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
118
|
+
):
|
|
119
|
+
client = None
|
|
120
|
+
try:
|
|
121
|
+
resolved_project = common.resolve_cli_project(
|
|
122
|
+
project,
|
|
123
|
+
project_id,
|
|
124
|
+
profile_name=profile_name,
|
|
125
|
+
)
|
|
126
|
+
client = common.make_client(profile_name)
|
|
127
|
+
serving = client.servings.get(name, project=resolved_project)
|
|
128
|
+
except Exception as exc:
|
|
129
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
130
|
+
finally:
|
|
131
|
+
if client is not None:
|
|
132
|
+
client.close()
|
|
133
|
+
|
|
134
|
+
_print_serving(serving, output)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@app.command("wait")
|
|
138
|
+
def wait_serving(
|
|
139
|
+
name: str = typer.Argument(..., help="Serving name."),
|
|
140
|
+
project: str | None = typer.Option(
|
|
141
|
+
None,
|
|
142
|
+
"--project",
|
|
143
|
+
help="Project name.",
|
|
144
|
+
show_default=False,
|
|
145
|
+
),
|
|
146
|
+
project_id: int | None = typer.Option(
|
|
147
|
+
None,
|
|
148
|
+
"--project-id",
|
|
149
|
+
min=1,
|
|
150
|
+
help="Project ID.",
|
|
151
|
+
show_default=False,
|
|
152
|
+
),
|
|
153
|
+
timeout: float = typer.Option(
|
|
154
|
+
1800, "--timeout", min=0.001, help="Wait timeout in seconds."
|
|
155
|
+
),
|
|
156
|
+
interval: float = typer.Option(
|
|
157
|
+
5, "--interval", min=0.001, help="Polling interval in seconds."
|
|
158
|
+
),
|
|
159
|
+
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
160
|
+
output: common.OutputChoice = typer.Option(
|
|
161
|
+
common.OutputChoice.table,
|
|
162
|
+
"--output",
|
|
163
|
+
help="Output format.",
|
|
164
|
+
),
|
|
165
|
+
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
166
|
+
):
|
|
167
|
+
common.validate_wait_options(timeout, interval)
|
|
168
|
+
client = None
|
|
169
|
+
try:
|
|
170
|
+
resolved_project = common.resolve_cli_project(
|
|
171
|
+
project,
|
|
172
|
+
project_id,
|
|
173
|
+
profile_name=profile_name,
|
|
174
|
+
)
|
|
175
|
+
client = common.make_client(profile_name)
|
|
176
|
+
serving = client.servings.wait(
|
|
177
|
+
name,
|
|
178
|
+
project=resolved_project,
|
|
179
|
+
timeout=timeout,
|
|
180
|
+
interval=interval,
|
|
181
|
+
)
|
|
182
|
+
except Exception as exc:
|
|
183
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
184
|
+
finally:
|
|
185
|
+
if client is not None:
|
|
186
|
+
client.close()
|
|
187
|
+
|
|
188
|
+
_print_serving(serving, output)
|
|
189
|
+
if serving.deploy_status == "Error":
|
|
190
|
+
raise typer.Exit(code=1)
|
|
80
191
|
|
|
81
192
|
|
|
82
193
|
@app.command("create")
|
|
@@ -121,9 +121,15 @@ class EvaluationContext:
|
|
|
121
121
|
if not root.is_dir():
|
|
122
122
|
raise _DatasetError
|
|
123
123
|
files = []
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
candidates = sorted(
|
|
125
|
+
(
|
|
126
|
+
candidate
|
|
127
|
+
for candidate in root.rglob("*")
|
|
128
|
+
if candidate.is_file() and candidate.suffix.lower() == ".parquet"
|
|
129
|
+
),
|
|
130
|
+
key=lambda path: path.relative_to(root).as_posix().encode(),
|
|
131
|
+
)
|
|
132
|
+
for candidate in candidates:
|
|
127
133
|
resolved = candidate.resolve(strict=True)
|
|
128
134
|
try:
|
|
129
135
|
resolved.relative_to(root)
|
|
@@ -233,7 +239,9 @@ def _validate_spec(spec: Any) -> None:
|
|
|
233
239
|
if dataset["format"] != "parquet":
|
|
234
240
|
raise ValueError("dataset.format must be parquet")
|
|
235
241
|
_string(dataset, "schemaHash", "dataset.schemaHash")
|
|
236
|
-
_json_object(dataset, "schema", "dataset.schema")
|
|
242
|
+
schema = _json_object(dataset, "schema", "dataset.schema")
|
|
243
|
+
if schema.get("version") != "v3":
|
|
244
|
+
raise ValueError("dataset.schema.version must be v3")
|
|
237
245
|
roles = _json_object(dataset, "roles", "dataset.roles")
|
|
238
246
|
if not 1 <= len(roles) <= 32:
|
|
239
247
|
raise ValueError("dataset.roles must contain 1 to 32 roles")
|