nufi-python 1.1.0__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nufi_python-1.1.0/src/nufi_python.egg-info → nufi_python-1.1.2}/PKG-INFO +91 -51
- {nufi_python-1.1.0 → nufi_python-1.1.2}/README.md +90 -50
- {nufi_python-1.1.0 → nufi_python-1.1.2}/pyproject.toml +1 -1
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/__init__.py +8 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/compiles.py +1 -39
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/datasets.py +91 -1
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/evaluations.py +28 -2
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/leaderboards.py +15 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/profile.py +2 -4
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/servings.py +126 -15
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/evaluator.py +12 -4
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/__init__.py +8 -3
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/compiles.py +28 -8
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/datasets.py +18 -1
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/evaluations.py +70 -4
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/leaderboards.py +3 -0
- nufi_python-1.1.2/src/nufi/models/pagination.py +16 -0
- nufi_python-1.1.2/src/nufi/models/servings.py +58 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/base.py +2 -29
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/compiles.py +8 -27
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/datasets.py +68 -26
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/evaluations.py +22 -27
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/leaderboards.py +9 -21
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/models.py +18 -54
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/servings.py +55 -10
- {nufi_python-1.1.0 → nufi_python-1.1.2/src/nufi_python.egg-info}/PKG-INFO +91 -51
- nufi_python-1.1.0/src/nufi/models/pagination.py +0 -33
- nufi_python-1.1.0/src/nufi/models/servings.py +0 -27
- {nufi_python-1.1.0 → nufi_python-1.1.2}/LICENSE +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/setup.cfg +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/auth.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/__init__.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/__init__.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/auth.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/common.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/models.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/monitoring.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/commands/projects.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/cli/main.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/client.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/config.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/exceptions.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/http_utils.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/monitoring.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/projects.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/models/registry.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/py.typed +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/__init__.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/custom_evaluation_templates.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/monitoring.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/resources/projects.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi/transport.py +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi_python.egg-info/SOURCES.txt +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi_python.egg-info/dependency_links.txt +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi_python.egg-info/entry_points.txt +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi_python.egg-info/requires.txt +0 -0
- {nufi_python-1.1.0 → nufi_python-1.1.2}/src/nufi_python.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nufi-python
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: Python SDK and CLI for the NuFi platform.
|
|
5
5
|
Author: Dudaji Inc.
|
|
6
6
|
License-Expression: MIT
|
|
@@ -22,34 +22,41 @@ Dynamic: license-file
|
|
|
22
22
|
|
|
23
23
|
# NuFi Python SDK/CLI
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
command.
|
|
25
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
26
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
27
29
|
|
|
28
30
|
```bash
|
|
29
|
-
|
|
30
|
-
python -m pip install ./dist/nufi_python-1.1.0-py3-none-any.whl
|
|
31
|
+
python -m pip install nufi-python
|
|
31
32
|
nufi --version
|
|
33
|
+
nufi --help
|
|
32
34
|
```
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
```
|
|
36
|
+
Install the optional evaluator dependencies when writing a custom evaluator that reads
|
|
37
|
+
NuFi Parquet datasets:
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
```bash
|
|
40
|
+
python -m pip install "nufi-python[evaluator]"
|
|
41
|
+
```
|
|
41
42
|
|
|
42
|
-
##
|
|
43
|
+
## Configure access
|
|
43
44
|
|
|
44
|
-
|
|
45
|
+
Ask your NuFi administrator for the installation's base domain and a project you can
|
|
46
|
+
access. Create a profile, log in, and select the project once:
|
|
45
47
|
|
|
46
48
|
```bash
|
|
47
|
-
nufi profile create
|
|
48
|
-
nufi profile use
|
|
49
|
+
nufi profile create production --base-domain nufi.example
|
|
50
|
+
nufi profile use production
|
|
49
51
|
nufi login
|
|
50
52
|
nufi project use team-a
|
|
51
53
|
```
|
|
52
54
|
|
|
55
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
56
|
+
printed verification URL elsewhere.
|
|
57
|
+
|
|
58
|
+
## CLI quickstart
|
|
59
|
+
|
|
53
60
|
The CLI workflow is composable; every long-running identity can be resumed with its
|
|
54
61
|
dedicated `wait` command:
|
|
55
62
|
|
|
@@ -69,7 +76,9 @@ nufi leaderboard submission wait quality SUBMISSION_ID
|
|
|
69
76
|
nufi leaderboard entries quality
|
|
70
77
|
```
|
|
71
78
|
|
|
72
|
-
|
|
79
|
+
## Python SDK quickstart
|
|
80
|
+
|
|
81
|
+
The same flow is available in Python:
|
|
73
82
|
|
|
74
83
|
```python
|
|
75
84
|
from nufi import NuFi
|
|
@@ -105,27 +114,74 @@ with NuFi.from_profile() as client:
|
|
|
105
114
|
entries = client.leaderboards.entries(leaderboard.id)
|
|
106
115
|
```
|
|
107
116
|
|
|
108
|
-
##
|
|
117
|
+
## Evaluations
|
|
109
118
|
|
|
110
|
-
|
|
119
|
+
Dataset import completion and Dataset profile completion are separate asynchronous
|
|
120
|
+
steps. Wait for the profile before previewing or creating an Evaluation Method:
|
|
111
121
|
|
|
112
122
|
```bash
|
|
113
|
-
|
|
123
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
124
|
+
--repository cais/mmlu --subset abstract_algebra --split test --wait
|
|
125
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
114
126
|
```
|
|
115
127
|
|
|
116
|
-
|
|
128
|
+
The SDK uses the import result's `dataset_version_id` for the same workflow. A failed
|
|
129
|
+
profile is returned as a terminal object so the caller can report its server error
|
|
130
|
+
without sending an Evaluation Method request:
|
|
117
131
|
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
uv run nufi profile
|
|
121
|
-
```
|
|
132
|
+
```python
|
|
133
|
+
from nufi import NuFi
|
|
122
134
|
|
|
123
|
-
|
|
135
|
+
client = NuFi.from_profile()
|
|
136
|
+
dataset_id = "DATASET_ID"
|
|
137
|
+
imported = client.datasets.versions.create_from_huggingface(
|
|
138
|
+
dataset_id,
|
|
139
|
+
"mmlu-aa-test",
|
|
140
|
+
repository="cais/mmlu",
|
|
141
|
+
subset="abstract_algebra",
|
|
142
|
+
split="test",
|
|
143
|
+
wait=True,
|
|
144
|
+
)
|
|
145
|
+
if not imported.dataset_version_id:
|
|
146
|
+
raise RuntimeError("Dataset import completed without a Dataset Version")
|
|
147
|
+
|
|
148
|
+
version_id = imported.dataset_version_id
|
|
149
|
+
profile = client.datasets.versions.wait_for_profile(dataset_id, version_id)
|
|
150
|
+
if profile.status != "ready":
|
|
151
|
+
raise RuntimeError(profile.error or "Dataset profile failed")
|
|
152
|
+
|
|
153
|
+
config = {
|
|
154
|
+
"schemaVersion": "nufi.evaluation.method.v2",
|
|
155
|
+
"runnerType": "lm_eval",
|
|
156
|
+
"template": "multiple_choice",
|
|
157
|
+
"datasetMapping": {
|
|
158
|
+
"input": "question",
|
|
159
|
+
"choices": "choices",
|
|
160
|
+
"reference": "answer",
|
|
161
|
+
},
|
|
162
|
+
"nativeConfig": {
|
|
163
|
+
"output_type": "multiple_choice",
|
|
164
|
+
"doc_to_text": "{{input}}",
|
|
165
|
+
"metric_list": [{"metric": "acc"}],
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
preview = client.evaluations.methods.preview(
|
|
169
|
+
dataset_id, version_id, config=config
|
|
170
|
+
)
|
|
171
|
+
method = client.evaluations.methods.create(
|
|
172
|
+
dataset_id, version_id, "mmlu-aa", config=config
|
|
173
|
+
)
|
|
174
|
+
client.close()
|
|
175
|
+
```
|
|
124
176
|
|
|
125
177
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
126
178
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
127
179
|
without a client-side allowlist.
|
|
128
180
|
|
|
181
|
+
The `nufi.evaluator.run` helper exits non-zero when evaluation or result submission
|
|
182
|
+
fails. NuFi determines Run success from Job success and durable API acceptance;
|
|
183
|
+
evaluator output is diagnostic only.
|
|
184
|
+
|
|
129
185
|
`evaluation method templates` currently exposes lm-eval templates, metrics, and
|
|
130
186
|
filters. It is an optional discovery command and is not needed when you already have a
|
|
131
187
|
Method config.
|
|
@@ -191,20 +247,20 @@ nativeConfig:
|
|
|
191
247
|
```
|
|
192
248
|
|
|
193
249
|
```bash
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
250
|
+
nufi evaluation method templates --project team-a
|
|
251
|
+
nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
|
|
252
|
+
nufi evaluation method create gsm8k v1 gsm8k-exact-match \
|
|
197
253
|
--config method.yaml --project team-a
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
254
|
+
nufi evaluation method list --dataset gsm8k --version v1 --project team-a
|
|
255
|
+
nufi evaluation method archive gsm8k-exact-match --yes --project team-a
|
|
256
|
+
nufi evaluation method restore gsm8k-exact-match --project team-a
|
|
201
257
|
|
|
202
|
-
|
|
258
|
+
nufi evaluation run create gsm8k-exact-match \
|
|
203
259
|
--model qwen2 --model-version v1 --artifact original \
|
|
204
260
|
--project team-a
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
261
|
+
nufi evaluation run list --dataset gsm8k --version v1 --project team-a
|
|
262
|
+
nufi evaluation run get RUN_ID --project team-a
|
|
263
|
+
nufi evaluation run cancel RUN_ID --yes --project team-a
|
|
208
264
|
```
|
|
209
265
|
|
|
210
266
|
Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
|
|
@@ -214,19 +270,3 @@ Run detail table output shows aggregate and diagnostic summaries only. Use expli
|
|
|
214
270
|
`--output json` to include the authorized bounded diagnostic input, output, reference,
|
|
215
271
|
score, and error fields returned by the API. Additive response fields are preserved;
|
|
216
272
|
legacy Method/Run responses missing required v2 fields are rejected.
|
|
217
|
-
|
|
218
|
-
Run tests:
|
|
219
|
-
|
|
220
|
-
```bash
|
|
221
|
-
uv run pytest
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
Build the package:
|
|
225
|
-
|
|
226
|
-
```bash
|
|
227
|
-
uv build --clear
|
|
228
|
-
uv run python scripts/verify_distribution.py
|
|
229
|
-
```
|
|
230
|
-
|
|
231
|
-
The verifier reports the CI source revision from `CI_COMMIT_SHA` and the SHA-256
|
|
232
|
-
digest of each artifact for the operator-controlled release handoff.
|
|
@@ -1,33 +1,40 @@
|
|
|
1
1
|
# NuFi Python SDK/CLI
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
command.
|
|
3
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
4
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
5
7
|
|
|
6
8
|
```bash
|
|
7
|
-
|
|
8
|
-
python -m pip install ./dist/nufi_python-1.1.0-py3-none-any.whl
|
|
9
|
+
python -m pip install nufi-python
|
|
9
10
|
nufi --version
|
|
11
|
+
nufi --help
|
|
10
12
|
```
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
```
|
|
14
|
+
Install the optional evaluator dependencies when writing a custom evaluator that reads
|
|
15
|
+
NuFi Parquet datasets:
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
```bash
|
|
18
|
+
python -m pip install "nufi-python[evaluator]"
|
|
19
|
+
```
|
|
19
20
|
|
|
20
|
-
##
|
|
21
|
+
## Configure access
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
Ask your NuFi administrator for the installation's base domain and a project you can
|
|
24
|
+
access. Create a profile, log in, and select the project once:
|
|
23
25
|
|
|
24
26
|
```bash
|
|
25
|
-
nufi profile create
|
|
26
|
-
nufi profile use
|
|
27
|
+
nufi profile create production --base-domain nufi.example
|
|
28
|
+
nufi profile use production
|
|
27
29
|
nufi login
|
|
28
30
|
nufi project use team-a
|
|
29
31
|
```
|
|
30
32
|
|
|
33
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
34
|
+
printed verification URL elsewhere.
|
|
35
|
+
|
|
36
|
+
## CLI quickstart
|
|
37
|
+
|
|
31
38
|
The CLI workflow is composable; every long-running identity can be resumed with its
|
|
32
39
|
dedicated `wait` command:
|
|
33
40
|
|
|
@@ -47,7 +54,9 @@ nufi leaderboard submission wait quality SUBMISSION_ID
|
|
|
47
54
|
nufi leaderboard entries quality
|
|
48
55
|
```
|
|
49
56
|
|
|
50
|
-
|
|
57
|
+
## Python SDK quickstart
|
|
58
|
+
|
|
59
|
+
The same flow is available in Python:
|
|
51
60
|
|
|
52
61
|
```python
|
|
53
62
|
from nufi import NuFi
|
|
@@ -83,27 +92,74 @@ with NuFi.from_profile() as client:
|
|
|
83
92
|
entries = client.leaderboards.entries(leaderboard.id)
|
|
84
93
|
```
|
|
85
94
|
|
|
86
|
-
##
|
|
95
|
+
## Evaluations
|
|
87
96
|
|
|
88
|
-
|
|
97
|
+
Dataset import completion and Dataset profile completion are separate asynchronous
|
|
98
|
+
steps. Wait for the profile before previewing or creating an Evaluation Method:
|
|
89
99
|
|
|
90
100
|
```bash
|
|
91
|
-
|
|
101
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
102
|
+
--repository cais/mmlu --subset abstract_algebra --split test --wait
|
|
103
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
92
104
|
```
|
|
93
105
|
|
|
94
|
-
|
|
106
|
+
The SDK uses the import result's `dataset_version_id` for the same workflow. A failed
|
|
107
|
+
profile is returned as a terminal object so the caller can report its server error
|
|
108
|
+
without sending an Evaluation Method request:
|
|
95
109
|
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
uv run nufi profile
|
|
99
|
-
```
|
|
110
|
+
```python
|
|
111
|
+
from nufi import NuFi
|
|
100
112
|
|
|
101
|
-
|
|
113
|
+
client = NuFi.from_profile()
|
|
114
|
+
dataset_id = "DATASET_ID"
|
|
115
|
+
imported = client.datasets.versions.create_from_huggingface(
|
|
116
|
+
dataset_id,
|
|
117
|
+
"mmlu-aa-test",
|
|
118
|
+
repository="cais/mmlu",
|
|
119
|
+
subset="abstract_algebra",
|
|
120
|
+
split="test",
|
|
121
|
+
wait=True,
|
|
122
|
+
)
|
|
123
|
+
if not imported.dataset_version_id:
|
|
124
|
+
raise RuntimeError("Dataset import completed without a Dataset Version")
|
|
125
|
+
|
|
126
|
+
version_id = imported.dataset_version_id
|
|
127
|
+
profile = client.datasets.versions.wait_for_profile(dataset_id, version_id)
|
|
128
|
+
if profile.status != "ready":
|
|
129
|
+
raise RuntimeError(profile.error or "Dataset profile failed")
|
|
130
|
+
|
|
131
|
+
config = {
|
|
132
|
+
"schemaVersion": "nufi.evaluation.method.v2",
|
|
133
|
+
"runnerType": "lm_eval",
|
|
134
|
+
"template": "multiple_choice",
|
|
135
|
+
"datasetMapping": {
|
|
136
|
+
"input": "question",
|
|
137
|
+
"choices": "choices",
|
|
138
|
+
"reference": "answer",
|
|
139
|
+
},
|
|
140
|
+
"nativeConfig": {
|
|
141
|
+
"output_type": "multiple_choice",
|
|
142
|
+
"doc_to_text": "{{input}}",
|
|
143
|
+
"metric_list": [{"metric": "acc"}],
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
preview = client.evaluations.methods.preview(
|
|
147
|
+
dataset_id, version_id, config=config
|
|
148
|
+
)
|
|
149
|
+
method = client.evaluations.methods.create(
|
|
150
|
+
dataset_id, version_id, "mmlu-aa", config=config
|
|
151
|
+
)
|
|
152
|
+
client.close()
|
|
153
|
+
```
|
|
102
154
|
|
|
103
155
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
104
156
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
105
157
|
without a client-side allowlist.
|
|
106
158
|
|
|
159
|
+
The `nufi.evaluator.run` helper exits non-zero when evaluation or result submission
|
|
160
|
+
fails. NuFi determines Run success from Job success and durable API acceptance;
|
|
161
|
+
evaluator output is diagnostic only.
|
|
162
|
+
|
|
107
163
|
`evaluation method templates` currently exposes lm-eval templates, metrics, and
|
|
108
164
|
filters. It is an optional discovery command and is not needed when you already have a
|
|
109
165
|
Method config.
|
|
@@ -169,20 +225,20 @@ nativeConfig:
|
|
|
169
225
|
```
|
|
170
226
|
|
|
171
227
|
```bash
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
228
|
+
nufi evaluation method templates --project team-a
|
|
229
|
+
nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
|
|
230
|
+
nufi evaluation method create gsm8k v1 gsm8k-exact-match \
|
|
175
231
|
--config method.yaml --project team-a
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
232
|
+
nufi evaluation method list --dataset gsm8k --version v1 --project team-a
|
|
233
|
+
nufi evaluation method archive gsm8k-exact-match --yes --project team-a
|
|
234
|
+
nufi evaluation method restore gsm8k-exact-match --project team-a
|
|
179
235
|
|
|
180
|
-
|
|
236
|
+
nufi evaluation run create gsm8k-exact-match \
|
|
181
237
|
--model qwen2 --model-version v1 --artifact original \
|
|
182
238
|
--project team-a
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
239
|
+
nufi evaluation run list --dataset gsm8k --version v1 --project team-a
|
|
240
|
+
nufi evaluation run get RUN_ID --project team-a
|
|
241
|
+
nufi evaluation run cancel RUN_ID --yes --project team-a
|
|
186
242
|
```
|
|
187
243
|
|
|
188
244
|
Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
|
|
@@ -192,19 +248,3 @@ Run detail table output shows aggregate and diagnostic summaries only. Use expli
|
|
|
192
248
|
`--output json` to include the authorized bounded diagnostic input, output, reference,
|
|
193
249
|
score, and error fields returned by the API. Additive response fields are preserved;
|
|
194
250
|
legacy Method/Run responses missing required v2 fields are rejected.
|
|
195
|
-
|
|
196
|
-
Run tests:
|
|
197
|
-
|
|
198
|
-
```bash
|
|
199
|
-
uv run pytest
|
|
200
|
-
```
|
|
201
|
-
|
|
202
|
-
Build the package:
|
|
203
|
-
|
|
204
|
-
```bash
|
|
205
|
-
uv build --clear
|
|
206
|
-
uv run python scripts/verify_distribution.py
|
|
207
|
-
```
|
|
208
|
-
|
|
209
|
-
The verifier reports the CI source revision from `CI_COMMIT_SHA` and the SHA-256
|
|
210
|
-
digest of each artifact for the operator-controlled release handoff.
|
|
@@ -4,9 +4,12 @@ from nufi.client import NuFi
|
|
|
4
4
|
from nufi.models import (
|
|
5
5
|
ArtifactValidation,
|
|
6
6
|
CompileRun,
|
|
7
|
+
CompileRunSummary,
|
|
8
|
+
DatasetVersionProfile,
|
|
7
9
|
EvaluationMethod,
|
|
8
10
|
EvaluationMethodTemplates,
|
|
9
11
|
EvaluationRun,
|
|
12
|
+
EvaluationRunSummary,
|
|
10
13
|
HuggingFaceMapping,
|
|
11
14
|
Leaderboard,
|
|
12
15
|
LeaderboardEntry,
|
|
@@ -22,6 +25,7 @@ from nufi.models import (
|
|
|
22
25
|
ProjectAccessTokenCreateResponse,
|
|
23
26
|
RegisteredModel,
|
|
24
27
|
Serving,
|
|
28
|
+
ServingSummary,
|
|
25
29
|
)
|
|
26
30
|
|
|
27
31
|
__version__ = version("nufi-python")
|
|
@@ -30,9 +34,12 @@ __all__ = [
|
|
|
30
34
|
"__version__",
|
|
31
35
|
"ArtifactValidation",
|
|
32
36
|
"CompileRun",
|
|
37
|
+
"CompileRunSummary",
|
|
38
|
+
"DatasetVersionProfile",
|
|
33
39
|
"EvaluationMethod",
|
|
34
40
|
"EvaluationMethodTemplates",
|
|
35
41
|
"EvaluationRun",
|
|
42
|
+
"EvaluationRunSummary",
|
|
36
43
|
"HuggingFaceMapping",
|
|
37
44
|
"Leaderboard",
|
|
38
45
|
"LeaderboardEntry",
|
|
@@ -49,4 +56,5 @@ __all__ = [
|
|
|
49
56
|
"ProjectAccessTokenCreateResponse",
|
|
50
57
|
"RegisteredModel",
|
|
51
58
|
"Serving",
|
|
59
|
+
"ServingSummary",
|
|
52
60
|
]
|
|
@@ -323,7 +323,7 @@ def get_compile(
|
|
|
323
323
|
),
|
|
324
324
|
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
325
325
|
output: common.OutputChoice = typer.Option(
|
|
326
|
-
common.OutputChoice.
|
|
326
|
+
common.OutputChoice.table,
|
|
327
327
|
"--output",
|
|
328
328
|
help="Output format.",
|
|
329
329
|
),
|
|
@@ -409,44 +409,6 @@ def wait_compile(
|
|
|
409
409
|
raise typer.Exit(code=1)
|
|
410
410
|
|
|
411
411
|
|
|
412
|
-
@app.command("retry")
|
|
413
|
-
def retry_compile(
|
|
414
|
-
name: str = typer.Argument(..., help="Compile CR name."),
|
|
415
|
-
step_name: str = typer.Argument(..., help="Step name."),
|
|
416
|
-
project: str | None = typer.Option(
|
|
417
|
-
None,
|
|
418
|
-
"--project",
|
|
419
|
-
help="Project name.",
|
|
420
|
-
show_default=False,
|
|
421
|
-
),
|
|
422
|
-
project_id: int | None = typer.Option(
|
|
423
|
-
None,
|
|
424
|
-
"--project-id",
|
|
425
|
-
min=1,
|
|
426
|
-
help="Project ID.",
|
|
427
|
-
show_default=False,
|
|
428
|
-
),
|
|
429
|
-
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
430
|
-
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
431
|
-
):
|
|
432
|
-
client = None
|
|
433
|
-
try:
|
|
434
|
-
resolved_project = common.resolve_cli_project(
|
|
435
|
-
project,
|
|
436
|
-
project_id,
|
|
437
|
-
profile_name=profile_name,
|
|
438
|
-
)
|
|
439
|
-
client = common.make_client(profile_name)
|
|
440
|
-
client.compiles.retry(name, step_name, project=resolved_project)
|
|
441
|
-
except Exception as exc:
|
|
442
|
-
common.handle_cli_error(exc, verbose=verbose)
|
|
443
|
-
finally:
|
|
444
|
-
if client is not None:
|
|
445
|
-
client.close()
|
|
446
|
-
|
|
447
|
-
common.console.print(f"[bold green]Compile step retried:[/] {name}/{step_name}")
|
|
448
|
-
|
|
449
|
-
|
|
450
412
|
@app.command("delete")
|
|
451
413
|
def delete_compile(
|
|
452
414
|
name: str = typer.Argument(..., help="Compile CR name."),
|
|
@@ -126,6 +126,27 @@ def _print_version_detail(version) -> None:
|
|
|
126
126
|
)
|
|
127
127
|
|
|
128
128
|
|
|
129
|
+
def _print_profile(profile) -> None:
|
|
130
|
+
common.print_table(
|
|
131
|
+
["Field", "Value"],
|
|
132
|
+
[
|
|
133
|
+
["Version ID", profile.version_id],
|
|
134
|
+
["Status", profile.status],
|
|
135
|
+
["Row Count", profile.row_count],
|
|
136
|
+
["Byte Size", profile.byte_size],
|
|
137
|
+
["Split Profiles", json.dumps(profile.split_profiles, ensure_ascii=False)],
|
|
138
|
+
[
|
|
139
|
+
"Column Profiles",
|
|
140
|
+
json.dumps(profile.column_profiles, ensure_ascii=False),
|
|
141
|
+
],
|
|
142
|
+
["Warnings", json.dumps(profile.warnings, ensure_ascii=False)],
|
|
143
|
+
["Schema Version", profile.profile_schema_version],
|
|
144
|
+
["Error", profile.error or ""],
|
|
145
|
+
["Computed At", profile.computed_at or ""],
|
|
146
|
+
],
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
129
150
|
@app.command("list")
|
|
130
151
|
def list_datasets(
|
|
131
152
|
project: str | None = typer.Option(
|
|
@@ -477,6 +498,73 @@ def get_version(
|
|
|
477
498
|
_print_version_detail(result)
|
|
478
499
|
|
|
479
500
|
|
|
501
|
+
@version_app.command("profile")
|
|
502
|
+
def get_version_profile(
|
|
503
|
+
dataset: str = typer.Argument(..., help="Dataset name or UUID."),
|
|
504
|
+
version: str = typer.Argument(..., help="Dataset version name or UUID."),
|
|
505
|
+
wait: bool = typer.Option(False, "--wait", help="Wait for profile completion."),
|
|
506
|
+
timeout: float = typer.Option(
|
|
507
|
+
1800, "--timeout", min=0.001, help="Wait timeout in seconds."
|
|
508
|
+
),
|
|
509
|
+
interval: float = typer.Option(
|
|
510
|
+
5, "--interval", min=0.001, help="Polling interval in seconds."
|
|
511
|
+
),
|
|
512
|
+
project: str | None = typer.Option(
|
|
513
|
+
None,
|
|
514
|
+
"--project",
|
|
515
|
+
help="Project name.",
|
|
516
|
+
show_default=False,
|
|
517
|
+
),
|
|
518
|
+
project_id: int | None = typer.Option(
|
|
519
|
+
None,
|
|
520
|
+
"--project-id",
|
|
521
|
+
min=1,
|
|
522
|
+
help="Project ID.",
|
|
523
|
+
show_default=False,
|
|
524
|
+
),
|
|
525
|
+
profile_name: str | None = typer.Option(None, "--profile", help="Profile name."),
|
|
526
|
+
output: common.OutputChoice = typer.Option(
|
|
527
|
+
common.OutputChoice.table,
|
|
528
|
+
"--output",
|
|
529
|
+
help="Output format.",
|
|
530
|
+
),
|
|
531
|
+
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
532
|
+
):
|
|
533
|
+
common.validate_wait_options(timeout, interval)
|
|
534
|
+
client = None
|
|
535
|
+
try:
|
|
536
|
+
resolved_project = _project_ref(project, project_id, profile_name)
|
|
537
|
+
client = common.make_client(profile_name)
|
|
538
|
+
dataset_id = _resolve_dataset_id(client, dataset, project=resolved_project)
|
|
539
|
+
version_id = _resolve_version_id(
|
|
540
|
+
client, dataset_id, version, project=resolved_project
|
|
541
|
+
)
|
|
542
|
+
if wait:
|
|
543
|
+
result = client.datasets.versions.wait_for_profile(
|
|
544
|
+
dataset_id,
|
|
545
|
+
version_id,
|
|
546
|
+
project=resolved_project,
|
|
547
|
+
timeout=timeout,
|
|
548
|
+
interval=interval,
|
|
549
|
+
)
|
|
550
|
+
else:
|
|
551
|
+
result = client.datasets.versions.profile(
|
|
552
|
+
dataset_id, version_id, project=resolved_project
|
|
553
|
+
)
|
|
554
|
+
except Exception as exc:
|
|
555
|
+
common.handle_cli_error(exc, verbose=verbose)
|
|
556
|
+
finally:
|
|
557
|
+
if client is not None:
|
|
558
|
+
client.close()
|
|
559
|
+
|
|
560
|
+
if output == common.OutputChoice.json:
|
|
561
|
+
common.print_json(result)
|
|
562
|
+
else:
|
|
563
|
+
_print_profile(result)
|
|
564
|
+
if wait and result.status == "failed":
|
|
565
|
+
raise typer.Exit(code=1)
|
|
566
|
+
|
|
567
|
+
|
|
480
568
|
@version_app.command("create")
|
|
481
569
|
def create_version(
|
|
482
570
|
dataset: str = typer.Argument(..., help="Dataset name or UUID."),
|
|
@@ -496,7 +584,9 @@ def create_version(
|
|
|
496
584
|
),
|
|
497
585
|
source: str = typer.Option(..., "--source", help="pvc, file, or huggingface."),
|
|
498
586
|
data_uri: str | None = typer.Option(None, "--data-uri", help="PVC data URI."),
|
|
499
|
-
file: Path | None = typer.Option(
|
|
587
|
+
file: Path | None = typer.Option(
|
|
588
|
+
None, "--file", help="JSONL, CSV, TSV, or Parquet file to upload."
|
|
589
|
+
),
|
|
500
590
|
repository: str | None = typer.Option(
|
|
501
591
|
None, "--repository", help="Hugging Face repository ID."
|
|
502
592
|
),
|