nufi-python 1.0.0__tar.gz → 1.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nufi_python-1.0.0 → nufi_python-1.1.1}/PKG-INFO +39 -53
- {nufi_python-1.0.0 → nufi_python-1.1.1}/README.md +38 -52
- {nufi_python-1.0.0 → nufi_python-1.1.1}/pyproject.toml +1 -1
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/__init__.py +0 -2
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/compiles.py +0 -7
- nufi_python-1.1.1/src/nufi/models/pagination.py +16 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/servings.py +3 -1
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/base.py +2 -29
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/compiles.py +5 -10
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/datasets.py +8 -23
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/evaluations.py +4 -14
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/leaderboards.py +6 -21
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/models.py +18 -54
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/PKG-INFO +39 -53
- nufi_python-1.0.0/src/nufi/models/pagination.py +0 -33
- {nufi_python-1.0.0 → nufi_python-1.1.1}/LICENSE +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/setup.cfg +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/__init__.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/auth.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/__init__.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/__init__.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/auth.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/common.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/compiles.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/custom_evaluation_templates.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/datasets.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/evaluations.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/leaderboards.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/models.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/monitoring.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/profile.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/projects.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/servings.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/main.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/client.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/config.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/evaluator.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/exceptions.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/http_utils.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/custom_evaluation_templates.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/datasets.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/evaluations.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/leaderboards.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/monitoring.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/projects.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/models/registry.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/py.typed +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/__init__.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/custom_evaluation_templates.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/monitoring.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/projects.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/resources/servings.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/transport.py +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/SOURCES.txt +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/dependency_links.txt +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/entry_points.txt +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/requires.txt +0 -0
- {nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi_python.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nufi-python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: Python SDK and CLI for the NuFi platform.
|
|
5
5
|
Author: Dudaji Inc.
|
|
6
6
|
License-Expression: MIT
|
|
@@ -22,34 +22,41 @@ Dynamic: license-file
|
|
|
22
22
|
|
|
23
23
|
# NuFi Python SDK/CLI
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
command.
|
|
25
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
26
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
27
29
|
|
|
28
30
|
```bash
|
|
29
|
-
|
|
30
|
-
python -m pip install ./dist/nufi_python-1.0.0-py3-none-any.whl
|
|
31
|
+
python -m pip install nufi-python
|
|
31
32
|
nufi --version
|
|
33
|
+
nufi --help
|
|
32
34
|
```
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
```
|
|
36
|
+
Install the optional evaluator dependencies when writing a custom evaluator that reads
|
|
37
|
+
NuFi Parquet datasets:
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
```bash
|
|
40
|
+
python -m pip install "nufi-python[evaluator]"
|
|
41
|
+
```
|
|
41
42
|
|
|
42
|
-
##
|
|
43
|
+
## Configure access
|
|
43
44
|
|
|
44
|
-
|
|
45
|
+
Ask your NuFi administrator for the installation's base domain and a project you can
|
|
46
|
+
access. Create a profile, log in, and select the project once:
|
|
45
47
|
|
|
46
48
|
```bash
|
|
47
|
-
nufi profile create
|
|
48
|
-
nufi profile use
|
|
49
|
+
nufi profile create production --base-domain nufi.example
|
|
50
|
+
nufi profile use production
|
|
49
51
|
nufi login
|
|
50
52
|
nufi project use team-a
|
|
51
53
|
```
|
|
52
54
|
|
|
55
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
56
|
+
printed verification URL elsewhere.
|
|
57
|
+
|
|
58
|
+
## CLI quickstart
|
|
59
|
+
|
|
53
60
|
The CLI workflow is composable; every long-running identity can be resumed with its
|
|
54
61
|
dedicated `wait` command:
|
|
55
62
|
|
|
@@ -69,7 +76,9 @@ nufi leaderboard submission wait quality SUBMISSION_ID
|
|
|
69
76
|
nufi leaderboard entries quality
|
|
70
77
|
```
|
|
71
78
|
|
|
72
|
-
|
|
79
|
+
## Python SDK quickstart
|
|
80
|
+
|
|
81
|
+
The same flow is available in Python:
|
|
73
82
|
|
|
74
83
|
```python
|
|
75
84
|
from nufi import NuFi
|
|
@@ -105,27 +114,16 @@ with NuFi.from_profile() as client:
|
|
|
105
114
|
entries = client.leaderboards.entries(leaderboard.id)
|
|
106
115
|
```
|
|
107
116
|
|
|
108
|
-
## Local Run
|
|
109
|
-
|
|
110
|
-
Install dependencies:
|
|
111
|
-
|
|
112
|
-
```bash
|
|
113
|
-
uv sync
|
|
114
|
-
```
|
|
115
|
-
|
|
116
|
-
Run the CLI locally:
|
|
117
|
-
|
|
118
|
-
```bash
|
|
119
|
-
uv run nufi --help
|
|
120
|
-
uv run nufi profile
|
|
121
|
-
```
|
|
122
|
-
|
|
123
117
|
## Evaluations
|
|
124
118
|
|
|
125
119
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
126
120
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
127
121
|
without a client-side allowlist.
|
|
128
122
|
|
|
123
|
+
The `nufi.evaluator.run` helper exits non-zero when evaluation or result submission
|
|
124
|
+
fails. NuFi determines Run success from Job success and durable API acceptance;
|
|
125
|
+
evaluator output is diagnostic only.
|
|
126
|
+
|
|
129
127
|
`evaluation method templates` currently exposes lm-eval templates, metrics, and
|
|
130
128
|
filters. It is an optional discovery command and is not needed when you already have a
|
|
131
129
|
Method config.
|
|
@@ -191,20 +189,20 @@ nativeConfig:
|
|
|
191
189
|
```
|
|
192
190
|
|
|
193
191
|
```bash
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
192
|
+
nufi evaluation method templates --project team-a
|
|
193
|
+
nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
|
|
194
|
+
nufi evaluation method create gsm8k v1 gsm8k-exact-match \
|
|
197
195
|
--config method.yaml --project team-a
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
196
|
+
nufi evaluation method list --dataset gsm8k --version v1 --project team-a
|
|
197
|
+
nufi evaluation method archive gsm8k-exact-match --yes --project team-a
|
|
198
|
+
nufi evaluation method restore gsm8k-exact-match --project team-a
|
|
201
199
|
|
|
202
|
-
|
|
200
|
+
nufi evaluation run create gsm8k-exact-match \
|
|
203
201
|
--model qwen2 --model-version v1 --artifact original \
|
|
204
202
|
--project team-a
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
203
|
+
nufi evaluation run list --dataset gsm8k --version v1 --project team-a
|
|
204
|
+
nufi evaluation run get RUN_ID --project team-a
|
|
205
|
+
nufi evaluation run cancel RUN_ID --yes --project team-a
|
|
208
206
|
```
|
|
209
207
|
|
|
210
208
|
Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
|
|
@@ -214,15 +212,3 @@ Run detail table output shows aggregate and diagnostic summaries only. Use expli
|
|
|
214
212
|
`--output json` to include the authorized bounded diagnostic input, output, reference,
|
|
215
213
|
score, and error fields returned by the API. Additive response fields are preserved;
|
|
216
214
|
legacy Method/Run responses missing required v2 fields are rejected.
|
|
217
|
-
|
|
218
|
-
Run tests:
|
|
219
|
-
|
|
220
|
-
```bash
|
|
221
|
-
uv run pytest
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
Build the package:
|
|
225
|
-
|
|
226
|
-
```bash
|
|
227
|
-
uv build --clear
|
|
228
|
-
```
|
|
@@ -1,33 +1,40 @@
|
|
|
1
1
|
# NuFi Python SDK/CLI
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
command.
|
|
3
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
4
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
5
7
|
|
|
6
8
|
```bash
|
|
7
|
-
|
|
8
|
-
python -m pip install ./dist/nufi_python-1.0.0-py3-none-any.whl
|
|
9
|
+
python -m pip install nufi-python
|
|
9
10
|
nufi --version
|
|
11
|
+
nufi --help
|
|
10
12
|
```
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
```
|
|
14
|
+
Install the optional evaluator dependencies when writing a custom evaluator that reads
|
|
15
|
+
NuFi Parquet datasets:
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
```bash
|
|
18
|
+
python -m pip install "nufi-python[evaluator]"
|
|
19
|
+
```
|
|
19
20
|
|
|
20
|
-
##
|
|
21
|
+
## Configure access
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
Ask your NuFi administrator for the installation's base domain and a project you can
|
|
24
|
+
access. Create a profile, log in, and select the project once:
|
|
23
25
|
|
|
24
26
|
```bash
|
|
25
|
-
nufi profile create
|
|
26
|
-
nufi profile use
|
|
27
|
+
nufi profile create production --base-domain nufi.example
|
|
28
|
+
nufi profile use production
|
|
27
29
|
nufi login
|
|
28
30
|
nufi project use team-a
|
|
29
31
|
```
|
|
30
32
|
|
|
33
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
34
|
+
printed verification URL elsewhere.
|
|
35
|
+
|
|
36
|
+
## CLI quickstart
|
|
37
|
+
|
|
31
38
|
The CLI workflow is composable; every long-running identity can be resumed with its
|
|
32
39
|
dedicated `wait` command:
|
|
33
40
|
|
|
@@ -47,7 +54,9 @@ nufi leaderboard submission wait quality SUBMISSION_ID
|
|
|
47
54
|
nufi leaderboard entries quality
|
|
48
55
|
```
|
|
49
56
|
|
|
50
|
-
|
|
57
|
+
## Python SDK quickstart
|
|
58
|
+
|
|
59
|
+
The same flow is available in Python:
|
|
51
60
|
|
|
52
61
|
```python
|
|
53
62
|
from nufi import NuFi
|
|
@@ -83,27 +92,16 @@ with NuFi.from_profile() as client:
|
|
|
83
92
|
entries = client.leaderboards.entries(leaderboard.id)
|
|
84
93
|
```
|
|
85
94
|
|
|
86
|
-
## Local Run
|
|
87
|
-
|
|
88
|
-
Install dependencies:
|
|
89
|
-
|
|
90
|
-
```bash
|
|
91
|
-
uv sync
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
Run the CLI locally:
|
|
95
|
-
|
|
96
|
-
```bash
|
|
97
|
-
uv run nufi --help
|
|
98
|
-
uv run nufi profile
|
|
99
|
-
```
|
|
100
|
-
|
|
101
95
|
## Evaluations
|
|
102
96
|
|
|
103
97
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
104
98
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
105
99
|
without a client-side allowlist.
|
|
106
100
|
|
|
101
|
+
The `nufi.evaluator.run` helper exits non-zero when evaluation or result submission
|
|
102
|
+
fails. NuFi determines Run success from Job success and durable API acceptance;
|
|
103
|
+
evaluator output is diagnostic only.
|
|
104
|
+
|
|
107
105
|
`evaluation method templates` currently exposes lm-eval templates, metrics, and
|
|
108
106
|
filters. It is an optional discovery command and is not needed when you already have a
|
|
109
107
|
Method config.
|
|
@@ -169,20 +167,20 @@ nativeConfig:
|
|
|
169
167
|
```
|
|
170
168
|
|
|
171
169
|
```bash
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
170
|
+
nufi evaluation method templates --project team-a
|
|
171
|
+
nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
|
|
172
|
+
nufi evaluation method create gsm8k v1 gsm8k-exact-match \
|
|
175
173
|
--config method.yaml --project team-a
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
174
|
+
nufi evaluation method list --dataset gsm8k --version v1 --project team-a
|
|
175
|
+
nufi evaluation method archive gsm8k-exact-match --yes --project team-a
|
|
176
|
+
nufi evaluation method restore gsm8k-exact-match --project team-a
|
|
179
177
|
|
|
180
|
-
|
|
178
|
+
nufi evaluation run create gsm8k-exact-match \
|
|
181
179
|
--model qwen2 --model-version v1 --artifact original \
|
|
182
180
|
--project team-a
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
181
|
+
nufi evaluation run list --dataset gsm8k --version v1 --project team-a
|
|
182
|
+
nufi evaluation run get RUN_ID --project team-a
|
|
183
|
+
nufi evaluation run cancel RUN_ID --yes --project team-a
|
|
186
184
|
```
|
|
187
185
|
|
|
188
186
|
Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
|
|
@@ -192,15 +190,3 @@ Run detail table output shows aggregate and diagnostic summaries only. Use expli
|
|
|
192
190
|
`--output json` to include the authorized bounded diagnostic input, output, reference,
|
|
193
191
|
score, and error fields returned by the API. Additive response fields are preserved;
|
|
194
192
|
legacy Method/Run responses missing required v2 fields are rejected.
|
|
195
|
-
|
|
196
|
-
Run tests:
|
|
197
|
-
|
|
198
|
-
```bash
|
|
199
|
-
uv run pytest
|
|
200
|
-
```
|
|
201
|
-
|
|
202
|
-
Build the package:
|
|
203
|
-
|
|
204
|
-
```bash
|
|
205
|
-
uv build --clear
|
|
206
|
-
```
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from nufi.models.compiles import (
|
|
2
2
|
CompileConfig,
|
|
3
|
-
CompileListResponse,
|
|
4
3
|
CompileOutputFile,
|
|
5
4
|
CompileRun,
|
|
6
5
|
CompileStep,
|
|
@@ -104,7 +103,6 @@ __all__ = [
|
|
|
104
103
|
"ClusterOverview",
|
|
105
104
|
"ClusterTimeseriesData",
|
|
106
105
|
"CompileConfig",
|
|
107
|
-
"CompileListResponse",
|
|
108
106
|
"CompileOutputFile",
|
|
109
107
|
"CompileRun",
|
|
110
108
|
"CompileStep",
|
|
@@ -70,10 +70,3 @@ class CompileRun(BaseModel):
|
|
|
70
70
|
@classmethod
|
|
71
71
|
def default_null_steps(cls, value: Any) -> Any:
|
|
72
72
|
return [] if value is None else value
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
class CompileListResponse(BaseModel):
|
|
76
|
-
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
77
|
-
|
|
78
|
-
items: list[CompileRun] = Field(default_factory=list)
|
|
79
|
-
total_count: int = Field(default=0, alias="totalCount")
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Generic, TypeVar
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
T = TypeVar("T")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Page(BaseModel, Generic[T]):
|
|
11
|
+
"""One server page with its pagination metadata."""
|
|
12
|
+
|
|
13
|
+
items: list[T]
|
|
14
|
+
total: int
|
|
15
|
+
page: int
|
|
16
|
+
limit: int
|
|
@@ -19,7 +19,9 @@ class Serving(BaseModel):
|
|
|
19
19
|
|
|
20
20
|
@model_validator(mode="before")
|
|
21
21
|
@classmethod
|
|
22
|
-
def
|
|
22
|
+
def normalize_defaults(cls, data: Any) -> Any:
|
|
23
|
+
if isinstance(data, dict) and data.get("status") is None:
|
|
24
|
+
data = {**data, "status": {}}
|
|
23
25
|
if isinstance(data, dict) and not data.get("name"):
|
|
24
26
|
metadata = data.get("metadata")
|
|
25
27
|
if isinstance(metadata, dict) and metadata.get("name"):
|
|
@@ -81,39 +81,12 @@ class BaseResource:
|
|
|
81
81
|
params["limit"] = resolved_limit
|
|
82
82
|
return params, resolved_page or 1, resolved_limit or 100
|
|
83
83
|
|
|
84
|
-
@
|
|
84
|
+
@staticmethod
|
|
85
85
|
def _page(
|
|
86
|
-
cls,
|
|
87
86
|
payload: dict[str, Any],
|
|
88
87
|
model: type[T],
|
|
89
|
-
*,
|
|
90
|
-
item_keys: tuple[str, ...] = ("items",),
|
|
91
|
-
page: int,
|
|
92
|
-
limit: int,
|
|
93
88
|
) -> Page[T]:
|
|
94
|
-
|
|
95
|
-
for key in item_keys:
|
|
96
|
-
value = payload.get(key)
|
|
97
|
-
if isinstance(value, list):
|
|
98
|
-
raw_items = value
|
|
99
|
-
break
|
|
100
|
-
items = [model.model_validate(item) for item in raw_items]
|
|
101
|
-
normalized = {
|
|
102
|
-
key: value
|
|
103
|
-
for key, value in payload.items()
|
|
104
|
-
if key not in {*item_keys, "totalCount"}
|
|
105
|
-
}
|
|
106
|
-
return Page[T].model_validate(
|
|
107
|
-
{
|
|
108
|
-
**normalized,
|
|
109
|
-
"items": items,
|
|
110
|
-
"total": int(
|
|
111
|
-
payload.get("total", payload.get("totalCount", len(items)))
|
|
112
|
-
),
|
|
113
|
-
"page": int(payload.get("page", page)),
|
|
114
|
-
"limit": int(payload.get("limit", limit)),
|
|
115
|
-
}
|
|
116
|
-
)
|
|
89
|
+
return Page[model].model_validate(payload)
|
|
117
90
|
|
|
118
91
|
@staticmethod
|
|
119
92
|
def _iter_all(
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
from collections.abc import Iterator
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
|
-
from nufi.models import
|
|
6
|
+
from nufi.models import CompileRun, Page
|
|
7
7
|
from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
|
|
8
8
|
|
|
9
9
|
_TERMINAL_PHASES = frozenset({"Registered", "Failed", "RegistrationFailed"})
|
|
@@ -32,7 +32,7 @@ class CompilesResource(BaseResource):
|
|
|
32
32
|
f"/api/v1/projects/{project_id}/npu-porting-pipelines",
|
|
33
33
|
params=params,
|
|
34
34
|
)
|
|
35
|
-
return
|
|
35
|
+
return self._page(response_json(response), CompileRun).items
|
|
36
36
|
|
|
37
37
|
def list_page(
|
|
38
38
|
self,
|
|
@@ -44,9 +44,9 @@ class CompilesResource(BaseResource):
|
|
|
44
44
|
page: int | None = None,
|
|
45
45
|
limit: int | None = None,
|
|
46
46
|
) -> Page[CompileRun]:
|
|
47
|
-
"""Return one compile-run page with
|
|
47
|
+
"""Return one compile-run page with server pagination metadata."""
|
|
48
48
|
project_id = self._resolve_project_id(project)
|
|
49
|
-
params,
|
|
49
|
+
params, _, _ = self._pagination_params(
|
|
50
50
|
page, limit, default_page=1, default_limit=100
|
|
51
51
|
)
|
|
52
52
|
if model_name:
|
|
@@ -59,12 +59,7 @@ class CompilesResource(BaseResource):
|
|
|
59
59
|
f"/api/v1/projects/{project_id}/npu-porting-pipelines",
|
|
60
60
|
params=params,
|
|
61
61
|
)
|
|
62
|
-
return self._page(
|
|
63
|
-
response_json(response),
|
|
64
|
-
CompileRun,
|
|
65
|
-
page=resolved_page,
|
|
66
|
-
limit=resolved_limit,
|
|
67
|
-
)
|
|
62
|
+
return self._page(response_json(response), CompileRun)
|
|
68
63
|
|
|
69
64
|
def iter_all(
|
|
70
65
|
self,
|
|
@@ -32,7 +32,7 @@ from nufi.models import (
|
|
|
32
32
|
from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
|
|
33
33
|
from nufi.transport import SyncTransport
|
|
34
34
|
|
|
35
|
-
MAX_DATASET_UPLOAD_SIZE =
|
|
35
|
+
MAX_DATASET_UPLOAD_SIZE = 10 * 1024**3
|
|
36
36
|
TUS_CHUNK_SIZE = 64 * 1024**2
|
|
37
37
|
TUS_RETRIES = 3
|
|
38
38
|
TUS_RETRY_DELAY = 1
|
|
@@ -52,7 +52,7 @@ def _validated_jsonl(path: Path) -> int:
|
|
|
52
52
|
if size == 0:
|
|
53
53
|
raise ValueError("Dataset upload file must not be empty.")
|
|
54
54
|
if size > MAX_DATASET_UPLOAD_SIZE:
|
|
55
|
-
raise ValueError("Dataset upload file must not exceed
|
|
55
|
+
raise ValueError("Dataset upload file must not exceed 10 GiB.")
|
|
56
56
|
return size
|
|
57
57
|
|
|
58
58
|
|
|
@@ -295,18 +295,13 @@ class DatasetsResource(BaseResource):
|
|
|
295
295
|
) -> Page[Dataset]:
|
|
296
296
|
"""Return one Dataset page with server pagination metadata."""
|
|
297
297
|
project_id = self._resolve_project_id(project)
|
|
298
|
-
params,
|
|
298
|
+
params, _, _ = self._pagination_params(
|
|
299
299
|
page, limit, default_page=1, default_limit=20
|
|
300
300
|
)
|
|
301
301
|
response = self._transport.get(
|
|
302
302
|
f"/api/v1/projects/{project_id}/datasets", params=params
|
|
303
303
|
)
|
|
304
|
-
return self._page(
|
|
305
|
-
response_json(response),
|
|
306
|
-
Dataset,
|
|
307
|
-
page=resolved_page,
|
|
308
|
-
limit=resolved_limit,
|
|
309
|
-
)
|
|
304
|
+
return self._page(response_json(response), Dataset)
|
|
310
305
|
|
|
311
306
|
def iter_all(
|
|
312
307
|
self,
|
|
@@ -396,19 +391,14 @@ class DatasetVersionsResource(BaseResource):
|
|
|
396
391
|
) -> Page[DatasetVersion]:
|
|
397
392
|
"""Return one Dataset Version page with pagination metadata."""
|
|
398
393
|
project_id = self._resolve_project_id(project)
|
|
399
|
-
params,
|
|
394
|
+
params, _, _ = self._pagination_params(
|
|
400
395
|
page, limit, default_page=1, default_limit=20
|
|
401
396
|
)
|
|
402
397
|
response = self._transport.get(
|
|
403
398
|
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
|
|
404
399
|
params=params,
|
|
405
400
|
)
|
|
406
|
-
return self._page(
|
|
407
|
-
response_json(response),
|
|
408
|
-
DatasetVersion,
|
|
409
|
-
page=resolved_page,
|
|
410
|
-
limit=resolved_limit,
|
|
411
|
-
)
|
|
401
|
+
return self._page(response_json(response), DatasetVersion)
|
|
412
402
|
|
|
413
403
|
def iter_all(
|
|
414
404
|
self,
|
|
@@ -605,19 +595,14 @@ class DatasetImportsResource(BaseResource):
|
|
|
605
595
|
) -> Page[DatasetImport]:
|
|
606
596
|
"""Return one Dataset import page with pagination metadata."""
|
|
607
597
|
project_id = self._resolve_project_id(project)
|
|
608
|
-
params,
|
|
598
|
+
params, _, _ = self._pagination_params(
|
|
609
599
|
page, limit, default_page=1, default_limit=20
|
|
610
600
|
)
|
|
611
601
|
response = self._transport.get(
|
|
612
602
|
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
|
|
613
603
|
params=params,
|
|
614
604
|
)
|
|
615
|
-
return self._page(
|
|
616
|
-
response_json(response),
|
|
617
|
-
DatasetImport,
|
|
618
|
-
page=resolved_page,
|
|
619
|
-
limit=resolved_limit,
|
|
620
|
-
)
|
|
605
|
+
return self._page(response_json(response), DatasetImport)
|
|
621
606
|
|
|
622
607
|
def iter_all(
|
|
623
608
|
self,
|
|
@@ -115,7 +115,7 @@ class EvaluationMethodsResource(BaseResource):
|
|
|
115
115
|
)
|
|
116
116
|
else:
|
|
117
117
|
path = f"/api/v1/projects/{project_id}/evaluations/tasks"
|
|
118
|
-
pagination,
|
|
118
|
+
pagination, _, _ = self._pagination_params(
|
|
119
119
|
page, limit, default_page=1, default_limit=20
|
|
120
120
|
)
|
|
121
121
|
response = self._transport.get(
|
|
@@ -134,12 +134,7 @@ class EvaluationMethodsResource(BaseResource):
|
|
|
134
134
|
**pagination,
|
|
135
135
|
},
|
|
136
136
|
)
|
|
137
|
-
return self._page(
|
|
138
|
-
response_json(response),
|
|
139
|
-
EvaluationMethod,
|
|
140
|
-
page=resolved_page,
|
|
141
|
-
limit=resolved_limit,
|
|
142
|
-
)
|
|
137
|
+
return self._page(response_json(response), EvaluationMethod)
|
|
143
138
|
|
|
144
139
|
def iter_all(
|
|
145
140
|
self,
|
|
@@ -311,7 +306,7 @@ class EvaluationsResource(BaseResource):
|
|
|
311
306
|
if dataset_version_id is not None and dataset_id is None:
|
|
312
307
|
raise ValueError("dataset_version_id requires dataset_id")
|
|
313
308
|
project_id = self._resolve_project_id(project)
|
|
314
|
-
pagination,
|
|
309
|
+
pagination, _, _ = self._pagination_params(
|
|
315
310
|
page, limit, default_page=1, default_limit=20
|
|
316
311
|
)
|
|
317
312
|
response = self._transport.get(
|
|
@@ -333,12 +328,7 @@ class EvaluationsResource(BaseResource):
|
|
|
333
328
|
**pagination,
|
|
334
329
|
},
|
|
335
330
|
)
|
|
336
|
-
return self._page(
|
|
337
|
-
response_json(response),
|
|
338
|
-
EvaluationRun,
|
|
339
|
-
page=resolved_page,
|
|
340
|
-
limit=resolved_limit,
|
|
341
|
-
)
|
|
331
|
+
return self._page(response_json(response), EvaluationRun)
|
|
342
332
|
|
|
343
333
|
def iter_all(
|
|
344
334
|
self,
|
|
@@ -60,19 +60,14 @@ class LeaderboardsResource(BaseResource):
|
|
|
60
60
|
) -> Page[Leaderboard]:
|
|
61
61
|
"""Return one Leaderboard page with pagination metadata."""
|
|
62
62
|
project_id = self._resolve_project_id(project)
|
|
63
|
-
pagination,
|
|
63
|
+
pagination, _, _ = self._pagination_params(
|
|
64
64
|
page, limit, default_page=1, default_limit=20
|
|
65
65
|
)
|
|
66
66
|
response = self._transport.get(
|
|
67
67
|
f"/api/v1/projects/{project_id}/leaderboards",
|
|
68
68
|
params={**_without_none({"status": status}), **pagination},
|
|
69
69
|
)
|
|
70
|
-
return self._page(
|
|
71
|
-
response_json(response),
|
|
72
|
-
Leaderboard,
|
|
73
|
-
page=resolved_page,
|
|
74
|
-
limit=resolved_limit,
|
|
75
|
-
)
|
|
70
|
+
return self._page(response_json(response), Leaderboard)
|
|
76
71
|
|
|
77
72
|
def iter_all(
|
|
78
73
|
self,
|
|
@@ -228,7 +223,7 @@ class LeaderboardsResource(BaseResource):
|
|
|
228
223
|
) -> Page[LeaderboardSubmission]:
|
|
229
224
|
"""Return one submission page with pagination metadata."""
|
|
230
225
|
project_id = self._resolve_project_id(project)
|
|
231
|
-
pagination,
|
|
226
|
+
pagination, _, _ = self._pagination_params(
|
|
232
227
|
page, limit, default_page=1, default_limit=20
|
|
233
228
|
)
|
|
234
229
|
response = self._transport.get(
|
|
@@ -236,12 +231,7 @@ class LeaderboardsResource(BaseResource):
|
|
|
236
231
|
f"{escape_path(leaderboard_id)}/submissions",
|
|
237
232
|
params=pagination,
|
|
238
233
|
)
|
|
239
|
-
return self._page(
|
|
240
|
-
response_json(response),
|
|
241
|
-
LeaderboardSubmission,
|
|
242
|
-
page=resolved_page,
|
|
243
|
-
limit=resolved_limit,
|
|
244
|
-
)
|
|
234
|
+
return self._page(response_json(response), LeaderboardSubmission)
|
|
245
235
|
|
|
246
236
|
def iter_all_submissions(
|
|
247
237
|
self,
|
|
@@ -367,7 +357,7 @@ class LeaderboardsResource(BaseResource):
|
|
|
367
357
|
) -> Page[LeaderboardEntry]:
|
|
368
358
|
"""Return one ranked-entry page with pagination metadata."""
|
|
369
359
|
project_id = self._resolve_project_id(project)
|
|
370
|
-
pagination,
|
|
360
|
+
pagination, _, _ = self._pagination_params(
|
|
371
361
|
page, limit, default_page=1, default_limit=20
|
|
372
362
|
)
|
|
373
363
|
response = self._transport.get(
|
|
@@ -383,12 +373,7 @@ class LeaderboardsResource(BaseResource):
|
|
|
383
373
|
**pagination,
|
|
384
374
|
},
|
|
385
375
|
)
|
|
386
|
-
return self._page(
|
|
387
|
-
response_json(response),
|
|
388
|
-
LeaderboardEntry,
|
|
389
|
-
page=resolved_page,
|
|
390
|
-
limit=resolved_limit,
|
|
391
|
-
)
|
|
376
|
+
return self._page(response_json(response), LeaderboardEntry)
|
|
392
377
|
|
|
393
378
|
def iter_all_entries(
|
|
394
379
|
self,
|
|
@@ -4,7 +4,6 @@ import time
|
|
|
4
4
|
from collections.abc import Iterator
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from nufi.exceptions import NotFoundError
|
|
8
7
|
from nufi.models import (
|
|
9
8
|
ArtifactValidation,
|
|
10
9
|
HuggingFaceMapping,
|
|
@@ -68,11 +67,7 @@ class ModelVersionsResource(BaseResource):
|
|
|
68
67
|
payload = response_json(
|
|
69
68
|
self._transport.get(self._path(project_id, model), params=params)
|
|
70
69
|
)
|
|
71
|
-
return
|
|
72
|
-
ModelVersion.model_validate(version)
|
|
73
|
-
for version in payload.get("versions", [])
|
|
74
|
-
if isinstance(version, dict)
|
|
75
|
-
]
|
|
70
|
+
return self._page(payload, ModelVersion).items
|
|
76
71
|
|
|
77
72
|
def list_page(
|
|
78
73
|
self,
|
|
@@ -84,19 +79,13 @@ class ModelVersionsResource(BaseResource):
|
|
|
84
79
|
) -> Page[ModelVersion]:
|
|
85
80
|
"""Return one version page with server pagination metadata."""
|
|
86
81
|
project_id = self._resolve_project_id(project)
|
|
87
|
-
params,
|
|
82
|
+
params, _, _ = self._pagination_params(
|
|
88
83
|
page, limit, default_page=1, default_limit=10
|
|
89
84
|
)
|
|
90
85
|
payload = response_json(
|
|
91
86
|
self._transport.get(self._path(project_id, model), params=params)
|
|
92
87
|
)
|
|
93
|
-
return self._page(
|
|
94
|
-
payload,
|
|
95
|
-
ModelVersion,
|
|
96
|
-
item_keys=("versions",),
|
|
97
|
-
page=resolved_page,
|
|
98
|
-
limit=resolved_limit,
|
|
99
|
-
)
|
|
88
|
+
return self._page(payload, ModelVersion)
|
|
100
89
|
|
|
101
90
|
def iter_all(
|
|
102
91
|
self,
|
|
@@ -364,19 +353,13 @@ class HuggingFaceImportsResource(BaseResource):
|
|
|
364
353
|
) -> Page[ModelImport]:
|
|
365
354
|
"""Return one Hugging Face import page."""
|
|
366
355
|
project_id = self._resolve_project_id(project)
|
|
367
|
-
params,
|
|
356
|
+
params, _, _ = self._pagination_params(
|
|
368
357
|
page, limit, default_page=1, default_limit=10
|
|
369
358
|
)
|
|
370
359
|
payload = response_json(
|
|
371
360
|
self._transport.get(self._path(project_id), params=params)
|
|
372
361
|
)
|
|
373
|
-
return self._page(
|
|
374
|
-
payload,
|
|
375
|
-
ModelImport,
|
|
376
|
-
item_keys=("jobs",),
|
|
377
|
-
page=resolved_page,
|
|
378
|
-
limit=resolved_limit,
|
|
379
|
-
)
|
|
362
|
+
return self._page(payload, ModelImport)
|
|
380
363
|
|
|
381
364
|
def iter_all(
|
|
382
365
|
self,
|
|
@@ -403,6 +386,12 @@ class HuggingFaceImportsResource(BaseResource):
|
|
|
403
386
|
for item in _response_list(self._transport.get(path))
|
|
404
387
|
]
|
|
405
388
|
|
|
389
|
+
def get(self, import_id: int, *, project: ProjectRef | None = None) -> ModelImport:
|
|
390
|
+
"""Get one Hugging Face import by ID."""
|
|
391
|
+
project_id = self._resolve_project_id(project)
|
|
392
|
+
response = self._transport.get(f"{self._path(project_id)}/{import_id}")
|
|
393
|
+
return ModelImport.model_validate(response_json(response))
|
|
394
|
+
|
|
406
395
|
def wait(
|
|
407
396
|
self,
|
|
408
397
|
import_id: int,
|
|
@@ -411,19 +400,10 @@ class HuggingFaceImportsResource(BaseResource):
|
|
|
411
400
|
timeout: float = 1800,
|
|
412
401
|
interval: float = 5,
|
|
413
402
|
) -> ModelImport:
|
|
414
|
-
"""Wait for
|
|
403
|
+
"""Wait for a Hugging Face import using its single-job GET route."""
|
|
415
404
|
deadline, interval = _wait_values(timeout, interval)
|
|
416
405
|
while True:
|
|
417
|
-
job =
|
|
418
|
-
(
|
|
419
|
-
item
|
|
420
|
-
for item in self.iter_all(project=project)
|
|
421
|
-
if item.id == import_id
|
|
422
|
-
),
|
|
423
|
-
None,
|
|
424
|
-
)
|
|
425
|
-
if job is None:
|
|
426
|
-
raise NotFoundError(f"Hugging Face import not found: {import_id}")
|
|
406
|
+
job = self.get(import_id, project=project)
|
|
427
407
|
if job.state in _IMPORT_TERMINAL_STATES:
|
|
428
408
|
return job
|
|
429
409
|
remaining = deadline - time.monotonic()
|
|
@@ -492,19 +472,13 @@ class MLflowImportsResource(BaseResource):
|
|
|
492
472
|
) -> Page[ModelImport]:
|
|
493
473
|
"""Return one MLflow import page."""
|
|
494
474
|
project_id = self._resolve_project_id(project)
|
|
495
|
-
params,
|
|
475
|
+
params, _, _ = self._pagination_params(
|
|
496
476
|
page, limit, default_page=1, default_limit=10
|
|
497
477
|
)
|
|
498
478
|
payload = response_json(
|
|
499
479
|
self._transport.get(f"{self._path(project_id)}/imports", params=params)
|
|
500
480
|
)
|
|
501
|
-
return self._page(
|
|
502
|
-
payload,
|
|
503
|
-
ModelImport,
|
|
504
|
-
item_keys=("jobs",),
|
|
505
|
-
page=resolved_page,
|
|
506
|
-
limit=resolved_limit,
|
|
507
|
-
)
|
|
481
|
+
return self._page(payload, ModelImport)
|
|
508
482
|
|
|
509
483
|
def iter_all(
|
|
510
484
|
self,
|
|
@@ -595,11 +569,7 @@ class ModelsResource(BaseResource):
|
|
|
595
569
|
f"/api/v1/projects/{project_id}/models", params=params
|
|
596
570
|
)
|
|
597
571
|
payload = response_json(response)
|
|
598
|
-
return
|
|
599
|
-
RegisteredModel.model_validate(model)
|
|
600
|
-
for model in payload.get("models", [])
|
|
601
|
-
if isinstance(model, dict)
|
|
602
|
-
]
|
|
572
|
+
return self._page(payload, RegisteredModel).items
|
|
603
573
|
|
|
604
574
|
def list_page(
|
|
605
575
|
self,
|
|
@@ -611,7 +581,7 @@ class ModelsResource(BaseResource):
|
|
|
611
581
|
) -> Page[RegisteredModel]:
|
|
612
582
|
"""Return one model page with server pagination metadata."""
|
|
613
583
|
project_id = self._resolve_project_id(project)
|
|
614
|
-
params,
|
|
584
|
+
params, _, _ = self._pagination_params(
|
|
615
585
|
page,
|
|
616
586
|
limit,
|
|
617
587
|
default_page=1,
|
|
@@ -622,13 +592,7 @@ class ModelsResource(BaseResource):
|
|
|
622
592
|
response = self._transport.get(
|
|
623
593
|
f"/api/v1/projects/{project_id}/models", params=params
|
|
624
594
|
)
|
|
625
|
-
return self._page(
|
|
626
|
-
response_json(response),
|
|
627
|
-
RegisteredModel,
|
|
628
|
-
item_keys=("models",),
|
|
629
|
-
page=resolved_page,
|
|
630
|
-
limit=resolved_limit,
|
|
631
|
-
)
|
|
595
|
+
return self._page(response_json(response), RegisteredModel)
|
|
632
596
|
|
|
633
597
|
def iter_all(
|
|
634
598
|
self,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nufi-python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: Python SDK and CLI for the NuFi platform.
|
|
5
5
|
Author: Dudaji Inc.
|
|
6
6
|
License-Expression: MIT
|
|
@@ -22,34 +22,41 @@ Dynamic: license-file
|
|
|
22
22
|
|
|
23
23
|
# NuFi Python SDK/CLI
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
command.
|
|
25
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
26
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
27
29
|
|
|
28
30
|
```bash
|
|
29
|
-
|
|
30
|
-
python -m pip install ./dist/nufi_python-1.0.0-py3-none-any.whl
|
|
31
|
+
python -m pip install nufi-python
|
|
31
32
|
nufi --version
|
|
33
|
+
nufi --help
|
|
32
34
|
```
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
```
|
|
36
|
+
Install the optional evaluator dependencies when writing a custom evaluator that reads
|
|
37
|
+
NuFi Parquet datasets:
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
```bash
|
|
40
|
+
python -m pip install "nufi-python[evaluator]"
|
|
41
|
+
```
|
|
41
42
|
|
|
42
|
-
##
|
|
43
|
+
## Configure access
|
|
43
44
|
|
|
44
|
-
|
|
45
|
+
Ask your NuFi administrator for the installation's base domain and a project you can
|
|
46
|
+
access. Create a profile, log in, and select the project once:
|
|
45
47
|
|
|
46
48
|
```bash
|
|
47
|
-
nufi profile create
|
|
48
|
-
nufi profile use
|
|
49
|
+
nufi profile create production --base-domain nufi.example
|
|
50
|
+
nufi profile use production
|
|
49
51
|
nufi login
|
|
50
52
|
nufi project use team-a
|
|
51
53
|
```
|
|
52
54
|
|
|
55
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
56
|
+
printed verification URL elsewhere.
|
|
57
|
+
|
|
58
|
+
## CLI quickstart
|
|
59
|
+
|
|
53
60
|
The CLI workflow is composable; every long-running identity can be resumed with its
|
|
54
61
|
dedicated `wait` command:
|
|
55
62
|
|
|
@@ -69,7 +76,9 @@ nufi leaderboard submission wait quality SUBMISSION_ID
|
|
|
69
76
|
nufi leaderboard entries quality
|
|
70
77
|
```
|
|
71
78
|
|
|
72
|
-
|
|
79
|
+
## Python SDK quickstart
|
|
80
|
+
|
|
81
|
+
The same flow is available in Python:
|
|
73
82
|
|
|
74
83
|
```python
|
|
75
84
|
from nufi import NuFi
|
|
@@ -105,27 +114,16 @@ with NuFi.from_profile() as client:
|
|
|
105
114
|
entries = client.leaderboards.entries(leaderboard.id)
|
|
106
115
|
```
|
|
107
116
|
|
|
108
|
-
## Local Run
|
|
109
|
-
|
|
110
|
-
Install dependencies:
|
|
111
|
-
|
|
112
|
-
```bash
|
|
113
|
-
uv sync
|
|
114
|
-
```
|
|
115
|
-
|
|
116
|
-
Run the CLI locally:
|
|
117
|
-
|
|
118
|
-
```bash
|
|
119
|
-
uv run nufi --help
|
|
120
|
-
uv run nufi profile
|
|
121
|
-
```
|
|
122
|
-
|
|
123
117
|
## Evaluations
|
|
124
118
|
|
|
125
119
|
Evaluation Method metadata is unconditional and v2-only. Method config is owned and
|
|
126
120
|
validated by the API; the SDK forwards the mapping and any caller-supplied Method type
|
|
127
121
|
without a client-side allowlist.
|
|
128
122
|
|
|
123
|
+
The `nufi.evaluator.run` helper exits non-zero when evaluation or result submission
|
|
124
|
+
fails. NuFi determines Run success from Job success and durable API acceptance;
|
|
125
|
+
evaluator output is diagnostic only.
|
|
126
|
+
|
|
129
127
|
`evaluation method templates` currently exposes lm-eval templates, metrics, and
|
|
130
128
|
filters. It is an optional discovery command and is not needed when you already have a
|
|
131
129
|
Method config.
|
|
@@ -191,20 +189,20 @@ nativeConfig:
|
|
|
191
189
|
```
|
|
192
190
|
|
|
193
191
|
```bash
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
192
|
+
nufi evaluation method templates --project team-a
|
|
193
|
+
nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
|
|
194
|
+
nufi evaluation method create gsm8k v1 gsm8k-exact-match \
|
|
197
195
|
--config method.yaml --project team-a
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
196
|
+
nufi evaluation method list --dataset gsm8k --version v1 --project team-a
|
|
197
|
+
nufi evaluation method archive gsm8k-exact-match --yes --project team-a
|
|
198
|
+
nufi evaluation method restore gsm8k-exact-match --project team-a
|
|
201
199
|
|
|
202
|
-
|
|
200
|
+
nufi evaluation run create gsm8k-exact-match \
|
|
203
201
|
--model qwen2 --model-version v1 --artifact original \
|
|
204
202
|
--project team-a
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
203
|
+
nufi evaluation run list --dataset gsm8k --version v1 --project team-a
|
|
204
|
+
nufi evaluation run get RUN_ID --project team-a
|
|
205
|
+
nufi evaluation run cancel RUN_ID --yes --project team-a
|
|
208
206
|
```
|
|
209
207
|
|
|
210
208
|
Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
|
|
@@ -214,15 +212,3 @@ Run detail table output shows aggregate and diagnostic summaries only. Use expli
|
|
|
214
212
|
`--output json` to include the authorized bounded diagnostic input, output, reference,
|
|
215
213
|
score, and error fields returned by the API. Additive response fields are preserved;
|
|
216
214
|
legacy Method/Run responses missing required v2 fields are rejected.
|
|
217
|
-
|
|
218
|
-
Run tests:
|
|
219
|
-
|
|
220
|
-
```bash
|
|
221
|
-
uv run pytest
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
Build the package:
|
|
225
|
-
|
|
226
|
-
```bash
|
|
227
|
-
uv build --clear
|
|
228
|
-
```
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import Generic, TypeVar
|
|
4
|
-
|
|
5
|
-
from pydantic import BaseModel, ConfigDict, model_validator
|
|
6
|
-
|
|
7
|
-
T = TypeVar("T")
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class Page(BaseModel, Generic[T]):
|
|
11
|
-
"""One server page with its pagination metadata."""
|
|
12
|
-
|
|
13
|
-
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
14
|
-
|
|
15
|
-
items: list[T]
|
|
16
|
-
total: int
|
|
17
|
-
page: int
|
|
18
|
-
limit: int
|
|
19
|
-
|
|
20
|
-
@model_validator(mode="before")
|
|
21
|
-
@classmethod
|
|
22
|
-
def _normalize_envelope(cls, value: object) -> object:
|
|
23
|
-
if not isinstance(value, dict):
|
|
24
|
-
return value
|
|
25
|
-
normalized = dict(value)
|
|
26
|
-
if "items" not in normalized:
|
|
27
|
-
for key in ("models", "versions", "artifacts", "jobs"):
|
|
28
|
-
if key in normalized:
|
|
29
|
-
normalized["items"] = normalized[key]
|
|
30
|
-
break
|
|
31
|
-
if "total" not in normalized and "totalCount" in normalized:
|
|
32
|
-
normalized["total"] = normalized["totalCount"]
|
|
33
|
-
return normalized
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nufi_python-1.0.0 → nufi_python-1.1.1}/src/nufi/cli/commands/custom_evaluation_templates.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|