nufi-python 1.1.2.post1__tar.gz → 1.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi_python-1.1.4/PKG-INFO +171 -0
- nufi_python-1.1.4/README.md +151 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/pyproject.toml +1 -4
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/__init__.py +22 -2
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/common.py +45 -3
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/datasets.py +32 -20
- nufi_python-1.1.4/src/nufi/cli/commands/evaluation_providers.py +290 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/evaluations.py +258 -245
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/leaderboards.py +50 -34
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/models.py +94 -70
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/monitoring.py +69 -17
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/servings.py +2 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/main.py +2 -2
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/exceptions.py +31 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/__init__.py +64 -40
- nufi_python-1.1.4/src/nufi/models/evaluation_providers.py +1034 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/evaluations.py +45 -149
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/leaderboards.py +19 -9
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/monitoring.py +63 -18
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/registry.py +0 -8
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/servings.py +2 -1
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/__init__.py +2 -2
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/base.py +38 -14
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/datasets.py +133 -40
- nufi_python-1.1.4/src/nufi/resources/evaluation_providers.py +200 -0
- nufi_python-1.1.4/src/nufi/resources/evaluations.py +637 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/leaderboards.py +54 -9
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/models.py +1 -11
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/projects.py +2 -10
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/servings.py +6 -0
- nufi_python-1.1.4/src/nufi_python.egg-info/PKG-INFO +171 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi_python.egg-info/SOURCES.txt +3 -4
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi_python.egg-info/requires.txt +0 -3
- nufi_python-1.1.2.post1/PKG-INFO +0 -274
- nufi_python-1.1.2.post1/README.md +0 -252
- nufi_python-1.1.2.post1/src/nufi/cli/commands/custom_evaluation_templates.py +0 -214
- nufi_python-1.1.2.post1/src/nufi/evaluator.py +0 -679
- nufi_python-1.1.2.post1/src/nufi/models/custom_evaluation_templates.py +0 -324
- nufi_python-1.1.2.post1/src/nufi/resources/custom_evaluation_templates.py +0 -126
- nufi_python-1.1.2.post1/src/nufi/resources/evaluations.py +0 -423
- nufi_python-1.1.2.post1/src/nufi_python.egg-info/PKG-INFO +0 -274
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/LICENSE +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/setup.cfg +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/auth.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/__init__.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/__init__.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/auth.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/compiles.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/doctor.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/profile.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/cli/commands/projects.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/client.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/config.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/http_utils.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/compiles.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/datasets.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/pagination.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/models/projects.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/py.typed +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/compiles.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/resources/monitoring.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi/transport.py +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi_python.egg-info/dependency_links.txt +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi_python.egg-info/entry_points.txt +0 -0
- {nufi_python-1.1.2.post1 → nufi_python-1.1.4}/src/nufi_python.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nufi-python
|
|
3
|
+
Version: 1.1.4
|
|
4
|
+
Summary: Python SDK and CLI for the NuFi platform.
|
|
5
|
+
Author: Dudaji Inc.
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Operating System :: OS Independent
|
|
9
|
+
Classifier: Typing :: Typed
|
|
10
|
+
Requires-Python: >=3.11
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: httpx>=0.25.0
|
|
14
|
+
Requires-Dist: PyYAML>=6.0
|
|
15
|
+
Requires-Dist: pydantic>=2.0.0
|
|
16
|
+
Requires-Dist: tuspy==1.1.0
|
|
17
|
+
Requires-Dist: typer>=0.16.0
|
|
18
|
+
Requires-Dist: rich>=13.0.0
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# NuFi Python SDK/CLI
|
|
22
|
+
|
|
23
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
24
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
python -m pip install nufi-python
|
|
30
|
+
nufi --version
|
|
31
|
+
nufi --help
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Configure access
|
|
35
|
+
|
|
36
|
+
You need a reachable NuFi installation, a user account, and access to a Project.
|
|
37
|
+
Ask your NuFi Administrator for the base domain and project name, then replace
|
|
38
|
+
`nufi.example` and `team-a` below:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
nufi profile create production --base-domain nufi.example
|
|
42
|
+
nufi profile use production
|
|
43
|
+
nufi login
|
|
44
|
+
nufi project use team-a
|
|
45
|
+
nufi model list
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
49
|
+
printed verification URL elsewhere. A successful `model list`, even when empty,
|
|
50
|
+
confirms access to the selected Project. The SDK also uses this saved Profile.
|
|
51
|
+
See [installation and authentication](https://docs.nufi.me/dev/docs/next/sdk-cli/installation-and-authentication)
|
|
52
|
+
for other connection settings.
|
|
53
|
+
|
|
54
|
+
## CLI quickstart
|
|
55
|
+
|
|
56
|
+
These examples assume a registered `qwen2` Model Version `v1` with an `original`
|
|
57
|
+
artifact, a compatible accelerator, and an existing Evaluation Method named
|
|
58
|
+
`METHOD_NAME`. Replace those names with your resources. The Leaderboard example
|
|
59
|
+
also assumes that the Method reports `exact_match` and that `quality` is a new
|
|
60
|
+
Leaderboard name. For model import and serving from scratch, follow the
|
|
61
|
+
[serving quickstart](https://docs.nufi.me/dev/docs/next/sdk-cli/quickstart).
|
|
62
|
+
|
|
63
|
+
Compile for RNGD when you need a compiled artifact:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
nufi compile create qwen2 --version v1 --wait
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Evaluate the original artifact, or submit it to a Leaderboard that starts its own
|
|
70
|
+
Evaluation Run:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
nufi evaluation run create METHOD_NAME --model qwen2 \
|
|
74
|
+
--model-version v1 --artifact original --wait
|
|
75
|
+
nufi leaderboard create quality --method METHOD_NAME \
|
|
76
|
+
--rank-metric exact_match --rank-direction desc
|
|
77
|
+
nufi leaderboard submission create quality --model qwen2 \
|
|
78
|
+
--model-version v1 --artifact original --wait
|
|
79
|
+
nufi leaderboard entries quality
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Compile succeeds at `Registered`; an Evaluation Run or a Submission's latest Run
|
|
83
|
+
succeeds at `succeeded`. The wait commands return a non-zero exit code for failure,
|
|
84
|
+
cancellation where supported, or timeout. A timeout stops local waiting and leaves
|
|
85
|
+
the server operation running. Resume with the printed name or ID:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
nufi compile wait COMPILE_NAME_OR_ID
|
|
89
|
+
nufi evaluation run wait RUN_ID
|
|
90
|
+
nufi leaderboard submission wait quality SUBMISSION_ID
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Use `--help` on a command for its options and `--output json` for structured
|
|
94
|
+
results. Evaluation detail JSON can include sensitive diagnostic inputs and
|
|
95
|
+
outputs; table output shows summaries. See the
|
|
96
|
+
[Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation)
|
|
97
|
+
for Method creation, Provider selection, and result inspection.
|
|
98
|
+
|
|
99
|
+
## Python SDK quickstart
|
|
100
|
+
|
|
101
|
+
After configuring the Profile above, use a context manager to close the client.
|
|
102
|
+
Replace `METHOD_ID` with an Evaluation Method UUID, `9` with your Model Artifact's
|
|
103
|
+
numeric ID, and `LEADERBOARD_ID` with the UUID of an existing Leaderboard such as
|
|
104
|
+
`quality` above. CLI `get` commands with `--output json` expose these IDs.
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from nufi import NuFi
|
|
108
|
+
|
|
109
|
+
with NuFi.from_profile() as client:
|
|
110
|
+
compile_run = client.compiles.create(model_name="qwen2", version="v1")
|
|
111
|
+
compiled = client.compiles.wait(compile_run.id)
|
|
112
|
+
if compiled.phase != "Registered" or compiled.registration_error:
|
|
113
|
+
raise RuntimeError(compiled.error_message or compiled.registration_error or "Compile failed")
|
|
114
|
+
|
|
115
|
+
run = client.evaluations.create(
|
|
116
|
+
evaluation_method_id="METHOD_ID",
|
|
117
|
+
model_artifact_id=9,
|
|
118
|
+
cpu="4",
|
|
119
|
+
memory="8Gi",
|
|
120
|
+
accelerator_count=1,
|
|
121
|
+
)
|
|
122
|
+
evaluated = client.evaluations.wait(run.id)
|
|
123
|
+
if evaluated.status != "succeeded":
|
|
124
|
+
raise RuntimeError(f"Evaluation {run.id} ended in {evaluated.status}")
|
|
125
|
+
print(evaluated.aggregate_metrics)
|
|
126
|
+
|
|
127
|
+
submission = client.leaderboards.submit("LEADERBOARD_ID", model_artifact_id=9)
|
|
128
|
+
completed = client.leaderboards.wait("LEADERBOARD_ID", submission.id)
|
|
129
|
+
print(completed.latest_run.status)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
SDK `wait()` methods return terminal objects, including failed or cancelled Runs;
|
|
133
|
+
check their status before using results. Request errors raise
|
|
134
|
+
`nufi.exceptions.NuFiError` subclasses, and a wait timeout raises `TimeoutError`.
|
|
135
|
+
Save the returned IDs to resume waiting without creating another operation.
|
|
136
|
+
See the
|
|
137
|
+
[Python Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/evaluation)
|
|
138
|
+
for Method creation and Dataset replacement.
|
|
139
|
+
|
|
140
|
+
## Dataset import and profiling
|
|
141
|
+
|
|
142
|
+
Dataset Import and Dataset Profiling complete independently. With an existing
|
|
143
|
+
`mmlu` Dataset and a project storage PVC named `DATASET_PVC`, import a version and
|
|
144
|
+
wait for its Profile before previewing or creating a Method that uses its fields:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
148
|
+
--repository cais/mmlu --subset abstract_algebra --split test \
|
|
149
|
+
--storage-pvc DATASET_PVC --wait
|
|
150
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The Hugging Face import requires cluster access to the source. Continue only when
|
|
154
|
+
the Profile is `ready`. In Python, use the import result's `dataset_version_id`
|
|
155
|
+
after checking that the import completed and supplied a version:
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from nufi import NuFi
|
|
159
|
+
|
|
160
|
+
with NuFi.from_profile() as client:
|
|
161
|
+
profile = client.datasets.versions.wait_for_profile("DATASET_ID", "VERSION_ID")
|
|
162
|
+
if profile.status != "ready":
|
|
163
|
+
raise RuntimeError(profile.error or "Dataset Profiling failed")
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
See the [Dataset reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/dataset)
|
|
167
|
+
for import, upload, and recovery, and the
|
|
168
|
+
[Evaluation Method guide](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation#nufi-evaluation-method-create)
|
|
169
|
+
for canonical Dataset preparation and custom Dataset bindings.
|
|
170
|
+
|
|
171
|
+
The linked references describe the current development version of NuFi.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# NuFi Python SDK/CLI
|
|
2
|
+
|
|
3
|
+
`nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
|
|
4
|
+
3.11 or newer and installs the `nufi` Python package and `nufi` command.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
python -m pip install nufi-python
|
|
10
|
+
nufi --version
|
|
11
|
+
nufi --help
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Configure access
|
|
15
|
+
|
|
16
|
+
You need a reachable NuFi installation, a user account, and access to a Project.
|
|
17
|
+
Ask your NuFi Administrator for the base domain and project name, then replace
|
|
18
|
+
`nufi.example` and `team-a` below:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
nufi profile create production --base-domain nufi.example
|
|
22
|
+
nufi profile use production
|
|
23
|
+
nufi login
|
|
24
|
+
nufi project use team-a
|
|
25
|
+
nufi model list
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
`login` uses the device flow. Add `--no-browser` on a headless host and open the
|
|
29
|
+
printed verification URL elsewhere. A successful `model list`, even when empty,
|
|
30
|
+
confirms access to the selected Project. The SDK also uses this saved Profile.
|
|
31
|
+
See [installation and authentication](https://docs.nufi.me/dev/docs/next/sdk-cli/installation-and-authentication)
|
|
32
|
+
for other connection settings.
|
|
33
|
+
|
|
34
|
+
## CLI quickstart
|
|
35
|
+
|
|
36
|
+
These examples assume a registered `qwen2` Model Version `v1` with an `original`
|
|
37
|
+
artifact, a compatible accelerator, and an existing Evaluation Method named
|
|
38
|
+
`METHOD_NAME`. Replace those names with your resources. The Leaderboard example
|
|
39
|
+
also assumes that the Method reports `exact_match` and that `quality` is a new
|
|
40
|
+
Leaderboard name. For model import and serving from scratch, follow the
|
|
41
|
+
[serving quickstart](https://docs.nufi.me/dev/docs/next/sdk-cli/quickstart).
|
|
42
|
+
|
|
43
|
+
Compile for RNGD when you need a compiled artifact:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
nufi compile create qwen2 --version v1 --wait
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Evaluate the original artifact, or submit it to a Leaderboard that starts its own
|
|
50
|
+
Evaluation Run:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
nufi evaluation run create METHOD_NAME --model qwen2 \
|
|
54
|
+
--model-version v1 --artifact original --wait
|
|
55
|
+
nufi leaderboard create quality --method METHOD_NAME \
|
|
56
|
+
--rank-metric exact_match --rank-direction desc
|
|
57
|
+
nufi leaderboard submission create quality --model qwen2 \
|
|
58
|
+
--model-version v1 --artifact original --wait
|
|
59
|
+
nufi leaderboard entries quality
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Compile succeeds at `Registered`; an Evaluation Run or a Submission's latest Run
|
|
63
|
+
succeeds at `succeeded`. The wait commands return a non-zero exit code for failure,
|
|
64
|
+
cancellation where supported, or timeout. A timeout stops local waiting and leaves
|
|
65
|
+
the server operation running. Resume with the printed name or ID:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
nufi compile wait COMPILE_NAME_OR_ID
|
|
69
|
+
nufi evaluation run wait RUN_ID
|
|
70
|
+
nufi leaderboard submission wait quality SUBMISSION_ID
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Use `--help` on a command for its options and `--output json` for structured
|
|
74
|
+
results. Evaluation detail JSON can include sensitive diagnostic inputs and
|
|
75
|
+
outputs; table output shows summaries. See the
|
|
76
|
+
[Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation)
|
|
77
|
+
for Method creation, Provider selection, and result inspection.
|
|
78
|
+
|
|
79
|
+
## Python SDK quickstart
|
|
80
|
+
|
|
81
|
+
After configuring the Profile above, use a context manager to close the client.
|
|
82
|
+
Replace `METHOD_ID` with an Evaluation Method UUID, `9` with your Model Artifact's
|
|
83
|
+
numeric ID, and `LEADERBOARD_ID` with the UUID of an existing Leaderboard such as
|
|
84
|
+
`quality` above. CLI `get` commands with `--output json` expose these IDs.
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from nufi import NuFi
|
|
88
|
+
|
|
89
|
+
with NuFi.from_profile() as client:
|
|
90
|
+
compile_run = client.compiles.create(model_name="qwen2", version="v1")
|
|
91
|
+
compiled = client.compiles.wait(compile_run.id)
|
|
92
|
+
if compiled.phase != "Registered" or compiled.registration_error:
|
|
93
|
+
raise RuntimeError(compiled.error_message or compiled.registration_error or "Compile failed")
|
|
94
|
+
|
|
95
|
+
run = client.evaluations.create(
|
|
96
|
+
evaluation_method_id="METHOD_ID",
|
|
97
|
+
model_artifact_id=9,
|
|
98
|
+
cpu="4",
|
|
99
|
+
memory="8Gi",
|
|
100
|
+
accelerator_count=1,
|
|
101
|
+
)
|
|
102
|
+
evaluated = client.evaluations.wait(run.id)
|
|
103
|
+
if evaluated.status != "succeeded":
|
|
104
|
+
raise RuntimeError(f"Evaluation {run.id} ended in {evaluated.status}")
|
|
105
|
+
print(evaluated.aggregate_metrics)
|
|
106
|
+
|
|
107
|
+
submission = client.leaderboards.submit("LEADERBOARD_ID", model_artifact_id=9)
|
|
108
|
+
completed = client.leaderboards.wait("LEADERBOARD_ID", submission.id)
|
|
109
|
+
print(completed.latest_run.status)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
SDK `wait()` methods return terminal objects, including failed or cancelled Runs;
|
|
113
|
+
check their status before using results. Request errors raise
|
|
114
|
+
`nufi.exceptions.NuFiError` subclasses, and a wait timeout raises `TimeoutError`.
|
|
115
|
+
Save the returned IDs to resume waiting without creating another operation.
|
|
116
|
+
See the
|
|
117
|
+
[Python Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/evaluation)
|
|
118
|
+
for Method creation and Dataset replacement.
|
|
119
|
+
|
|
120
|
+
## Dataset import and profiling
|
|
121
|
+
|
|
122
|
+
Dataset Import and Dataset Profiling complete independently. With an existing
|
|
123
|
+
`mmlu` Dataset and a project storage PVC named `DATASET_PVC`, import a version and
|
|
124
|
+
wait for its Profile before previewing or creating a Method that uses its fields:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
nufi dataset version create mmlu mmlu-aa-test --source huggingface \
|
|
128
|
+
--repository cais/mmlu --subset abstract_algebra --split test \
|
|
129
|
+
--storage-pvc DATASET_PVC --wait
|
|
130
|
+
nufi dataset version profile mmlu mmlu-aa-test --wait --output json
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The Hugging Face import requires cluster access to the source. Continue only when
|
|
134
|
+
the Profile is `ready`. In Python, use the import result's `dataset_version_id`
|
|
135
|
+
after checking that the import completed and supplied a version:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from nufi import NuFi
|
|
139
|
+
|
|
140
|
+
with NuFi.from_profile() as client:
|
|
141
|
+
profile = client.datasets.versions.wait_for_profile("DATASET_ID", "VERSION_ID")
|
|
142
|
+
if profile.status != "ready":
|
|
143
|
+
raise RuntimeError(profile.error or "Dataset Profiling failed")
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
See the [Dataset reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/dataset)
|
|
147
|
+
for import, upload, and recovery, and the
|
|
148
|
+
[Evaluation Method guide](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation#nufi-evaluation-method-create)
|
|
149
|
+
for canonical Dataset preparation and custom Dataset bindings.
|
|
150
|
+
|
|
151
|
+
The linked references describe the current development version of NuFi.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nufi-python"
|
|
3
|
-
version = "1.1.
|
|
3
|
+
version = "1.1.4"
|
|
4
4
|
authors = [
|
|
5
5
|
{ name = "Dudaji Inc." },
|
|
6
6
|
]
|
|
@@ -22,9 +22,6 @@ dependencies = [
|
|
|
22
22
|
"rich>=13.0.0",
|
|
23
23
|
]
|
|
24
24
|
|
|
25
|
-
[project.optional-dependencies]
|
|
26
|
-
evaluator = ["pyarrow>=24.0.0"]
|
|
27
|
-
|
|
28
25
|
[build-system]
|
|
29
26
|
requires = ["setuptools>=61.0"]
|
|
30
27
|
build-backend = "setuptools.build_meta"
|
|
@@ -6,8 +6,18 @@ from nufi.models import (
|
|
|
6
6
|
CompileRun,
|
|
7
7
|
CompileRunSummary,
|
|
8
8
|
DatasetVersionProfile,
|
|
9
|
+
EvaluationProvider,
|
|
10
|
+
EvaluationProviderCatalog,
|
|
9
11
|
EvaluationMethod,
|
|
10
|
-
|
|
12
|
+
EvaluationMethodPreview,
|
|
13
|
+
EvaluationMethodRequest,
|
|
14
|
+
EvaluationMethodCreateInput,
|
|
15
|
+
EvaluationProviderOutputMetric,
|
|
16
|
+
EvaluationDatasetPreparation,
|
|
17
|
+
EvaluationMethodSummary,
|
|
18
|
+
EvaluationProviderRelease,
|
|
19
|
+
EvaluationProviderReleaseManifest,
|
|
20
|
+
EvaluationProviderReleaseSummary,
|
|
11
21
|
EvaluationRun,
|
|
12
22
|
EvaluationRunSummary,
|
|
13
23
|
HuggingFaceMapping,
|
|
@@ -36,8 +46,18 @@ __all__ = [
|
|
|
36
46
|
"CompileRun",
|
|
37
47
|
"CompileRunSummary",
|
|
38
48
|
"DatasetVersionProfile",
|
|
49
|
+
"EvaluationProvider",
|
|
50
|
+
"EvaluationProviderCatalog",
|
|
39
51
|
"EvaluationMethod",
|
|
40
|
-
"
|
|
52
|
+
"EvaluationMethodPreview",
|
|
53
|
+
"EvaluationMethodRequest",
|
|
54
|
+
"EvaluationMethodCreateInput",
|
|
55
|
+
"EvaluationProviderOutputMetric",
|
|
56
|
+
"EvaluationDatasetPreparation",
|
|
57
|
+
"EvaluationMethodSummary",
|
|
58
|
+
"EvaluationProviderRelease",
|
|
59
|
+
"EvaluationProviderReleaseManifest",
|
|
60
|
+
"EvaluationProviderReleaseSummary",
|
|
41
61
|
"EvaluationRun",
|
|
42
62
|
"EvaluationRunSummary",
|
|
43
63
|
"HuggingFaceMapping",
|
|
@@ -6,10 +6,11 @@ import os
|
|
|
6
6
|
from collections.abc import Callable
|
|
7
7
|
from enum import Enum
|
|
8
8
|
from typing import Any
|
|
9
|
+
from uuid import UUID
|
|
9
10
|
|
|
10
11
|
import typer
|
|
11
12
|
from rich.console import Console
|
|
12
|
-
from rich.table import Table
|
|
13
|
+
from rich.table import Column, Table
|
|
13
14
|
from rich.text import Text
|
|
14
15
|
|
|
15
16
|
from nufi import NuFi
|
|
@@ -123,12 +124,52 @@ def print_table(
|
|
|
123
124
|
width: int | None = None,
|
|
124
125
|
show_lines: bool = False,
|
|
125
126
|
) -> None:
|
|
126
|
-
|
|
127
|
+
column_values = [
|
|
128
|
+
[row[index] for row in rows if str(row[index])] for index in range(len(columns))
|
|
129
|
+
]
|
|
130
|
+
uuid_columns = {
|
|
131
|
+
index
|
|
132
|
+
for index, values in enumerate(column_values)
|
|
133
|
+
if any(_is_uuid(value) for value in values)
|
|
134
|
+
}
|
|
135
|
+
uuid_only_columns = {
|
|
136
|
+
index
|
|
137
|
+
for index in uuid_columns
|
|
138
|
+
if all(_is_uuid(value) for value in column_values[index])
|
|
139
|
+
}
|
|
140
|
+
if uuid_columns:
|
|
141
|
+
minimum_width = (
|
|
142
|
+
sum(
|
|
143
|
+
36 if index in uuid_columns else len(column)
|
|
144
|
+
for index, column in enumerate(columns)
|
|
145
|
+
)
|
|
146
|
+
+ 3 * len(columns)
|
|
147
|
+
+ 1
|
|
148
|
+
)
|
|
149
|
+
width = max(width or console.width, minimum_width)
|
|
150
|
+
table = Table(
|
|
151
|
+
*(
|
|
152
|
+
Column(column, min_width=36, no_wrap=index in uuid_only_columns)
|
|
153
|
+
if index in uuid_columns
|
|
154
|
+
else Column(column)
|
|
155
|
+
for index, column in enumerate(columns)
|
|
156
|
+
),
|
|
157
|
+
width=width,
|
|
158
|
+
show_lines=show_lines,
|
|
159
|
+
)
|
|
127
160
|
for row in rows:
|
|
128
161
|
table.add_row(*(Text(str(value)) for value in row))
|
|
129
162
|
console.print(table, width=width, crop=False)
|
|
130
163
|
|
|
131
164
|
|
|
165
|
+
def _is_uuid(value: object) -> bool:
|
|
166
|
+
text = str(value)
|
|
167
|
+
try:
|
|
168
|
+
return str(UUID(text)) == text.lower()
|
|
169
|
+
except ValueError:
|
|
170
|
+
return False
|
|
171
|
+
|
|
172
|
+
|
|
132
173
|
def print_page_table(
|
|
133
174
|
page: Page[Any],
|
|
134
175
|
columns: list[str],
|
|
@@ -202,4 +243,5 @@ def handle_cli_error(exc: Exception, *, verbose: bool) -> None:
|
|
|
202
243
|
else:
|
|
203
244
|
detail = str(exc)
|
|
204
245
|
error_console.print(f"Error: {detail}", style="bold red", markup=False)
|
|
205
|
-
|
|
246
|
+
timed_out = isinstance(exc, TimeoutError) or isinstance(exc.__cause__, TimeoutError)
|
|
247
|
+
raise typer.Exit(code=124 if timed_out else 1) from exc
|
|
@@ -43,26 +43,24 @@ def _is_uuid(value: str) -> bool:
|
|
|
43
43
|
return True
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
def _get_dataset(client, dataset: str, *, project: str | int):
|
|
47
|
+
try:
|
|
48
|
+
return client.datasets.get(dataset, project=project)
|
|
49
|
+
except ValidationError as exc:
|
|
50
|
+
message = str(exc)
|
|
51
|
+
if message.startswith("Multiple datasets named "):
|
|
52
|
+
raise ValidationError(message.replace("datasets", "Datasets", 1)) from None
|
|
53
|
+
raise
|
|
54
|
+
|
|
55
|
+
|
|
46
56
|
def _resolve_dataset_id(client, dataset: str, *, project: str | int) -> str:
|
|
47
|
-
|
|
48
|
-
return dataset
|
|
49
|
-
return common.resolve_named_resource(
|
|
50
|
-
client.datasets.iter_all(project=project),
|
|
51
|
-
dataset,
|
|
52
|
-
resource="Dataset",
|
|
53
|
-
).id
|
|
57
|
+
return _get_dataset(client, dataset, project=project).id
|
|
54
58
|
|
|
55
59
|
|
|
56
60
|
def _resolve_version_id(
|
|
57
61
|
client, dataset_id: str, version: str, *, project: str | int
|
|
58
62
|
) -> str:
|
|
59
|
-
|
|
60
|
-
return version
|
|
61
|
-
return common.resolve_named_resource(
|
|
62
|
-
client.datasets.versions.iter_all(dataset_id, project=project),
|
|
63
|
-
version,
|
|
64
|
-
resource="Dataset Version",
|
|
65
|
-
).id
|
|
63
|
+
return client.datasets.versions.get(dataset_id, version, project=project).id
|
|
66
64
|
|
|
67
65
|
|
|
68
66
|
def _project_ref(
|
|
@@ -251,8 +249,7 @@ def get_dataset(
|
|
|
251
249
|
try:
|
|
252
250
|
resolved_project = _project_ref(project, project_id, profile_name)
|
|
253
251
|
client = common.make_client(profile_name)
|
|
254
|
-
|
|
255
|
-
result = client.datasets.get(dataset_id, project=resolved_project)
|
|
252
|
+
result = _get_dataset(client, dataset, project=resolved_project)
|
|
256
253
|
except Exception as exc:
|
|
257
254
|
common.handle_cli_error(exc, verbose=verbose)
|
|
258
255
|
finally:
|
|
@@ -288,6 +285,11 @@ def create_dataset(
|
|
|
288
285
|
profile_name: str | None = typer.Option(
|
|
289
286
|
None, "--profile", help="CLI Profile name."
|
|
290
287
|
),
|
|
288
|
+
output: common.OutputChoice = typer.Option(
|
|
289
|
+
common.OutputChoice.table,
|
|
290
|
+
"--output",
|
|
291
|
+
help="Output format.",
|
|
292
|
+
),
|
|
291
293
|
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
292
294
|
):
|
|
293
295
|
client = None
|
|
@@ -305,6 +307,10 @@ def create_dataset(
|
|
|
305
307
|
if client is not None:
|
|
306
308
|
client.close()
|
|
307
309
|
|
|
310
|
+
if output == common.OutputChoice.json:
|
|
311
|
+
common.print_json(dataset)
|
|
312
|
+
return
|
|
313
|
+
|
|
308
314
|
common.console.print(f"[bold green]Dataset created:[/] {escape(dataset.name)}")
|
|
309
315
|
|
|
310
316
|
|
|
@@ -494,11 +500,8 @@ def get_version(
|
|
|
494
500
|
resolved_project = _project_ref(project, project_id, profile_name)
|
|
495
501
|
client = common.make_client(profile_name)
|
|
496
502
|
dataset_id = _resolve_dataset_id(client, dataset, project=resolved_project)
|
|
497
|
-
version_id = _resolve_version_id(
|
|
498
|
-
client, dataset_id, version, project=resolved_project
|
|
499
|
-
)
|
|
500
503
|
result = client.datasets.versions.get(
|
|
501
|
-
dataset_id,
|
|
504
|
+
dataset_id, version, project=resolved_project
|
|
502
505
|
)
|
|
503
506
|
except Exception as exc:
|
|
504
507
|
common.handle_cli_error(exc, verbose=verbose)
|
|
@@ -625,6 +628,11 @@ def create_version(
|
|
|
625
628
|
profile_name: str | None = typer.Option(
|
|
626
629
|
None, "--profile", help="CLI Profile name."
|
|
627
630
|
),
|
|
631
|
+
output: common.OutputChoice = typer.Option(
|
|
632
|
+
common.OutputChoice.table,
|
|
633
|
+
"--output",
|
|
634
|
+
help="Output format.",
|
|
635
|
+
),
|
|
628
636
|
verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
|
|
629
637
|
):
|
|
630
638
|
client = None
|
|
@@ -702,6 +710,10 @@ def create_version(
|
|
|
702
710
|
if client is not None:
|
|
703
711
|
client.close()
|
|
704
712
|
|
|
713
|
+
if output == common.OutputChoice.json:
|
|
714
|
+
common.print_json(result)
|
|
715
|
+
return
|
|
716
|
+
|
|
705
717
|
if source == "pvc":
|
|
706
718
|
common.console.print(f"[bold green]Dataset Version created:[/] {escape(name)}")
|
|
707
719
|
return
|