nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,228 @@
1
+ Metadata-Version: 2.4
2
+ Name: nufi-python
3
+ Version: 1.0.0
4
+ Summary: Python SDK and CLI for the NuFi platform.
5
+ Author: Dudaji Inc.
6
+ License-Expression: MIT
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Operating System :: OS Independent
9
+ Classifier: Typing :: Typed
10
+ Requires-Python: >=3.11
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: httpx>=0.25.0
14
+ Requires-Dist: PyYAML>=6.0
15
+ Requires-Dist: pydantic>=2.0.0
16
+ Requires-Dist: tuspy==1.1.0
17
+ Requires-Dist: typer>=0.16.0
18
+ Requires-Dist: rich>=13.0.0
19
+ Provides-Extra: evaluator
20
+ Requires-Dist: pyarrow>=24.0.0; extra == "evaluator"
21
+ Dynamic: license-file
22
+
23
+ # NuFi Python SDK/CLI
24
+
25
+ The distribution is `nufi-python`; it installs the `nufi` Python package and `nufi`
26
+ command.
27
+
28
+ ```bash
29
+ uv build --clear
30
+ python -m pip install ./dist/nufi_python-1.0.0-py3-none-any.whl
31
+ nufi --version
32
+ ```
33
+
34
+ ```python
35
+ from nufi import NuFi
36
+ ```
37
+
38
+ For verified-wheel installation, model registration, Dataset selection, evaluation,
39
+ Leaderboard ranking, compatibility, and troubleshooting, follow the
40
+ [SDK and CLI workflow guide](../docs/sdk-cli-guide/README.md).
41
+
42
+ ## Installed quickstart
43
+
44
+ Create a profile, log in, and select the project once:
45
+
46
+ ```bash
47
+ nufi profile create dev --base-domain nufi.local
48
+ nufi profile use dev
49
+ nufi login
50
+ nufi project use team-a
51
+ ```
52
+
53
+ The CLI workflow is composable; every long-running identity can be resumed with its
54
+ dedicated `wait` command:
55
+
56
+ ```bash
57
+ nufi model version create qwen2 v1 --volume models --path /qwen2
58
+ nufi compile create qwen2 --version v1 --wait
59
+ nufi compile wait COMPILE_NAME_OR_ID
60
+ nufi dataset list
61
+ nufi evaluation run create METHOD_NAME --model qwen2 \
62
+ --model-version v1 --artifact original --wait
63
+ nufi evaluation run wait RUN_ID
64
+ nufi leaderboard create quality --method METHOD_NAME \
65
+ --rank-metric exact_match --rank-direction desc
66
+ nufi leaderboard submission create quality --model qwen2 \
67
+ --model-version v1 --artifact original --wait
68
+ nufi leaderboard submission wait quality SUBMISSION_ID
69
+ nufi leaderboard entries quality
70
+ ```
71
+
72
+ The same flow is available without local orchestration in Python:
73
+
74
+ ```python
75
+ from nufi import NuFi
76
+
77
+ with NuFi.from_profile() as client:
78
+ compile_run = client.compiles.create(model_name="qwen2", version="v1")
79
+ compiled = client.compiles.wait(compile_run.id)
80
+
81
+ evaluation_run = client.evaluations.create(
82
+ evaluation_method_id="METHOD_ID",
83
+ model_artifact_id=9,
84
+ cpu="4",
85
+ memory="8Gi",
86
+ accelerator_count=1,
87
+ )
88
+ evaluated = client.evaluations.wait(evaluation_run.id)
89
+
90
+ leaderboard = client.leaderboards.create(
91
+ "quality",
92
+ evaluation_method_id="METHOD_ID",
93
+ ranking_policy={
94
+ "primary": {
95
+ "metric": {"source": "aggregateMetrics", "name": "exact_match"},
96
+ "direction": "desc",
97
+ }
98
+ },
99
+ )
100
+ submission = client.leaderboards.submit(
101
+ leaderboard.id,
102
+ model_artifact_id=9,
103
+ )
104
+ completed = client.leaderboards.wait(leaderboard.id, submission.id)
105
+ entries = client.leaderboards.entries(leaderboard.id)
106
+ ```
107
+
108
+ ## Local Run
109
+
110
+ Install dependencies:
111
+
112
+ ```bash
113
+ uv sync
114
+ ```
115
+
116
+ Run the CLI locally:
117
+
118
+ ```bash
119
+ uv run nufi --help
120
+ uv run nufi profile
121
+ ```
122
+
123
+ ## Evaluations
124
+
125
+ Evaluation Method metadata is unconditional and v2-only. Method config is owned and
126
+ validated by the API; the SDK forwards the mapping and any caller-supplied Method type
127
+ without a client-side allowlist.
128
+
129
+ `evaluation method templates` currently exposes lm-eval templates, metrics, and
130
+ filters. It is an optional discovery command and is not needed when you already have a
131
+ Method config.
132
+
133
+ ```python
134
+ from nufi import NuFi
135
+
136
+ client = NuFi.from_profile()
137
+ project = "team-a"
138
+ config = {
139
+ "schemaVersion": "nufi.evaluation.method.v2",
140
+ "runnerType": "lm_eval",
141
+ "template": "short_answer",
142
+ "datasetMapping": {"input": "question", "reference": "answer"},
143
+ "nativeConfig": {
144
+ "output_type": "generate_until",
145
+ "doc_to_text": "{{input}}",
146
+ "doc_to_target": "{{reference}}",
147
+ "metric_list": [{"metric": "exact_match"}],
148
+ },
149
+ }
150
+
151
+ templates = client.evaluations.methods.templates(project=project)
152
+ preview = client.evaluations.methods.preview(
153
+ "DATASET_ID", "VERSION_ID", project=project, config=config
154
+ )
155
+ method = client.evaluations.methods.create(
156
+ "DATASET_ID", "VERSION_ID", "gsm8k-exact-match", project=project, config=config
157
+ )
158
+ run = client.evaluations.create(
159
+ project=project,
160
+ evaluation_method_id=method.id,
161
+ model_artifact_id=9,
162
+ cpu="1",
163
+ memory="2Gi",
164
+ accelerator_count=1,
165
+ )
166
+ result = client.evaluations.get(run.id, project=project)
167
+ client.evaluations.cancel(run.id, project=project)
168
+ client.close()
169
+ ```
170
+
171
+ The CLI accepts Dataset, Dataset Version, and Evaluation Method names or UUIDs. Run
172
+ arguments remain UUIDs because Runs have no name. Select a Model Artifact with
173
+ `--model`, `--model-version`, and `--artifact`; `--model-artifact-id` remains available
174
+ for automation. Config, default-runtime, and runner-config files may be JSON or YAML
175
+ and must have a mapping at the document root.
176
+
177
+ ```yaml
178
+ # method.yaml
179
+ schemaVersion: nufi.evaluation.method.v2
180
+ runnerType: lm_eval
181
+ template: short_answer
182
+ datasetMapping:
183
+ input: question
184
+ reference: answer
185
+ nativeConfig:
186
+ output_type: generate_until
187
+ doc_to_text: "{{input}}"
188
+ doc_to_target: "{{reference}}"
189
+ metric_list:
190
+ - metric: exact_match
191
+ ```
192
+
193
+ ```bash
194
+ uv run nufi evaluation method templates --project team-a
195
+ uv run nufi evaluation method validate gsm8k v1 --config method.yaml --project team-a
196
+ uv run nufi evaluation method create gsm8k v1 gsm8k-exact-match \
197
+ --config method.yaml --project team-a
198
+ uv run nufi evaluation method list --dataset gsm8k --version v1 --project team-a
199
+ uv run nufi evaluation method archive gsm8k-exact-match --yes --project team-a
200
+ uv run nufi evaluation method restore gsm8k-exact-match --project team-a
201
+
202
+ uv run nufi evaluation run create gsm8k-exact-match \
203
+ --model qwen2 --model-version v1 --artifact original \
204
+ --project team-a
205
+ uv run nufi evaluation run list --dataset gsm8k --version v1 --project team-a
206
+ uv run nufi evaluation run get RUN_ID --project team-a
207
+ uv run nufi evaluation run cancel RUN_ID --yes --project team-a
208
+ ```
209
+
210
+ Run creation defaults to `--cpu 4`, `--memory 8Gi`, and `--accelerator-count 1`;
211
+ pass any option to override its default.
212
+
213
+ Run detail table output shows aggregate and diagnostic summaries only. Use explicit
214
+ `--output json` to include the authorized bounded diagnostic input, output, reference,
215
+ score, and error fields returned by the API. Additive response fields are preserved;
216
+ legacy Method/Run responses missing required v2 fields are rejected.
217
+
218
+ Run tests:
219
+
220
+ ```bash
221
+ uv run pytest
222
+ ```
223
+
224
+ Build the package:
225
+
226
+ ```bash
227
+ uv build --clear
228
+ ```
@@ -0,0 +1,52 @@
1
+ nufi/__init__.py,sha256=a6Zu7yJOlVKA-U-zFgZxlaQmWDLEtsndvZsecKpA__A,1055
2
+ nufi/auth.py,sha256=KjZG-N0i6-JzccsIlP9VmZFZIk9XHxDxWFUlrmxl_1M,12664
3
+ nufi/client.py,sha256=9l_1pvwCaNx8VynLNlqh0_regb9aJn5GyA-k0reeq_s,3355
4
+ nufi/config.py,sha256=EJ4IMZ79wpvV_h848nVBUfLHS9lMW1DH7c4kgZgWCOk,8583
5
+ nufi/evaluator.py,sha256=782GEqNRmtoJZkb5_AT6YogiiHdMsbPmnEb382UIATY,20429
6
+ nufi/exceptions.py,sha256=lFKUzpjE6CI9KQKMgK10tRmMeYrTJ_iAOZet472XGgE,1582
7
+ nufi/http_utils.py,sha256=TGzta0lp69LNbtDGQdqRCU3x0pLZAWgIJ12dig9c5Ug,1365
8
+ nufi/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
9
+ nufi/transport.py,sha256=2SHZxBaOBVV_6mYKVebZv9COvWO9sDugZNSaS6k_lzk,9177
10
+ nufi/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
+ nufi/cli/main.py,sha256=bgrfIAlkSUQbP-J7qA3nyIQZ-VRjOQ6032pkeQMIHN8,2092
12
+ nufi/cli/commands/__init__.py,sha256=c7ZlZdHhiwfEa80F8ZBkTq6D349BkHF8_pWzYUdLCLc,46
13
+ nufi/cli/commands/auth.py,sha256=umtb4Ur-TQI4y4UoTpDrSET1Qau5diw9GQIuFsMZSZg,3999
14
+ nufi/cli/commands/common.py,sha256=YHnb-euruDm9ivAzYAHO6zxmLod_kCqpO_jtcQVzB1c,5789
15
+ nufi/cli/commands/compiles.py,sha256=oF1ag6L_MFNUY3-xZ5-YyTWPkapEGgJjvfn6DvL86Mg,14429
16
+ nufi/cli/commands/custom_evaluation_templates.py,sha256=2hQ9htTL9WMPXLePRmi0iQTY2OyCduF9xDJRh037R8Q,6547
17
+ nufi/cli/commands/datasets.py,sha256=ni6xbl_eIy2dDWGM0iaVz7jX-ufQeNsiTayt9OjmEWw,22374
18
+ nufi/cli/commands/evaluations.py,sha256=dZLZaEA2IWLmjAcAM45THQQRybLI_RmCTIz3jNK4OiI,28506
19
+ nufi/cli/commands/leaderboards.py,sha256=4y3Yb_cFa9L6A8A6y1OfX_e3KuMlcZzJ5W9V6sjK9zM,25401
20
+ nufi/cli/commands/models.py,sha256=sCglKwJ-AHIs1ai_8YLxaETbx54DjBBfnNpxFDhh868,42644
21
+ nufi/cli/commands/monitoring.py,sha256=qWjdTZBMLVOoZ5j32FOdiSsUviw0zqBherMCESfd4Lo,10748
22
+ nufi/cli/commands/profile.py,sha256=HeYjA_672RKfjrNlRF9r7GbSlgq1Tb5CUi9gIttoxUs,3869
23
+ nufi/cli/commands/projects.py,sha256=pJ9zL6ytBrP-dzTceEW24un12gaeCoUtjUtKCayQoKg,9218
24
+ nufi/cli/commands/servings.py,sha256=oLbynvTsYpahiMls8G6LOEAI-PcHKFtTRxG-3H05wJg,8740
25
+ nufi/models/__init__.py,sha256=ziZTe-SRTK1hirsYD0X8xXIULy8DsnY_KFpZxieQUqQ,4987
26
+ nufi/models/compiles.py,sha256=m2ncN3ab4JWuEUstTnlC6PLdnTqM4zcq3obnh--M3ZQ,3094
27
+ nufi/models/custom_evaluation_templates.py,sha256=BvfpRT6dkdROZuxM7ZNztm0XT2UuJAyiDdkKAmjPbjE,3517
28
+ nufi/models/datasets.py,sha256=w7G1KcFS-OoYq8GLfv429Y2duk7MwWEXMUjWKa9TYKs,3325
29
+ nufi/models/evaluations.py,sha256=pEOhJHZk2S_-0-etSVDYpxeN8SGhxVMDBPcFbCh2wEk,9604
30
+ nufi/models/leaderboards.py,sha256=EnutNfXiRweyxZsA79vcpqRPjDqi1jz9yR9vdg_vA3s,5538
31
+ nufi/models/monitoring.py,sha256=QR-MfoIgniUq7JyA2NU8tihKq6o-nrcbQfha5696dB4,6235
32
+ nufi/models/pagination.py,sha256=zOqwd_F2W1hkUnIzQHRfM7jFnSgOZd-rkf0j_CWD3DQ,970
33
+ nufi/models/projects.py,sha256=9hvsvrdYae6fWbVlvFDcnlwCq5r6cFHIf2ptsWFSMFQ,910
34
+ nufi/models/registry.py,sha256=SMCG6cuukjXNdanBrOnXjJ43rgcCwayQ7Ec-Ls2tmzM,5025
35
+ nufi/models/servings.py,sha256=EhgjuJx5oh3W3h_SegzYjTVe4xMzzq9dKhLLLIsw3_8,1034
36
+ nufi/resources/__init__.py,sha256=GNSPvX78s2MGRrFPDFU-qp3xv0Gc6XTZd-nptitMEg8,1141
37
+ nufi/resources/base.py,sha256=YbR9Nxs8MO3sOVmKdxEV7sHsJY5g3fu4yvZAkvoUrRU,6473
38
+ nufi/resources/compiles.py,sha256=IZ6AG25Pl5T6ZohngWclQvLrLFdwYs7aZUPwkHq7WvU,6750
39
+ nufi/resources/custom_evaluation_templates.py,sha256=xlJ_RTN_3FFH7sK559pg_89zvH_TcwYLBajANCzDdEc,4859
40
+ nufi/resources/datasets.py,sha256=R0OFoG4szFpau_PZYdGG7WaN_zcvjvOJNgQ-Ui5If8g,26848
41
+ nufi/resources/evaluations.py,sha256=a2NE3j27UvoUIrVVaPIz1BpgFR4HmX__gJSwFu6C-Ok,15544
42
+ nufi/resources/leaderboards.py,sha256=bdVsYp2GRzg4invDc9Vs74JMNTiPLR2GYlQXIRx82mI,14096
43
+ nufi/resources/models.py,sha256=V4rUmk1i-3QVlDCNH5dEXiR7-spX-q3BPeKBaegY_N8,23922
44
+ nufi/resources/monitoring.py,sha256=So6RiHMl0gIYvUfwsaF32NdhTncIS_Kik125NGSurjU,2817
45
+ nufi/resources/projects.py,sha256=Z8Sggxew6LXdiy-od8k5B_A1HiQ_ZYb2JEm8JyblVgo,4086
46
+ nufi/resources/servings.py,sha256=MPZQrUvekac-H_SIF_xAzqnNoGj9BCd4ACaJhWfpb3A,2790
47
+ nufi_python-1.0.0.dist-info/licenses/LICENSE,sha256=8sYgF7-GzNegaXcC5JvrIsjz0qzmLAj80uu8_ZYNGDQ,1068
48
+ nufi_python-1.0.0.dist-info/METADATA,sha256=NAd3DC8oXvDFlnA7v6YP0In9ZLT4IpXxfX0GE8wh51E,6769
49
+ nufi_python-1.0.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
50
+ nufi_python-1.0.0.dist-info/entry_points.txt,sha256=LUxXF-4cGgte3B0Uvn19qqL5z_G_ncB9vowzMnU4K00,43
51
+ nufi_python-1.0.0.dist-info/top_level.txt,sha256=8FzgB8tx8hf3uq9bjUHwMV_-bnbRkgoSmyQ7jl7R1MI,5
52
+ nufi_python-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ nufi = nufi.cli.main:app
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Dudaji Inc.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ nufi