nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,428 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Iterator
4
+ from typing import Any
5
+
6
+ from nufi.models import (
7
+ CustomEvaluationMethodConfig,
8
+ EvaluationMethod,
9
+ EvaluationMethodListResponse,
10
+ EvaluationMethodPreview,
11
+ EvaluationMethodTemplates,
12
+ EvaluationRun,
13
+ EvaluationRunListResponse,
14
+ Page,
15
+ )
16
+ from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
17
+ from nufi.resources.custom_evaluation_templates import (
18
+ CustomEvaluationTemplatesResource,
19
+ )
20
+
21
+ _TERMINAL_RUN_STATUSES = frozenset({"succeeded", "failed", "cancelled"})
22
+
23
+
24
+ def _without_none(payload: dict[str, Any]) -> dict[str, Any]:
25
+ return {key: value for key, value in payload.items() if value is not None}
26
+
27
+
28
+ def _config_payload(
29
+ config: dict[str, Any] | CustomEvaluationMethodConfig,
30
+ ) -> dict[str, Any]:
31
+ if isinstance(config, CustomEvaluationMethodConfig):
32
+ return config.model_dump(mode="json", by_alias=True)
33
+ return config
34
+
35
+
36
+ def _method_selector(
37
+ *, template_id: str | None, method_type: str | None
38
+ ) -> dict[str, str]:
39
+ if template_id is not None and method_type is not None:
40
+ raise ValueError("template_id cannot be combined with method_type")
41
+ if template_id is not None:
42
+ return {"templateId": template_id}
43
+ return {"type": method_type or "lm_eval"}
44
+
45
+
46
+ class EvaluationMethodsResource(BaseResource):
47
+ def templates(
48
+ self, *, project: ProjectRef | None = None
49
+ ) -> EvaluationMethodTemplates:
50
+ project_id = self._resolve_project_id(project)
51
+ response = self._transport.get(
52
+ f"/api/v1/projects/{project_id}/evaluations/tasks/templates"
53
+ )
54
+ return EvaluationMethodTemplates.model_validate(response_json(response))
55
+
56
+ def list(
57
+ self,
58
+ *,
59
+ project: ProjectRef | None = None,
60
+ dataset_id: str | None = None,
61
+ dataset_version_id: str | None = None,
62
+ method_type: str | None = None,
63
+ status: str | None = None,
64
+ archived: bool | None = None,
65
+ page: int | None = None,
66
+ limit: int | None = None,
67
+ ) -> list[EvaluationMethod]:
68
+ if dataset_version_id is not None and dataset_id is None:
69
+ raise ValueError("dataset_version_id requires dataset_id")
70
+ project_id = self._resolve_project_id(project)
71
+ if dataset_version_id is not None:
72
+ path = (
73
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
74
+ f"/versions/{escape_path(dataset_version_id)}/evaluation-tasks"
75
+ )
76
+ else:
77
+ path = f"/api/v1/projects/{project_id}/evaluations/tasks"
78
+ pagination, _, _ = self._pagination_params(page, limit)
79
+ params = {
80
+ **_without_none(
81
+ {
82
+ "datasetId": dataset_id if dataset_version_id is None else None,
83
+ "type": method_type,
84
+ "status": status,
85
+ "archived": archived,
86
+ }
87
+ ),
88
+ **pagination,
89
+ }
90
+ response = self._transport.get(path, params=params)
91
+ return EvaluationMethodListResponse.model_validate(
92
+ response_json(response)
93
+ ).items
94
+
95
+ def list_page(
96
+ self,
97
+ *,
98
+ project: ProjectRef | None = None,
99
+ dataset_id: str | None = None,
100
+ dataset_version_id: str | None = None,
101
+ method_type: str | None = None,
102
+ status: str | None = None,
103
+ archived: bool | None = None,
104
+ page: int | None = None,
105
+ limit: int | None = None,
106
+ ) -> Page[EvaluationMethod]:
107
+ """Return one Evaluation Method page with pagination metadata."""
108
+ if dataset_version_id is not None and dataset_id is None:
109
+ raise ValueError("dataset_version_id requires dataset_id")
110
+ project_id = self._resolve_project_id(project)
111
+ if dataset_version_id is not None:
112
+ path = (
113
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
114
+ f"/versions/{escape_path(dataset_version_id)}/evaluation-tasks"
115
+ )
116
+ else:
117
+ path = f"/api/v1/projects/{project_id}/evaluations/tasks"
118
+ pagination, resolved_page, resolved_limit = self._pagination_params(
119
+ page, limit, default_page=1, default_limit=20
120
+ )
121
+ response = self._transport.get(
122
+ path,
123
+ params={
124
+ **_without_none(
125
+ {
126
+ "datasetId": (
127
+ dataset_id if dataset_version_id is None else None
128
+ ),
129
+ "type": method_type,
130
+ "status": status,
131
+ "archived": archived,
132
+ }
133
+ ),
134
+ **pagination,
135
+ },
136
+ )
137
+ return self._page(
138
+ response_json(response),
139
+ EvaluationMethod,
140
+ page=resolved_page,
141
+ limit=resolved_limit,
142
+ )
143
+
144
+ def iter_all(
145
+ self,
146
+ *,
147
+ project: ProjectRef | None = None,
148
+ dataset_id: str | None = None,
149
+ dataset_version_id: str | None = None,
150
+ method_type: str | None = None,
151
+ status: str | None = None,
152
+ archived: bool | None = None,
153
+ limit: int = 100,
154
+ ) -> Iterator[EvaluationMethod]:
155
+ """Iterate all matching Evaluation Methods in server order."""
156
+ return self._iter_all(
157
+ lambda page, page_limit: self.list_page(
158
+ project=project,
159
+ dataset_id=dataset_id,
160
+ dataset_version_id=dataset_version_id,
161
+ method_type=method_type,
162
+ status=status,
163
+ archived=archived,
164
+ page=page,
165
+ limit=page_limit,
166
+ ),
167
+ limit=limit,
168
+ )
169
+
170
+ def get(
171
+ self, method_id: str, *, project: ProjectRef | None = None
172
+ ) -> EvaluationMethod:
173
+ project_id = self._resolve_project_id(project)
174
+ response = self._transport.get(
175
+ f"/api/v1/projects/{project_id}/evaluations/tasks/{escape_path(method_id)}"
176
+ )
177
+ return EvaluationMethod.model_validate(response_json(response))
178
+
179
+ def preview(
180
+ self,
181
+ dataset_id: str,
182
+ dataset_version_id: str,
183
+ *,
184
+ project: ProjectRef | None = None,
185
+ config: dict[str, Any] | CustomEvaluationMethodConfig,
186
+ template_id: str | None = None,
187
+ method_type: str | None = None,
188
+ ) -> EvaluationMethodPreview:
189
+ project_id = self._resolve_project_id(project)
190
+ response = self._transport.post(
191
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
192
+ f"/versions/{escape_path(dataset_version_id)}/evaluation-tasks/preview",
193
+ json={
194
+ **_method_selector(template_id=template_id, method_type=method_type),
195
+ "config": _config_payload(config),
196
+ },
197
+ )
198
+ return EvaluationMethodPreview.model_validate(response_json(response))
199
+
200
+ def create(
201
+ self,
202
+ dataset_id: str,
203
+ dataset_version_id: str,
204
+ name: str,
205
+ *,
206
+ project: ProjectRef | None = None,
207
+ config: dict[str, Any] | CustomEvaluationMethodConfig,
208
+ template_id: str | None = None,
209
+ method_type: str | None = None,
210
+ default_runtime: dict[str, Any] | None = None,
211
+ description: str | None = None,
212
+ ) -> EvaluationMethod:
213
+ project_id = self._resolve_project_id(project)
214
+ response = self._transport.post(
215
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
216
+ f"/versions/{escape_path(dataset_version_id)}/evaluation-tasks",
217
+ json=_without_none(
218
+ {
219
+ "name": name,
220
+ **_method_selector(
221
+ template_id=template_id, method_type=method_type
222
+ ),
223
+ "config": _config_payload(config),
224
+ "defaultRuntime": default_runtime,
225
+ "description": description,
226
+ }
227
+ ),
228
+ )
229
+ return EvaluationMethod.model_validate(response_json(response))
230
+
231
+ def archive(
232
+ self,
233
+ method_id: str,
234
+ *,
235
+ project: ProjectRef | None = None,
236
+ reason: str | None = None,
237
+ ) -> None:
238
+ project_id = self._resolve_project_id(project)
239
+ self._transport.post(
240
+ f"/api/v1/projects/{project_id}/evaluations/tasks/"
241
+ f"{escape_path(method_id)}/archive",
242
+ json=_without_none({"reason": reason}),
243
+ )
244
+
245
+ def restore(self, method_id: str, *, project: ProjectRef | None = None) -> None:
246
+ project_id = self._resolve_project_id(project)
247
+ self._transport.post(
248
+ f"/api/v1/projects/{project_id}/evaluations/tasks/"
249
+ f"{escape_path(method_id)}/restore"
250
+ )
251
+
252
+
253
+ class EvaluationsResource(BaseResource):
254
+ def __init__(self, *args: Any, **kwargs: Any):
255
+ super().__init__(*args, **kwargs)
256
+ self.templates = CustomEvaluationTemplatesResource(
257
+ self._transport, self._default_project
258
+ )
259
+ self.methods = EvaluationMethodsResource(self._transport, self._default_project)
260
+
261
+ def list(
262
+ self,
263
+ *,
264
+ project: ProjectRef | None = None,
265
+ dataset_id: str | None = None,
266
+ dataset_version_id: str | None = None,
267
+ model_artifact_id: int | None = None,
268
+ model_artifact_ids: list[int] | None = None,
269
+ status: str | None = None,
270
+ page: int | None = None,
271
+ limit: int | None = None,
272
+ ) -> list[EvaluationRun]:
273
+ if dataset_version_id is not None and dataset_id is None:
274
+ raise ValueError("dataset_version_id requires dataset_id")
275
+ project_id = self._resolve_project_id(project)
276
+ pagination, _, _ = self._pagination_params(page, limit)
277
+ response = self._transport.get(
278
+ f"/api/v1/projects/{project_id}/evaluations",
279
+ params={
280
+ **_without_none(
281
+ {
282
+ "datasetId": dataset_id,
283
+ "datasetVersionId": dataset_version_id,
284
+ "modelArtifactId": model_artifact_id,
285
+ "modelArtifactIds": (
286
+ ",".join(str(item) for item in model_artifact_ids)
287
+ if model_artifact_ids
288
+ else None
289
+ ),
290
+ "status": status,
291
+ }
292
+ ),
293
+ **pagination,
294
+ },
295
+ )
296
+ return EvaluationRunListResponse.model_validate(response_json(response)).items
297
+
298
+ def list_page(
299
+ self,
300
+ *,
301
+ project: ProjectRef | None = None,
302
+ dataset_id: str | None = None,
303
+ dataset_version_id: str | None = None,
304
+ model_artifact_id: int | None = None,
305
+ model_artifact_ids: list[int] | None = None,
306
+ status: str | None = None,
307
+ page: int | None = None,
308
+ limit: int | None = None,
309
+ ) -> Page[EvaluationRun]:
310
+ """Return one Evaluation Run page with pagination metadata."""
311
+ if dataset_version_id is not None and dataset_id is None:
312
+ raise ValueError("dataset_version_id requires dataset_id")
313
+ project_id = self._resolve_project_id(project)
314
+ pagination, resolved_page, resolved_limit = self._pagination_params(
315
+ page, limit, default_page=1, default_limit=20
316
+ )
317
+ response = self._transport.get(
318
+ f"/api/v1/projects/{project_id}/evaluations",
319
+ params={
320
+ **_without_none(
321
+ {
322
+ "datasetId": dataset_id,
323
+ "datasetVersionId": dataset_version_id,
324
+ "modelArtifactId": model_artifact_id,
325
+ "modelArtifactIds": (
326
+ ",".join(str(item) for item in model_artifact_ids)
327
+ if model_artifact_ids
328
+ else None
329
+ ),
330
+ "status": status,
331
+ }
332
+ ),
333
+ **pagination,
334
+ },
335
+ )
336
+ return self._page(
337
+ response_json(response),
338
+ EvaluationRun,
339
+ page=resolved_page,
340
+ limit=resolved_limit,
341
+ )
342
+
343
+ def iter_all(
344
+ self,
345
+ *,
346
+ project: ProjectRef | None = None,
347
+ dataset_id: str | None = None,
348
+ dataset_version_id: str | None = None,
349
+ model_artifact_id: int | None = None,
350
+ model_artifact_ids: list[int] | None = None,
351
+ status: str | None = None,
352
+ limit: int = 100,
353
+ ) -> Iterator[EvaluationRun]:
354
+ """Iterate all matching Evaluation Runs in server order."""
355
+ return self._iter_all(
356
+ lambda page, page_limit: self.list_page(
357
+ project=project,
358
+ dataset_id=dataset_id,
359
+ dataset_version_id=dataset_version_id,
360
+ model_artifact_id=model_artifact_id,
361
+ model_artifact_ids=model_artifact_ids,
362
+ status=status,
363
+ page=page,
364
+ limit=page_limit,
365
+ ),
366
+ limit=limit,
367
+ )
368
+
369
+ def create(
370
+ self,
371
+ *,
372
+ project: ProjectRef | None = None,
373
+ evaluation_method_id: str,
374
+ model_artifact_id: int,
375
+ cpu: str,
376
+ memory: str,
377
+ accelerator_count: int,
378
+ runner_config: dict[str, Any] | None = None,
379
+ ) -> EvaluationRun:
380
+ project_id = self._resolve_project_id(project)
381
+ response = self._transport.post(
382
+ f"/api/v1/projects/{project_id}/evaluations",
383
+ json={
384
+ "evaluationTaskId": evaluation_method_id,
385
+ "modelArtifactId": model_artifact_id,
386
+ "runnerConfig": runner_config or {},
387
+ "targetResources": {
388
+ "cpu": cpu,
389
+ "memory": memory,
390
+ "acceleratorCount": accelerator_count,
391
+ },
392
+ },
393
+ )
394
+ return EvaluationRun.model_validate(response_json(response))
395
+
396
+ def get(self, run_id: str, *, project: ProjectRef | None = None) -> EvaluationRun:
397
+ project_id = self._resolve_project_id(project)
398
+ response = self._transport.get(
399
+ f"/api/v1/projects/{project_id}/evaluations/{escape_path(run_id)}"
400
+ )
401
+ return EvaluationRun.model_validate(response_json(response))
402
+
403
+ def wait(
404
+ self,
405
+ run_id: str,
406
+ *,
407
+ project: ProjectRef | None = None,
408
+ timeout: float = 1800,
409
+ interval: float = 5,
410
+ ) -> EvaluationRun:
411
+ """Wait for an Evaluation Run using only its GET route."""
412
+ return self._wait(
413
+ lambda: self.get(run_id, project=project),
414
+ lambda run: run.status,
415
+ _TERMINAL_RUN_STATUSES,
416
+ target=f"Evaluation Run {run_id}",
417
+ timeout=timeout,
418
+ interval=interval,
419
+ )
420
+
421
+ def cancel(
422
+ self, run_id: str, *, project: ProjectRef | None = None
423
+ ) -> EvaluationRun:
424
+ project_id = self._resolve_project_id(project)
425
+ response = self._transport.post(
426
+ f"/api/v1/projects/{project_id}/evaluations/{escape_path(run_id)}/cancel"
427
+ )
428
+ return EvaluationRun.model_validate(response_json(response))