nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,413 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Iterator
4
+ from typing import Any
5
+
6
+ from pydantic import BaseModel
7
+
8
+ from nufi.models import (
9
+ Leaderboard,
10
+ LeaderboardEntry,
11
+ LeaderboardEntryListResponse,
12
+ LeaderboardGatePolicy,
13
+ LeaderboardListResponse,
14
+ LeaderboardRun,
15
+ LeaderboardRunListResponse,
16
+ LeaderboardRankingPolicy,
17
+ LeaderboardSubmission,
18
+ LeaderboardSubmissionListResponse,
19
+ Page,
20
+ )
21
+ from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
22
+
23
+ _TERMINAL_SUBMISSION_STATUSES = frozenset({"succeeded", "failed", "cancelled"})
24
+
25
+
26
+ def _without_none(payload: dict[str, Any]) -> dict[str, Any]:
27
+ return {key: value for key, value in payload.items() if value is not None}
28
+
29
+
30
+ def _json_value(value: Any) -> Any:
31
+ if isinstance(value, BaseModel):
32
+ return value.model_dump(mode="json", by_alias=True, exclude_none=True)
33
+ return value
34
+
35
+
36
+ class LeaderboardsResource(BaseResource):
37
+ def list(
38
+ self,
39
+ *,
40
+ project: ProjectRef | None = None,
41
+ status: str | None = None,
42
+ page: int | None = None,
43
+ limit: int | None = None,
44
+ ) -> list[Leaderboard]:
45
+ project_id = self._resolve_project_id(project)
46
+ pagination, _, _ = self._pagination_params(page, limit)
47
+ response = self._transport.get(
48
+ f"/api/v1/projects/{project_id}/leaderboards",
49
+ params={**_without_none({"status": status}), **pagination},
50
+ )
51
+ return LeaderboardListResponse.model_validate(response_json(response)).items
52
+
53
+ def list_page(
54
+ self,
55
+ *,
56
+ project: ProjectRef | None = None,
57
+ status: str | None = None,
58
+ page: int | None = None,
59
+ limit: int | None = None,
60
+ ) -> Page[Leaderboard]:
61
+ """Return one Leaderboard page with pagination metadata."""
62
+ project_id = self._resolve_project_id(project)
63
+ pagination, resolved_page, resolved_limit = self._pagination_params(
64
+ page, limit, default_page=1, default_limit=20
65
+ )
66
+ response = self._transport.get(
67
+ f"/api/v1/projects/{project_id}/leaderboards",
68
+ params={**_without_none({"status": status}), **pagination},
69
+ )
70
+ return self._page(
71
+ response_json(response),
72
+ Leaderboard,
73
+ page=resolved_page,
74
+ limit=resolved_limit,
75
+ )
76
+
77
+ def iter_all(
78
+ self,
79
+ *,
80
+ project: ProjectRef | None = None,
81
+ status: str | None = None,
82
+ limit: int = 100,
83
+ ) -> Iterator[Leaderboard]:
84
+ """Iterate all matching Leaderboards in server order."""
85
+ return self._iter_all(
86
+ lambda page, page_limit: self.list_page(
87
+ project=project,
88
+ status=status,
89
+ page=page,
90
+ limit=page_limit,
91
+ ),
92
+ limit=limit,
93
+ )
94
+
95
+ def create(
96
+ self,
97
+ name: str,
98
+ *,
99
+ project: ProjectRef | None = None,
100
+ evaluation_method_id: str,
101
+ ranking_policy: LeaderboardRankingPolicy | dict[str, Any],
102
+ runner_config: dict[str, Any] | None = None,
103
+ gate_policy: LeaderboardGatePolicy | dict[str, Any] | None = None,
104
+ description: str | None = None,
105
+ ) -> Leaderboard:
106
+ project_id = self._resolve_project_id(project)
107
+ response = self._transport.post(
108
+ f"/api/v1/projects/{project_id}/leaderboards",
109
+ json=_without_none(
110
+ {
111
+ "name": name,
112
+ "description": description,
113
+ "evaluationMethodId": evaluation_method_id,
114
+ "runnerConfig": runner_config,
115
+ "gatePolicy": _json_value(gate_policy),
116
+ "rankingPolicy": _json_value(ranking_policy),
117
+ }
118
+ ),
119
+ )
120
+ return Leaderboard.model_validate(response_json(response))
121
+
122
+ def get(
123
+ self, leaderboard_id: str, *, project: ProjectRef | None = None
124
+ ) -> Leaderboard:
125
+ project_id = self._resolve_project_id(project)
126
+ response = self._transport.get(
127
+ f"/api/v1/projects/{project_id}/leaderboards/{escape_path(leaderboard_id)}"
128
+ )
129
+ return Leaderboard.model_validate(response_json(response))
130
+
131
+ def update(
132
+ self,
133
+ leaderboard_id: str,
134
+ *,
135
+ project: ProjectRef | None = None,
136
+ name: str | None = None,
137
+ description: str | None = None,
138
+ ) -> Leaderboard:
139
+ project_id = self._resolve_project_id(project)
140
+ response = self._transport.patch(
141
+ f"/api/v1/projects/{project_id}/leaderboards/{escape_path(leaderboard_id)}",
142
+ json=_without_none({"name": name, "description": description}),
143
+ )
144
+ return Leaderboard.model_validate(response_json(response))
145
+
146
+ def archive(
147
+ self, leaderboard_id: str, *, project: ProjectRef | None = None
148
+ ) -> Leaderboard:
149
+ return self._change_status(leaderboard_id, project=project, action="archive")
150
+
151
+ def restore(
152
+ self, leaderboard_id: str, *, project: ProjectRef | None = None
153
+ ) -> Leaderboard:
154
+ return self._change_status(leaderboard_id, project=project, action="restore")
155
+
156
+ def _change_status(
157
+ self,
158
+ leaderboard_id: str,
159
+ *,
160
+ project: ProjectRef | None = None,
161
+ action: str,
162
+ ) -> Leaderboard:
163
+ project_id = self._resolve_project_id(project)
164
+ response = self._transport.post(
165
+ f"/api/v1/projects/{project_id}/leaderboards/"
166
+ f"{escape_path(leaderboard_id)}/{action}"
167
+ )
168
+ return Leaderboard.model_validate(response_json(response))
169
+
170
+ def submit(
171
+ self,
172
+ leaderboard_id: str,
173
+ *,
174
+ project: ProjectRef | None = None,
175
+ model_artifact_id: int,
176
+ cpu: str | None = None,
177
+ memory: str | None = None,
178
+ accelerator_count: int | None = None,
179
+ ) -> LeaderboardSubmission:
180
+ resources = (cpu, memory, accelerator_count)
181
+ if any(value is not None for value in resources) and not all(
182
+ value is not None for value in resources
183
+ ):
184
+ raise ValueError(
185
+ "cpu, memory, and accelerator_count must be provided together"
186
+ )
187
+ payload: dict[str, Any] = {"modelArtifactId": model_artifact_id}
188
+ if cpu is not None:
189
+ payload["targetResources"] = {
190
+ "cpu": cpu,
191
+ "memory": memory,
192
+ "acceleratorCount": accelerator_count,
193
+ }
194
+ project_id = self._resolve_project_id(project)
195
+ response = self._transport.post(
196
+ f"/api/v1/projects/{project_id}/leaderboards/"
197
+ f"{escape_path(leaderboard_id)}/submissions",
198
+ json=payload,
199
+ )
200
+ return LeaderboardSubmission.model_validate(response_json(response))
201
+
202
+ def list_submissions(
203
+ self,
204
+ leaderboard_id: str,
205
+ *,
206
+ project: ProjectRef | None = None,
207
+ page: int | None = None,
208
+ limit: int | None = None,
209
+ ) -> list[LeaderboardSubmission]:
210
+ project_id = self._resolve_project_id(project)
211
+ pagination, _, _ = self._pagination_params(page, limit)
212
+ response = self._transport.get(
213
+ f"/api/v1/projects/{project_id}/leaderboards/"
214
+ f"{escape_path(leaderboard_id)}/submissions",
215
+ params=pagination,
216
+ )
217
+ return LeaderboardSubmissionListResponse.model_validate(
218
+ response_json(response)
219
+ ).items
220
+
221
+ def list_submissions_page(
222
+ self,
223
+ leaderboard_id: str,
224
+ *,
225
+ project: ProjectRef | None = None,
226
+ page: int | None = None,
227
+ limit: int | None = None,
228
+ ) -> Page[LeaderboardSubmission]:
229
+ """Return one submission page with pagination metadata."""
230
+ project_id = self._resolve_project_id(project)
231
+ pagination, resolved_page, resolved_limit = self._pagination_params(
232
+ page, limit, default_page=1, default_limit=20
233
+ )
234
+ response = self._transport.get(
235
+ f"/api/v1/projects/{project_id}/leaderboards/"
236
+ f"{escape_path(leaderboard_id)}/submissions",
237
+ params=pagination,
238
+ )
239
+ return self._page(
240
+ response_json(response),
241
+ LeaderboardSubmission,
242
+ page=resolved_page,
243
+ limit=resolved_limit,
244
+ )
245
+
246
+ def iter_all_submissions(
247
+ self,
248
+ leaderboard_id: str,
249
+ *,
250
+ project: ProjectRef | None = None,
251
+ limit: int = 100,
252
+ ) -> Iterator[LeaderboardSubmission]:
253
+ """Iterate all submissions for a Leaderboard in server order."""
254
+ return self._iter_all(
255
+ lambda page, page_limit: self.list_submissions_page(
256
+ leaderboard_id,
257
+ project=project,
258
+ page=page,
259
+ limit=page_limit,
260
+ ),
261
+ limit=limit,
262
+ )
263
+
264
+ def get_submission(
265
+ self,
266
+ leaderboard_id: str,
267
+ submission_id: str,
268
+ *,
269
+ project: ProjectRef | None = None,
270
+ ) -> LeaderboardSubmission:
271
+ project_id = self._resolve_project_id(project)
272
+ response = self._transport.get(
273
+ f"/api/v1/projects/{project_id}/leaderboards/"
274
+ f"{escape_path(leaderboard_id)}/submissions/{escape_path(submission_id)}"
275
+ )
276
+ return LeaderboardSubmission.model_validate(response_json(response))
277
+
278
+ def wait(
279
+ self,
280
+ leaderboard_id: str,
281
+ submission_id: str,
282
+ *,
283
+ project: ProjectRef | None = None,
284
+ timeout: float = 1800,
285
+ interval: float = 5,
286
+ ) -> LeaderboardSubmission:
287
+ """Wait for a Leaderboard submission using only its existing GET route."""
288
+ return self._wait(
289
+ lambda: self.get_submission(leaderboard_id, submission_id, project=project),
290
+ lambda submission: (
291
+ submission.latest_run.status if submission.latest_run else None
292
+ ),
293
+ _TERMINAL_SUBMISSION_STATUSES,
294
+ target=f"Leaderboard submission {submission_id}",
295
+ timeout=timeout,
296
+ interval=interval,
297
+ )
298
+
299
+ def rerun(
300
+ self,
301
+ leaderboard_id: str,
302
+ submission_id: str,
303
+ *,
304
+ project: ProjectRef | None = None,
305
+ ) -> LeaderboardSubmission:
306
+ project_id = self._resolve_project_id(project)
307
+ response = self._transport.post(
308
+ f"/api/v1/projects/{project_id}/leaderboards/"
309
+ f"{escape_path(leaderboard_id)}/submissions/"
310
+ f"{escape_path(submission_id)}/runs"
311
+ )
312
+ return LeaderboardSubmission.model_validate(response_json(response))
313
+
314
+ def list_runs(
315
+ self,
316
+ leaderboard_id: str,
317
+ submission_id: str,
318
+ *,
319
+ project: ProjectRef | None = None,
320
+ ) -> list[LeaderboardRun]:
321
+ project_id = self._resolve_project_id(project)
322
+ response = self._transport.get(
323
+ f"/api/v1/projects/{project_id}/leaderboards/"
324
+ f"{escape_path(leaderboard_id)}/submissions/"
325
+ f"{escape_path(submission_id)}/runs"
326
+ )
327
+ return LeaderboardRunListResponse.model_validate(response_json(response)).items
328
+
329
+ def entries(
330
+ self,
331
+ leaderboard_id: str,
332
+ *,
333
+ project: ProjectRef | None = None,
334
+ platform: str | None = None,
335
+ accelerator_count: int | None = None,
336
+ page: int | None = None,
337
+ limit: int | None = None,
338
+ ) -> list[LeaderboardEntry]:
339
+ project_id = self._resolve_project_id(project)
340
+ pagination, _, _ = self._pagination_params(page, limit)
341
+ response = self._transport.get(
342
+ f"/api/v1/projects/{project_id}/leaderboards/"
343
+ f"{escape_path(leaderboard_id)}/entries",
344
+ params={
345
+ **_without_none(
346
+ {
347
+ "platform": platform,
348
+ "acceleratorCount": accelerator_count,
349
+ }
350
+ ),
351
+ **pagination,
352
+ },
353
+ )
354
+ return LeaderboardEntryListResponse.model_validate(
355
+ response_json(response)
356
+ ).items
357
+
358
+ def entries_page(
359
+ self,
360
+ leaderboard_id: str,
361
+ *,
362
+ project: ProjectRef | None = None,
363
+ platform: str | None = None,
364
+ accelerator_count: int | None = None,
365
+ page: int | None = None,
366
+ limit: int | None = None,
367
+ ) -> Page[LeaderboardEntry]:
368
+ """Return one ranked-entry page with pagination metadata."""
369
+ project_id = self._resolve_project_id(project)
370
+ pagination, resolved_page, resolved_limit = self._pagination_params(
371
+ page, limit, default_page=1, default_limit=20
372
+ )
373
+ response = self._transport.get(
374
+ f"/api/v1/projects/{project_id}/leaderboards/"
375
+ f"{escape_path(leaderboard_id)}/entries",
376
+ params={
377
+ **_without_none(
378
+ {
379
+ "platform": platform,
380
+ "acceleratorCount": accelerator_count,
381
+ }
382
+ ),
383
+ **pagination,
384
+ },
385
+ )
386
+ return self._page(
387
+ response_json(response),
388
+ LeaderboardEntry,
389
+ page=resolved_page,
390
+ limit=resolved_limit,
391
+ )
392
+
393
+ def iter_all_entries(
394
+ self,
395
+ leaderboard_id: str,
396
+ *,
397
+ project: ProjectRef | None = None,
398
+ platform: str | None = None,
399
+ accelerator_count: int | None = None,
400
+ limit: int = 100,
401
+ ) -> Iterator[LeaderboardEntry]:
402
+ """Iterate all ranked entries in server order."""
403
+ return self._iter_all(
404
+ lambda page, page_limit: self.entries_page(
405
+ leaderboard_id,
406
+ project=project,
407
+ platform=platform,
408
+ accelerator_count=accelerator_count,
409
+ page=page,
410
+ limit=page_limit,
411
+ ),
412
+ limit=limit,
413
+ )