nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
|
|
8
|
+
from nufi.models import (
|
|
9
|
+
Leaderboard,
|
|
10
|
+
LeaderboardEntry,
|
|
11
|
+
LeaderboardEntryListResponse,
|
|
12
|
+
LeaderboardGatePolicy,
|
|
13
|
+
LeaderboardListResponse,
|
|
14
|
+
LeaderboardRun,
|
|
15
|
+
LeaderboardRunListResponse,
|
|
16
|
+
LeaderboardRankingPolicy,
|
|
17
|
+
LeaderboardSubmission,
|
|
18
|
+
LeaderboardSubmissionListResponse,
|
|
19
|
+
Page,
|
|
20
|
+
)
|
|
21
|
+
from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
|
|
22
|
+
|
|
23
|
+
_TERMINAL_SUBMISSION_STATUSES = frozenset({"succeeded", "failed", "cancelled"})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _without_none(payload: dict[str, Any]) -> dict[str, Any]:
|
|
27
|
+
return {key: value for key, value in payload.items() if value is not None}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _json_value(value: Any) -> Any:
|
|
31
|
+
if isinstance(value, BaseModel):
|
|
32
|
+
return value.model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
33
|
+
return value
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LeaderboardsResource(BaseResource):
|
|
37
|
+
def list(
|
|
38
|
+
self,
|
|
39
|
+
*,
|
|
40
|
+
project: ProjectRef | None = None,
|
|
41
|
+
status: str | None = None,
|
|
42
|
+
page: int | None = None,
|
|
43
|
+
limit: int | None = None,
|
|
44
|
+
) -> list[Leaderboard]:
|
|
45
|
+
project_id = self._resolve_project_id(project)
|
|
46
|
+
pagination, _, _ = self._pagination_params(page, limit)
|
|
47
|
+
response = self._transport.get(
|
|
48
|
+
f"/api/v1/projects/{project_id}/leaderboards",
|
|
49
|
+
params={**_without_none({"status": status}), **pagination},
|
|
50
|
+
)
|
|
51
|
+
return LeaderboardListResponse.model_validate(response_json(response)).items
|
|
52
|
+
|
|
53
|
+
def list_page(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
project: ProjectRef | None = None,
|
|
57
|
+
status: str | None = None,
|
|
58
|
+
page: int | None = None,
|
|
59
|
+
limit: int | None = None,
|
|
60
|
+
) -> Page[Leaderboard]:
|
|
61
|
+
"""Return one Leaderboard page with pagination metadata."""
|
|
62
|
+
project_id = self._resolve_project_id(project)
|
|
63
|
+
pagination, resolved_page, resolved_limit = self._pagination_params(
|
|
64
|
+
page, limit, default_page=1, default_limit=20
|
|
65
|
+
)
|
|
66
|
+
response = self._transport.get(
|
|
67
|
+
f"/api/v1/projects/{project_id}/leaderboards",
|
|
68
|
+
params={**_without_none({"status": status}), **pagination},
|
|
69
|
+
)
|
|
70
|
+
return self._page(
|
|
71
|
+
response_json(response),
|
|
72
|
+
Leaderboard,
|
|
73
|
+
page=resolved_page,
|
|
74
|
+
limit=resolved_limit,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def iter_all(
|
|
78
|
+
self,
|
|
79
|
+
*,
|
|
80
|
+
project: ProjectRef | None = None,
|
|
81
|
+
status: str | None = None,
|
|
82
|
+
limit: int = 100,
|
|
83
|
+
) -> Iterator[Leaderboard]:
|
|
84
|
+
"""Iterate all matching Leaderboards in server order."""
|
|
85
|
+
return self._iter_all(
|
|
86
|
+
lambda page, page_limit: self.list_page(
|
|
87
|
+
project=project,
|
|
88
|
+
status=status,
|
|
89
|
+
page=page,
|
|
90
|
+
limit=page_limit,
|
|
91
|
+
),
|
|
92
|
+
limit=limit,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def create(
|
|
96
|
+
self,
|
|
97
|
+
name: str,
|
|
98
|
+
*,
|
|
99
|
+
project: ProjectRef | None = None,
|
|
100
|
+
evaluation_method_id: str,
|
|
101
|
+
ranking_policy: LeaderboardRankingPolicy | dict[str, Any],
|
|
102
|
+
runner_config: dict[str, Any] | None = None,
|
|
103
|
+
gate_policy: LeaderboardGatePolicy | dict[str, Any] | None = None,
|
|
104
|
+
description: str | None = None,
|
|
105
|
+
) -> Leaderboard:
|
|
106
|
+
project_id = self._resolve_project_id(project)
|
|
107
|
+
response = self._transport.post(
|
|
108
|
+
f"/api/v1/projects/{project_id}/leaderboards",
|
|
109
|
+
json=_without_none(
|
|
110
|
+
{
|
|
111
|
+
"name": name,
|
|
112
|
+
"description": description,
|
|
113
|
+
"evaluationMethodId": evaluation_method_id,
|
|
114
|
+
"runnerConfig": runner_config,
|
|
115
|
+
"gatePolicy": _json_value(gate_policy),
|
|
116
|
+
"rankingPolicy": _json_value(ranking_policy),
|
|
117
|
+
}
|
|
118
|
+
),
|
|
119
|
+
)
|
|
120
|
+
return Leaderboard.model_validate(response_json(response))
|
|
121
|
+
|
|
122
|
+
def get(
|
|
123
|
+
self, leaderboard_id: str, *, project: ProjectRef | None = None
|
|
124
|
+
) -> Leaderboard:
|
|
125
|
+
project_id = self._resolve_project_id(project)
|
|
126
|
+
response = self._transport.get(
|
|
127
|
+
f"/api/v1/projects/{project_id}/leaderboards/{escape_path(leaderboard_id)}"
|
|
128
|
+
)
|
|
129
|
+
return Leaderboard.model_validate(response_json(response))
|
|
130
|
+
|
|
131
|
+
def update(
|
|
132
|
+
self,
|
|
133
|
+
leaderboard_id: str,
|
|
134
|
+
*,
|
|
135
|
+
project: ProjectRef | None = None,
|
|
136
|
+
name: str | None = None,
|
|
137
|
+
description: str | None = None,
|
|
138
|
+
) -> Leaderboard:
|
|
139
|
+
project_id = self._resolve_project_id(project)
|
|
140
|
+
response = self._transport.patch(
|
|
141
|
+
f"/api/v1/projects/{project_id}/leaderboards/{escape_path(leaderboard_id)}",
|
|
142
|
+
json=_without_none({"name": name, "description": description}),
|
|
143
|
+
)
|
|
144
|
+
return Leaderboard.model_validate(response_json(response))
|
|
145
|
+
|
|
146
|
+
def archive(
|
|
147
|
+
self, leaderboard_id: str, *, project: ProjectRef | None = None
|
|
148
|
+
) -> Leaderboard:
|
|
149
|
+
return self._change_status(leaderboard_id, project=project, action="archive")
|
|
150
|
+
|
|
151
|
+
def restore(
|
|
152
|
+
self, leaderboard_id: str, *, project: ProjectRef | None = None
|
|
153
|
+
) -> Leaderboard:
|
|
154
|
+
return self._change_status(leaderboard_id, project=project, action="restore")
|
|
155
|
+
|
|
156
|
+
def _change_status(
|
|
157
|
+
self,
|
|
158
|
+
leaderboard_id: str,
|
|
159
|
+
*,
|
|
160
|
+
project: ProjectRef | None = None,
|
|
161
|
+
action: str,
|
|
162
|
+
) -> Leaderboard:
|
|
163
|
+
project_id = self._resolve_project_id(project)
|
|
164
|
+
response = self._transport.post(
|
|
165
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
166
|
+
f"{escape_path(leaderboard_id)}/{action}"
|
|
167
|
+
)
|
|
168
|
+
return Leaderboard.model_validate(response_json(response))
|
|
169
|
+
|
|
170
|
+
def submit(
|
|
171
|
+
self,
|
|
172
|
+
leaderboard_id: str,
|
|
173
|
+
*,
|
|
174
|
+
project: ProjectRef | None = None,
|
|
175
|
+
model_artifact_id: int,
|
|
176
|
+
cpu: str | None = None,
|
|
177
|
+
memory: str | None = None,
|
|
178
|
+
accelerator_count: int | None = None,
|
|
179
|
+
) -> LeaderboardSubmission:
|
|
180
|
+
resources = (cpu, memory, accelerator_count)
|
|
181
|
+
if any(value is not None for value in resources) and not all(
|
|
182
|
+
value is not None for value in resources
|
|
183
|
+
):
|
|
184
|
+
raise ValueError(
|
|
185
|
+
"cpu, memory, and accelerator_count must be provided together"
|
|
186
|
+
)
|
|
187
|
+
payload: dict[str, Any] = {"modelArtifactId": model_artifact_id}
|
|
188
|
+
if cpu is not None:
|
|
189
|
+
payload["targetResources"] = {
|
|
190
|
+
"cpu": cpu,
|
|
191
|
+
"memory": memory,
|
|
192
|
+
"acceleratorCount": accelerator_count,
|
|
193
|
+
}
|
|
194
|
+
project_id = self._resolve_project_id(project)
|
|
195
|
+
response = self._transport.post(
|
|
196
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
197
|
+
f"{escape_path(leaderboard_id)}/submissions",
|
|
198
|
+
json=payload,
|
|
199
|
+
)
|
|
200
|
+
return LeaderboardSubmission.model_validate(response_json(response))
|
|
201
|
+
|
|
202
|
+
def list_submissions(
|
|
203
|
+
self,
|
|
204
|
+
leaderboard_id: str,
|
|
205
|
+
*,
|
|
206
|
+
project: ProjectRef | None = None,
|
|
207
|
+
page: int | None = None,
|
|
208
|
+
limit: int | None = None,
|
|
209
|
+
) -> list[LeaderboardSubmission]:
|
|
210
|
+
project_id = self._resolve_project_id(project)
|
|
211
|
+
pagination, _, _ = self._pagination_params(page, limit)
|
|
212
|
+
response = self._transport.get(
|
|
213
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
214
|
+
f"{escape_path(leaderboard_id)}/submissions",
|
|
215
|
+
params=pagination,
|
|
216
|
+
)
|
|
217
|
+
return LeaderboardSubmissionListResponse.model_validate(
|
|
218
|
+
response_json(response)
|
|
219
|
+
).items
|
|
220
|
+
|
|
221
|
+
def list_submissions_page(
|
|
222
|
+
self,
|
|
223
|
+
leaderboard_id: str,
|
|
224
|
+
*,
|
|
225
|
+
project: ProjectRef | None = None,
|
|
226
|
+
page: int | None = None,
|
|
227
|
+
limit: int | None = None,
|
|
228
|
+
) -> Page[LeaderboardSubmission]:
|
|
229
|
+
"""Return one submission page with pagination metadata."""
|
|
230
|
+
project_id = self._resolve_project_id(project)
|
|
231
|
+
pagination, resolved_page, resolved_limit = self._pagination_params(
|
|
232
|
+
page, limit, default_page=1, default_limit=20
|
|
233
|
+
)
|
|
234
|
+
response = self._transport.get(
|
|
235
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
236
|
+
f"{escape_path(leaderboard_id)}/submissions",
|
|
237
|
+
params=pagination,
|
|
238
|
+
)
|
|
239
|
+
return self._page(
|
|
240
|
+
response_json(response),
|
|
241
|
+
LeaderboardSubmission,
|
|
242
|
+
page=resolved_page,
|
|
243
|
+
limit=resolved_limit,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
def iter_all_submissions(
|
|
247
|
+
self,
|
|
248
|
+
leaderboard_id: str,
|
|
249
|
+
*,
|
|
250
|
+
project: ProjectRef | None = None,
|
|
251
|
+
limit: int = 100,
|
|
252
|
+
) -> Iterator[LeaderboardSubmission]:
|
|
253
|
+
"""Iterate all submissions for a Leaderboard in server order."""
|
|
254
|
+
return self._iter_all(
|
|
255
|
+
lambda page, page_limit: self.list_submissions_page(
|
|
256
|
+
leaderboard_id,
|
|
257
|
+
project=project,
|
|
258
|
+
page=page,
|
|
259
|
+
limit=page_limit,
|
|
260
|
+
),
|
|
261
|
+
limit=limit,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
def get_submission(
|
|
265
|
+
self,
|
|
266
|
+
leaderboard_id: str,
|
|
267
|
+
submission_id: str,
|
|
268
|
+
*,
|
|
269
|
+
project: ProjectRef | None = None,
|
|
270
|
+
) -> LeaderboardSubmission:
|
|
271
|
+
project_id = self._resolve_project_id(project)
|
|
272
|
+
response = self._transport.get(
|
|
273
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
274
|
+
f"{escape_path(leaderboard_id)}/submissions/{escape_path(submission_id)}"
|
|
275
|
+
)
|
|
276
|
+
return LeaderboardSubmission.model_validate(response_json(response))
|
|
277
|
+
|
|
278
|
+
def wait(
|
|
279
|
+
self,
|
|
280
|
+
leaderboard_id: str,
|
|
281
|
+
submission_id: str,
|
|
282
|
+
*,
|
|
283
|
+
project: ProjectRef | None = None,
|
|
284
|
+
timeout: float = 1800,
|
|
285
|
+
interval: float = 5,
|
|
286
|
+
) -> LeaderboardSubmission:
|
|
287
|
+
"""Wait for a Leaderboard submission using only its existing GET route."""
|
|
288
|
+
return self._wait(
|
|
289
|
+
lambda: self.get_submission(leaderboard_id, submission_id, project=project),
|
|
290
|
+
lambda submission: (
|
|
291
|
+
submission.latest_run.status if submission.latest_run else None
|
|
292
|
+
),
|
|
293
|
+
_TERMINAL_SUBMISSION_STATUSES,
|
|
294
|
+
target=f"Leaderboard submission {submission_id}",
|
|
295
|
+
timeout=timeout,
|
|
296
|
+
interval=interval,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
def rerun(
|
|
300
|
+
self,
|
|
301
|
+
leaderboard_id: str,
|
|
302
|
+
submission_id: str,
|
|
303
|
+
*,
|
|
304
|
+
project: ProjectRef | None = None,
|
|
305
|
+
) -> LeaderboardSubmission:
|
|
306
|
+
project_id = self._resolve_project_id(project)
|
|
307
|
+
response = self._transport.post(
|
|
308
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
309
|
+
f"{escape_path(leaderboard_id)}/submissions/"
|
|
310
|
+
f"{escape_path(submission_id)}/runs"
|
|
311
|
+
)
|
|
312
|
+
return LeaderboardSubmission.model_validate(response_json(response))
|
|
313
|
+
|
|
314
|
+
def list_runs(
|
|
315
|
+
self,
|
|
316
|
+
leaderboard_id: str,
|
|
317
|
+
submission_id: str,
|
|
318
|
+
*,
|
|
319
|
+
project: ProjectRef | None = None,
|
|
320
|
+
) -> list[LeaderboardRun]:
|
|
321
|
+
project_id = self._resolve_project_id(project)
|
|
322
|
+
response = self._transport.get(
|
|
323
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
324
|
+
f"{escape_path(leaderboard_id)}/submissions/"
|
|
325
|
+
f"{escape_path(submission_id)}/runs"
|
|
326
|
+
)
|
|
327
|
+
return LeaderboardRunListResponse.model_validate(response_json(response)).items
|
|
328
|
+
|
|
329
|
+
def entries(
|
|
330
|
+
self,
|
|
331
|
+
leaderboard_id: str,
|
|
332
|
+
*,
|
|
333
|
+
project: ProjectRef | None = None,
|
|
334
|
+
platform: str | None = None,
|
|
335
|
+
accelerator_count: int | None = None,
|
|
336
|
+
page: int | None = None,
|
|
337
|
+
limit: int | None = None,
|
|
338
|
+
) -> list[LeaderboardEntry]:
|
|
339
|
+
project_id = self._resolve_project_id(project)
|
|
340
|
+
pagination, _, _ = self._pagination_params(page, limit)
|
|
341
|
+
response = self._transport.get(
|
|
342
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
343
|
+
f"{escape_path(leaderboard_id)}/entries",
|
|
344
|
+
params={
|
|
345
|
+
**_without_none(
|
|
346
|
+
{
|
|
347
|
+
"platform": platform,
|
|
348
|
+
"acceleratorCount": accelerator_count,
|
|
349
|
+
}
|
|
350
|
+
),
|
|
351
|
+
**pagination,
|
|
352
|
+
},
|
|
353
|
+
)
|
|
354
|
+
return LeaderboardEntryListResponse.model_validate(
|
|
355
|
+
response_json(response)
|
|
356
|
+
).items
|
|
357
|
+
|
|
358
|
+
def entries_page(
|
|
359
|
+
self,
|
|
360
|
+
leaderboard_id: str,
|
|
361
|
+
*,
|
|
362
|
+
project: ProjectRef | None = None,
|
|
363
|
+
platform: str | None = None,
|
|
364
|
+
accelerator_count: int | None = None,
|
|
365
|
+
page: int | None = None,
|
|
366
|
+
limit: int | None = None,
|
|
367
|
+
) -> Page[LeaderboardEntry]:
|
|
368
|
+
"""Return one ranked-entry page with pagination metadata."""
|
|
369
|
+
project_id = self._resolve_project_id(project)
|
|
370
|
+
pagination, resolved_page, resolved_limit = self._pagination_params(
|
|
371
|
+
page, limit, default_page=1, default_limit=20
|
|
372
|
+
)
|
|
373
|
+
response = self._transport.get(
|
|
374
|
+
f"/api/v1/projects/{project_id}/leaderboards/"
|
|
375
|
+
f"{escape_path(leaderboard_id)}/entries",
|
|
376
|
+
params={
|
|
377
|
+
**_without_none(
|
|
378
|
+
{
|
|
379
|
+
"platform": platform,
|
|
380
|
+
"acceleratorCount": accelerator_count,
|
|
381
|
+
}
|
|
382
|
+
),
|
|
383
|
+
**pagination,
|
|
384
|
+
},
|
|
385
|
+
)
|
|
386
|
+
return self._page(
|
|
387
|
+
response_json(response),
|
|
388
|
+
LeaderboardEntry,
|
|
389
|
+
page=resolved_page,
|
|
390
|
+
limit=resolved_limit,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
def iter_all_entries(
|
|
394
|
+
self,
|
|
395
|
+
leaderboard_id: str,
|
|
396
|
+
*,
|
|
397
|
+
project: ProjectRef | None = None,
|
|
398
|
+
platform: str | None = None,
|
|
399
|
+
accelerator_count: int | None = None,
|
|
400
|
+
limit: int = 100,
|
|
401
|
+
) -> Iterator[LeaderboardEntry]:
|
|
402
|
+
"""Iterate all ranked entries in server order."""
|
|
403
|
+
return self._iter_all(
|
|
404
|
+
lambda page, page_limit: self.entries_page(
|
|
405
|
+
leaderboard_id,
|
|
406
|
+
project=project,
|
|
407
|
+
platform=platform,
|
|
408
|
+
accelerator_count=accelerator_count,
|
|
409
|
+
page=page,
|
|
410
|
+
limit=page_limit,
|
|
411
|
+
),
|
|
412
|
+
limit=limit,
|
|
413
|
+
)
|