nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,786 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections.abc import Callable, Iterator
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import urljoin
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
import requests
|
|
11
|
+
from tusclient.client import TusClient
|
|
12
|
+
from tusclient.exceptions import TusCommunicationError, TusUploadFailed
|
|
13
|
+
from tusclient.request import TusRequest, catch_requests_error
|
|
14
|
+
from tusclient.uploader import Uploader
|
|
15
|
+
|
|
16
|
+
from nufi.exceptions import (
|
|
17
|
+
APIError,
|
|
18
|
+
ConnectionError as NuFiConnectionError,
|
|
19
|
+
DatasetUploadError,
|
|
20
|
+
ValidationError,
|
|
21
|
+
)
|
|
22
|
+
from nufi.http_utils import local_dev_url_and_headers
|
|
23
|
+
from nufi.models import (
|
|
24
|
+
Dataset,
|
|
25
|
+
DatasetImport,
|
|
26
|
+
DatasetImportListResponse,
|
|
27
|
+
DatasetListResponse,
|
|
28
|
+
Page,
|
|
29
|
+
DatasetVersion,
|
|
30
|
+
DatasetVersionListResponse,
|
|
31
|
+
)
|
|
32
|
+
from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
|
|
33
|
+
from nufi.transport import SyncTransport
|
|
34
|
+
|
|
35
|
+
MAX_DATASET_UPLOAD_SIZE = 100 * 1024**3
|
|
36
|
+
TUS_CHUNK_SIZE = 64 * 1024**2
|
|
37
|
+
TUS_RETRIES = 3
|
|
38
|
+
TUS_RETRY_DELAY = 1
|
|
39
|
+
TUS_ERROR_BODY_LIMIT = 4096
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _without_none(payload: dict[str, Any]) -> dict[str, Any]:
|
|
43
|
+
return {key: value for key, value in payload.items() if value is not None}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _validated_jsonl(path: Path) -> int:
|
|
47
|
+
if not path.is_file():
|
|
48
|
+
raise FileNotFoundError(path)
|
|
49
|
+
if path.suffix.lower() != ".jsonl":
|
|
50
|
+
raise ValueError("Dataset upload file must use the .jsonl extension.")
|
|
51
|
+
size = path.stat().st_size
|
|
52
|
+
if size == 0:
|
|
53
|
+
raise ValueError("Dataset upload file must not be empty.")
|
|
54
|
+
if size > MAX_DATASET_UPLOAD_SIZE:
|
|
55
|
+
raise ValueError("Dataset upload file must not exceed 100 GiB.")
|
|
56
|
+
return size
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _raise_upload_error(
|
|
60
|
+
error: TusCommunicationError,
|
|
61
|
+
import_id: str,
|
|
62
|
+
upload_url: str,
|
|
63
|
+
transport: SyncTransport,
|
|
64
|
+
) -> None:
|
|
65
|
+
status_code = error.status_code
|
|
66
|
+
if status_code is not None and 400 <= status_code < 500:
|
|
67
|
+
response = httpx.Response(
|
|
68
|
+
status_code,
|
|
69
|
+
headers=getattr(error, "response_headers", {}),
|
|
70
|
+
content=error.response_content or b"",
|
|
71
|
+
)
|
|
72
|
+
mapped = transport._map_http_error(response)
|
|
73
|
+
if status_code in {409, 412, 413, 415} and type(mapped) is APIError:
|
|
74
|
+
mapped = ValidationError(
|
|
75
|
+
str(mapped),
|
|
76
|
+
status_code=mapped.status_code,
|
|
77
|
+
response_body=mapped.response_body,
|
|
78
|
+
request_id=mapped.request_id,
|
|
79
|
+
)
|
|
80
|
+
raise mapped from error
|
|
81
|
+
raise DatasetUploadError(import_id, upload_url) from error
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _response_prefix(response: requests.Response) -> bytes:
|
|
85
|
+
if response.raw is None:
|
|
86
|
+
return b""
|
|
87
|
+
try:
|
|
88
|
+
try:
|
|
89
|
+
return response.raw.read(TUS_ERROR_BODY_LIMIT, decode_content=True)
|
|
90
|
+
except TypeError:
|
|
91
|
+
return response.raw.read(TUS_ERROR_BODY_LIMIT)
|
|
92
|
+
finally:
|
|
93
|
+
response.close()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class _TimedTusRequest(TusRequest):
|
|
97
|
+
def __init__(self, uploader: "_TimedTusUploader"):
|
|
98
|
+
self.uploader = uploader
|
|
99
|
+
super().__init__(uploader)
|
|
100
|
+
|
|
101
|
+
def perform(self) -> None:
|
|
102
|
+
try:
|
|
103
|
+
chunk = self.file.read(self._content_length)
|
|
104
|
+
self.add_checksum(chunk)
|
|
105
|
+
|
|
106
|
+
def send() -> requests.Response:
|
|
107
|
+
return requests.patch(
|
|
108
|
+
self._url,
|
|
109
|
+
data=chunk,
|
|
110
|
+
headers=self._request_headers,
|
|
111
|
+
verify=self.verify_tls_cert,
|
|
112
|
+
stream=True,
|
|
113
|
+
cert=self.client_cert,
|
|
114
|
+
timeout=self.uploader.request_timeout,
|
|
115
|
+
allow_redirects=False,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
response = send()
|
|
119
|
+
refreshed_headers = (
|
|
120
|
+
self.uploader.refresh_authorization()
|
|
121
|
+
if response.status_code == 401
|
|
122
|
+
else None
|
|
123
|
+
)
|
|
124
|
+
if refreshed_headers is not None:
|
|
125
|
+
_response_prefix(response)
|
|
126
|
+
self._request_headers.update(self.uploader.get_headers())
|
|
127
|
+
self._request_headers.update(refreshed_headers)
|
|
128
|
+
response = send()
|
|
129
|
+
self.status_code = response.status_code
|
|
130
|
+
self.response_headers = {
|
|
131
|
+
key.lower(): value for key, value in response.headers.items()
|
|
132
|
+
}
|
|
133
|
+
self.response_content = _response_prefix(response)
|
|
134
|
+
except requests.RequestException as exc:
|
|
135
|
+
raise TusUploadFailed(exc) from exc
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class _TimedTusUploader(Uploader):
|
|
139
|
+
def __init__(
|
|
140
|
+
self,
|
|
141
|
+
*args: Any,
|
|
142
|
+
request_timeout: float,
|
|
143
|
+
authorization_headers: Callable[[bool], dict[str, str] | None],
|
|
144
|
+
**kwargs: Any,
|
|
145
|
+
):
|
|
146
|
+
self.request_timeout = request_timeout
|
|
147
|
+
self.authorization_headers = authorization_headers
|
|
148
|
+
super().__init__(*args, **kwargs)
|
|
149
|
+
|
|
150
|
+
def get_headers(self) -> dict[str, str]:
|
|
151
|
+
if headers := self.authorization_headers(False):
|
|
152
|
+
self.client.set_headers(headers)
|
|
153
|
+
return super().get_headers()
|
|
154
|
+
|
|
155
|
+
def refresh_authorization(self) -> dict[str, str] | None:
|
|
156
|
+
headers = self.authorization_headers(True)
|
|
157
|
+
if headers is None:
|
|
158
|
+
return None
|
|
159
|
+
self.client.set_headers(headers)
|
|
160
|
+
return headers
|
|
161
|
+
|
|
162
|
+
@catch_requests_error
|
|
163
|
+
def create_url(self) -> str:
|
|
164
|
+
def send(
|
|
165
|
+
refreshed_headers: dict[str, str] | None = None,
|
|
166
|
+
) -> requests.Response:
|
|
167
|
+
headers = self.get_url_creation_headers()
|
|
168
|
+
headers.update(refreshed_headers or {})
|
|
169
|
+
return requests.post(
|
|
170
|
+
self.client.url,
|
|
171
|
+
headers=headers,
|
|
172
|
+
verify=self.verify_tls_cert,
|
|
173
|
+
cert=self.client_cert,
|
|
174
|
+
timeout=self.request_timeout,
|
|
175
|
+
stream=True,
|
|
176
|
+
allow_redirects=False,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
response = send()
|
|
180
|
+
refreshed_headers = (
|
|
181
|
+
self.refresh_authorization() if response.status_code == 401 else None
|
|
182
|
+
)
|
|
183
|
+
if refreshed_headers is not None:
|
|
184
|
+
_response_prefix(response)
|
|
185
|
+
response = send(refreshed_headers)
|
|
186
|
+
location = response.headers.get("location")
|
|
187
|
+
response_content = _response_prefix(response)
|
|
188
|
+
if not 200 <= response.status_code < 300 or location is None:
|
|
189
|
+
error = TusCommunicationError(
|
|
190
|
+
"Dataset upload creation failed.",
|
|
191
|
+
response.status_code,
|
|
192
|
+
response_content,
|
|
193
|
+
)
|
|
194
|
+
error.response_headers = dict(response.headers)
|
|
195
|
+
raise error
|
|
196
|
+
return urljoin(self.client.url, location)
|
|
197
|
+
|
|
198
|
+
@catch_requests_error
|
|
199
|
+
def get_offset(self) -> int:
|
|
200
|
+
def send(
|
|
201
|
+
refreshed_headers: dict[str, str] | None = None,
|
|
202
|
+
) -> requests.Response:
|
|
203
|
+
headers = self.get_headers()
|
|
204
|
+
headers.update(refreshed_headers or {})
|
|
205
|
+
return requests.head(
|
|
206
|
+
self.url,
|
|
207
|
+
headers=headers,
|
|
208
|
+
verify=self.verify_tls_cert,
|
|
209
|
+
cert=self.client_cert,
|
|
210
|
+
timeout=self.request_timeout,
|
|
211
|
+
stream=True,
|
|
212
|
+
allow_redirects=False,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
response = send()
|
|
216
|
+
refreshed_headers = (
|
|
217
|
+
self.refresh_authorization() if response.status_code == 401 else None
|
|
218
|
+
)
|
|
219
|
+
if refreshed_headers is not None:
|
|
220
|
+
_response_prefix(response)
|
|
221
|
+
response = send(refreshed_headers)
|
|
222
|
+
offset = response.headers.get("upload-offset")
|
|
223
|
+
response_content = _response_prefix(response)
|
|
224
|
+
if not 200 <= response.status_code < 300 or offset is None:
|
|
225
|
+
error = TusCommunicationError(
|
|
226
|
+
"Dataset upload offset recovery failed.",
|
|
227
|
+
response.status_code,
|
|
228
|
+
response_content,
|
|
229
|
+
)
|
|
230
|
+
error.response_headers = dict(response.headers)
|
|
231
|
+
raise error
|
|
232
|
+
return int(offset)
|
|
233
|
+
|
|
234
|
+
def _do_request(self) -> None:
|
|
235
|
+
self.request = _TimedTusRequest(self)
|
|
236
|
+
try:
|
|
237
|
+
self.request.perform()
|
|
238
|
+
if not 200 <= self.request.status_code < 300:
|
|
239
|
+
error = TusUploadFailed(
|
|
240
|
+
"",
|
|
241
|
+
self.request.status_code,
|
|
242
|
+
self.request.response_content,
|
|
243
|
+
)
|
|
244
|
+
error.response_headers = self.request.response_headers
|
|
245
|
+
raise error
|
|
246
|
+
except TusUploadFailed as exc:
|
|
247
|
+
self._retry_or_cry(exc)
|
|
248
|
+
|
|
249
|
+
def _retry_or_cry(self, error: TusCommunicationError) -> None:
|
|
250
|
+
if error.status_code is not None and 400 <= error.status_code < 500:
|
|
251
|
+
raise error
|
|
252
|
+
if self.retries <= self._retried:
|
|
253
|
+
raise error
|
|
254
|
+
time.sleep(self.retry_delay)
|
|
255
|
+
self._retried += 1
|
|
256
|
+
try:
|
|
257
|
+
self.offset = self.get_offset()
|
|
258
|
+
except TusCommunicationError as exc:
|
|
259
|
+
self._retry_or_cry(exc)
|
|
260
|
+
return
|
|
261
|
+
if not 0 <= self.offset <= self.stop_at:
|
|
262
|
+
raise TusCommunicationError("Dataset upload offset exceeds file bounds.")
|
|
263
|
+
if self.offset == self.stop_at:
|
|
264
|
+
self.request.response_headers["upload-offset"] = str(self.offset)
|
|
265
|
+
return
|
|
266
|
+
self._do_request()
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class DatasetsResource(BaseResource):
|
|
270
|
+
def __init__(self, *args: Any, **kwargs: Any):
|
|
271
|
+
super().__init__(*args, **kwargs)
|
|
272
|
+
self.versions = DatasetVersionsResource(self._transport, self._default_project)
|
|
273
|
+
self.imports = DatasetImportsResource(self._transport, self._default_project)
|
|
274
|
+
|
|
275
|
+
def list(
|
|
276
|
+
self,
|
|
277
|
+
*,
|
|
278
|
+
project: ProjectRef | None = None,
|
|
279
|
+
page: int | None = None,
|
|
280
|
+
limit: int | None = None,
|
|
281
|
+
) -> list[Dataset]:
|
|
282
|
+
project_id = self._resolve_project_id(project)
|
|
283
|
+
params, _, _ = self._pagination_params(page, limit)
|
|
284
|
+
response = self._transport.get(
|
|
285
|
+
f"/api/v1/projects/{project_id}/datasets", params=params
|
|
286
|
+
)
|
|
287
|
+
return DatasetListResponse.model_validate(response_json(response)).items
|
|
288
|
+
|
|
289
|
+
def list_page(
|
|
290
|
+
self,
|
|
291
|
+
*,
|
|
292
|
+
project: ProjectRef | None = None,
|
|
293
|
+
page: int | None = None,
|
|
294
|
+
limit: int | None = None,
|
|
295
|
+
) -> Page[Dataset]:
|
|
296
|
+
"""Return one Dataset page with server pagination metadata."""
|
|
297
|
+
project_id = self._resolve_project_id(project)
|
|
298
|
+
params, resolved_page, resolved_limit = self._pagination_params(
|
|
299
|
+
page, limit, default_page=1, default_limit=20
|
|
300
|
+
)
|
|
301
|
+
response = self._transport.get(
|
|
302
|
+
f"/api/v1/projects/{project_id}/datasets", params=params
|
|
303
|
+
)
|
|
304
|
+
return self._page(
|
|
305
|
+
response_json(response),
|
|
306
|
+
Dataset,
|
|
307
|
+
page=resolved_page,
|
|
308
|
+
limit=resolved_limit,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
def iter_all(
|
|
312
|
+
self,
|
|
313
|
+
*,
|
|
314
|
+
project: ProjectRef | None = None,
|
|
315
|
+
limit: int = 100,
|
|
316
|
+
) -> Iterator[Dataset]:
|
|
317
|
+
"""Iterate all Datasets in server order."""
|
|
318
|
+
return self._iter_all(
|
|
319
|
+
lambda page, page_limit: self.list_page(
|
|
320
|
+
project=project, page=page, limit=page_limit
|
|
321
|
+
),
|
|
322
|
+
limit=limit,
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
def create(
|
|
326
|
+
self,
|
|
327
|
+
name: str,
|
|
328
|
+
*,
|
|
329
|
+
project: ProjectRef | None = None,
|
|
330
|
+
description: str | None = None,
|
|
331
|
+
) -> Dataset:
|
|
332
|
+
project_id = self._resolve_project_id(project)
|
|
333
|
+
payload = {"name": name}
|
|
334
|
+
if description is not None:
|
|
335
|
+
payload["description"] = description
|
|
336
|
+
response = self._transport.post(
|
|
337
|
+
f"/api/v1/projects/{project_id}/datasets",
|
|
338
|
+
json=payload,
|
|
339
|
+
)
|
|
340
|
+
return Dataset.model_validate(response_json(response))
|
|
341
|
+
|
|
342
|
+
def get(self, dataset_id: str, *, project: ProjectRef | None = None) -> Dataset:
|
|
343
|
+
project_id = self._resolve_project_id(project)
|
|
344
|
+
response = self._transport.get(
|
|
345
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
346
|
+
)
|
|
347
|
+
return Dataset.model_validate(response_json(response))
|
|
348
|
+
|
|
349
|
+
def archive(
|
|
350
|
+
self,
|
|
351
|
+
dataset_id: str,
|
|
352
|
+
*,
|
|
353
|
+
project: ProjectRef | None = None,
|
|
354
|
+
reason: str | None = None,
|
|
355
|
+
) -> None:
|
|
356
|
+
project_id = self._resolve_project_id(project)
|
|
357
|
+
payload = {}
|
|
358
|
+
if reason is not None:
|
|
359
|
+
payload["reason"] = reason
|
|
360
|
+
self._transport.post(
|
|
361
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/archive",
|
|
362
|
+
json=payload,
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
def restore(self, dataset_id: str, *, project: ProjectRef | None = None) -> None:
|
|
366
|
+
project_id = self._resolve_project_id(project)
|
|
367
|
+
self._transport.post(
|
|
368
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/restore"
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
class DatasetVersionsResource(BaseResource):
|
|
373
|
+
def list(
|
|
374
|
+
self,
|
|
375
|
+
dataset_id: str,
|
|
376
|
+
*,
|
|
377
|
+
project: ProjectRef | None = None,
|
|
378
|
+
page: int | None = None,
|
|
379
|
+
limit: int | None = None,
|
|
380
|
+
) -> list[DatasetVersion]:
|
|
381
|
+
project_id = self._resolve_project_id(project)
|
|
382
|
+
params, _, _ = self._pagination_params(page, limit)
|
|
383
|
+
response = self._transport.get(
|
|
384
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
|
|
385
|
+
params=params,
|
|
386
|
+
)
|
|
387
|
+
return DatasetVersionListResponse.model_validate(response_json(response)).items
|
|
388
|
+
|
|
389
|
+
def list_page(
|
|
390
|
+
self,
|
|
391
|
+
dataset_id: str,
|
|
392
|
+
*,
|
|
393
|
+
project: ProjectRef | None = None,
|
|
394
|
+
page: int | None = None,
|
|
395
|
+
limit: int | None = None,
|
|
396
|
+
) -> Page[DatasetVersion]:
|
|
397
|
+
"""Return one Dataset Version page with pagination metadata."""
|
|
398
|
+
project_id = self._resolve_project_id(project)
|
|
399
|
+
params, resolved_page, resolved_limit = self._pagination_params(
|
|
400
|
+
page, limit, default_page=1, default_limit=20
|
|
401
|
+
)
|
|
402
|
+
response = self._transport.get(
|
|
403
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
|
|
404
|
+
params=params,
|
|
405
|
+
)
|
|
406
|
+
return self._page(
|
|
407
|
+
response_json(response),
|
|
408
|
+
DatasetVersion,
|
|
409
|
+
page=resolved_page,
|
|
410
|
+
limit=resolved_limit,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
def iter_all(
|
|
414
|
+
self,
|
|
415
|
+
dataset_id: str,
|
|
416
|
+
*,
|
|
417
|
+
project: ProjectRef | None = None,
|
|
418
|
+
limit: int = 100,
|
|
419
|
+
) -> Iterator[DatasetVersion]:
|
|
420
|
+
"""Iterate all versions of a Dataset in server order."""
|
|
421
|
+
return self._iter_all(
|
|
422
|
+
lambda page, page_limit: self.list_page(
|
|
423
|
+
dataset_id,
|
|
424
|
+
project=project,
|
|
425
|
+
page=page,
|
|
426
|
+
limit=page_limit,
|
|
427
|
+
),
|
|
428
|
+
limit=limit,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
def get(
|
|
432
|
+
self,
|
|
433
|
+
dataset_id: str,
|
|
434
|
+
version_id: str,
|
|
435
|
+
*,
|
|
436
|
+
project: ProjectRef | None = None,
|
|
437
|
+
) -> DatasetVersion:
|
|
438
|
+
project_id = self._resolve_project_id(project)
|
|
439
|
+
response = self._transport.get(
|
|
440
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
441
|
+
f"/versions/{escape_path(version_id)}"
|
|
442
|
+
)
|
|
443
|
+
return DatasetVersion.model_validate(response_json(response))
|
|
444
|
+
|
|
445
|
+
def archive(
|
|
446
|
+
self,
|
|
447
|
+
dataset_id: str,
|
|
448
|
+
version_id: str,
|
|
449
|
+
*,
|
|
450
|
+
project: ProjectRef | None = None,
|
|
451
|
+
reason: str | None = None,
|
|
452
|
+
) -> None:
|
|
453
|
+
project_id = self._resolve_project_id(project)
|
|
454
|
+
self._transport.post(
|
|
455
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
456
|
+
f"/versions/{escape_path(version_id)}/archive",
|
|
457
|
+
json=_without_none({"reason": reason}),
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
def restore(
|
|
461
|
+
self,
|
|
462
|
+
dataset_id: str,
|
|
463
|
+
version_id: str,
|
|
464
|
+
*,
|
|
465
|
+
project: ProjectRef | None = None,
|
|
466
|
+
) -> None:
|
|
467
|
+
project_id = self._resolve_project_id(project)
|
|
468
|
+
self._transport.post(
|
|
469
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
470
|
+
f"/versions/{escape_path(version_id)}/restore"
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
def create_from_pvc(
|
|
474
|
+
self,
|
|
475
|
+
dataset_id: str,
|
|
476
|
+
name: str,
|
|
477
|
+
*,
|
|
478
|
+
project: ProjectRef | None = None,
|
|
479
|
+
data_uri: str,
|
|
480
|
+
description: str | None = None,
|
|
481
|
+
) -> DatasetVersion:
|
|
482
|
+
project_id = self._resolve_project_id(project)
|
|
483
|
+
response = self._transport.post(
|
|
484
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
|
|
485
|
+
json=_without_none(
|
|
486
|
+
{"name": name, "description": description, "dataUri": data_uri}
|
|
487
|
+
),
|
|
488
|
+
)
|
|
489
|
+
return DatasetVersion.model_validate(response_json(response))
|
|
490
|
+
|
|
491
|
+
def create_from_file(
|
|
492
|
+
self,
|
|
493
|
+
dataset_id: str,
|
|
494
|
+
name: str,
|
|
495
|
+
*,
|
|
496
|
+
project: ProjectRef | None = None,
|
|
497
|
+
file: str | Path,
|
|
498
|
+
description: str | None = None,
|
|
499
|
+
storage_pvc: str | None = None,
|
|
500
|
+
wait: bool = False,
|
|
501
|
+
timeout: float = 1800,
|
|
502
|
+
interval: float = 5,
|
|
503
|
+
) -> DatasetImport:
|
|
504
|
+
path = Path(file)
|
|
505
|
+
size = _validated_jsonl(path)
|
|
506
|
+
source_config = _without_none(
|
|
507
|
+
{
|
|
508
|
+
"filename": path.name,
|
|
509
|
+
"sizeBytes": size,
|
|
510
|
+
"description": description,
|
|
511
|
+
"storagePvc": storage_pvc,
|
|
512
|
+
}
|
|
513
|
+
)
|
|
514
|
+
created = self._parent_imports().create(
|
|
515
|
+
dataset_id,
|
|
516
|
+
project=project,
|
|
517
|
+
requested_version_name=name,
|
|
518
|
+
kind="file_upload",
|
|
519
|
+
source_config=source_config,
|
|
520
|
+
)
|
|
521
|
+
uploaded = self._parent_imports().upload(
|
|
522
|
+
dataset_id, created.id, path, project=project
|
|
523
|
+
)
|
|
524
|
+
if wait:
|
|
525
|
+
return self._parent_imports().wait(
|
|
526
|
+
dataset_id,
|
|
527
|
+
uploaded.id,
|
|
528
|
+
project=project,
|
|
529
|
+
timeout=timeout,
|
|
530
|
+
interval=interval,
|
|
531
|
+
)
|
|
532
|
+
return uploaded
|
|
533
|
+
|
|
534
|
+
def create_from_huggingface(
|
|
535
|
+
self,
|
|
536
|
+
dataset_id: str,
|
|
537
|
+
name: str,
|
|
538
|
+
*,
|
|
539
|
+
project: ProjectRef | None = None,
|
|
540
|
+
repository: str,
|
|
541
|
+
description: str | None = None,
|
|
542
|
+
subset: str | None = None,
|
|
543
|
+
split: str | None = None,
|
|
544
|
+
revision: str | None = None,
|
|
545
|
+
storage_pvc: str | None = None,
|
|
546
|
+
wait: bool = False,
|
|
547
|
+
timeout: float = 1800,
|
|
548
|
+
interval: float = 5,
|
|
549
|
+
) -> DatasetImport:
|
|
550
|
+
source_config = _without_none(
|
|
551
|
+
{
|
|
552
|
+
"repository": repository,
|
|
553
|
+
"description": description,
|
|
554
|
+
"subset": subset,
|
|
555
|
+
"split": split,
|
|
556
|
+
"revision": revision,
|
|
557
|
+
"storagePvc": storage_pvc,
|
|
558
|
+
}
|
|
559
|
+
)
|
|
560
|
+
created = self._parent_imports().create(
|
|
561
|
+
dataset_id,
|
|
562
|
+
project=project,
|
|
563
|
+
requested_version_name=name,
|
|
564
|
+
kind="huggingface",
|
|
565
|
+
source_config=source_config,
|
|
566
|
+
)
|
|
567
|
+
if wait:
|
|
568
|
+
return self._parent_imports().wait(
|
|
569
|
+
dataset_id,
|
|
570
|
+
created.id,
|
|
571
|
+
project=project,
|
|
572
|
+
timeout=timeout,
|
|
573
|
+
interval=interval,
|
|
574
|
+
)
|
|
575
|
+
return created
|
|
576
|
+
|
|
577
|
+
def _parent_imports(self) -> "DatasetImportsResource":
|
|
578
|
+
return DatasetImportsResource(self._transport, self._default_project)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
class DatasetImportsResource(BaseResource):
|
|
582
|
+
def list(
|
|
583
|
+
self,
|
|
584
|
+
dataset_id: str,
|
|
585
|
+
*,
|
|
586
|
+
project: ProjectRef | None = None,
|
|
587
|
+
page: int | None = None,
|
|
588
|
+
limit: int | None = None,
|
|
589
|
+
) -> list[DatasetImport]:
|
|
590
|
+
project_id = self._resolve_project_id(project)
|
|
591
|
+
params, _, _ = self._pagination_params(page, limit)
|
|
592
|
+
response = self._transport.get(
|
|
593
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
|
|
594
|
+
params=params,
|
|
595
|
+
)
|
|
596
|
+
return DatasetImportListResponse.model_validate(response_json(response)).items
|
|
597
|
+
|
|
598
|
+
def list_page(
|
|
599
|
+
self,
|
|
600
|
+
dataset_id: str,
|
|
601
|
+
*,
|
|
602
|
+
project: ProjectRef | None = None,
|
|
603
|
+
page: int | None = None,
|
|
604
|
+
limit: int | None = None,
|
|
605
|
+
) -> Page[DatasetImport]:
|
|
606
|
+
"""Return one Dataset import page with pagination metadata."""
|
|
607
|
+
project_id = self._resolve_project_id(project)
|
|
608
|
+
params, resolved_page, resolved_limit = self._pagination_params(
|
|
609
|
+
page, limit, default_page=1, default_limit=20
|
|
610
|
+
)
|
|
611
|
+
response = self._transport.get(
|
|
612
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
|
|
613
|
+
params=params,
|
|
614
|
+
)
|
|
615
|
+
return self._page(
|
|
616
|
+
response_json(response),
|
|
617
|
+
DatasetImport,
|
|
618
|
+
page=resolved_page,
|
|
619
|
+
limit=resolved_limit,
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
def iter_all(
|
|
623
|
+
self,
|
|
624
|
+
dataset_id: str,
|
|
625
|
+
*,
|
|
626
|
+
project: ProjectRef | None = None,
|
|
627
|
+
limit: int = 100,
|
|
628
|
+
) -> Iterator[DatasetImport]:
|
|
629
|
+
"""Iterate all imports for a Dataset in server order."""
|
|
630
|
+
return self._iter_all(
|
|
631
|
+
lambda page, page_limit: self.list_page(
|
|
632
|
+
dataset_id,
|
|
633
|
+
project=project,
|
|
634
|
+
page=page,
|
|
635
|
+
limit=page_limit,
|
|
636
|
+
),
|
|
637
|
+
limit=limit,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
def create(
|
|
641
|
+
self,
|
|
642
|
+
dataset_id: str,
|
|
643
|
+
*,
|
|
644
|
+
project: ProjectRef | None = None,
|
|
645
|
+
requested_version_name: str | None = None,
|
|
646
|
+
kind: str,
|
|
647
|
+
source_config: dict[str, Any] | None = None,
|
|
648
|
+
) -> DatasetImport:
|
|
649
|
+
project_id = self._resolve_project_id(project)
|
|
650
|
+
payload = _without_none(
|
|
651
|
+
{
|
|
652
|
+
"requestedVersionName": requested_version_name,
|
|
653
|
+
"kind": kind,
|
|
654
|
+
"sourceConfig": source_config,
|
|
655
|
+
}
|
|
656
|
+
)
|
|
657
|
+
response = self._transport.post(
|
|
658
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
|
|
659
|
+
json=payload,
|
|
660
|
+
)
|
|
661
|
+
return DatasetImport.model_validate(response_json(response))
|
|
662
|
+
|
|
663
|
+
def get(
|
|
664
|
+
self,
|
|
665
|
+
dataset_id: str,
|
|
666
|
+
import_id: str,
|
|
667
|
+
*,
|
|
668
|
+
project: ProjectRef | None = None,
|
|
669
|
+
) -> DatasetImport:
|
|
670
|
+
project_id = self._resolve_project_id(project)
|
|
671
|
+
response = self._transport.get(
|
|
672
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
673
|
+
f"/imports/{escape_path(import_id)}"
|
|
674
|
+
)
|
|
675
|
+
return DatasetImport.model_validate(response_json(response))
|
|
676
|
+
|
|
677
|
+
def upload(
|
|
678
|
+
self,
|
|
679
|
+
dataset_id: str,
|
|
680
|
+
import_id: str,
|
|
681
|
+
file: str | Path,
|
|
682
|
+
*,
|
|
683
|
+
project: ProjectRef | None = None,
|
|
684
|
+
) -> DatasetImport:
|
|
685
|
+
project_id = self._resolve_project_id(project)
|
|
686
|
+
path = Path(file)
|
|
687
|
+
size = _validated_jsonl(path)
|
|
688
|
+
upload_path = (
|
|
689
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
690
|
+
f"/imports/{escape_path(import_id)}/upload"
|
|
691
|
+
)
|
|
692
|
+
resource_path = f"{upload_path}/{escape_path(import_id)}"
|
|
693
|
+
public_upload_url = f"{self._transport.base_url}{resource_path}"
|
|
694
|
+
headers = {
|
|
695
|
+
key: value
|
|
696
|
+
for key, value in self._transport.headers.items()
|
|
697
|
+
if key.lower() != "content-type"
|
|
698
|
+
}
|
|
699
|
+
headers["Accept-Encoding"] = "identity"
|
|
700
|
+
request_base_url, headers = local_dev_url_and_headers(
|
|
701
|
+
self._transport.base_url, headers
|
|
702
|
+
)
|
|
703
|
+
endpoint = f"{request_base_url}{upload_path}"
|
|
704
|
+
expected_resource_url = f"{request_base_url}{resource_path}"
|
|
705
|
+
try:
|
|
706
|
+
uploader = _TimedTusUploader(
|
|
707
|
+
client=TusClient(endpoint, headers=headers),
|
|
708
|
+
file_path=str(path),
|
|
709
|
+
chunk_size=TUS_CHUNK_SIZE,
|
|
710
|
+
retries=TUS_RETRIES,
|
|
711
|
+
retry_delay=TUS_RETRY_DELAY,
|
|
712
|
+
request_timeout=self._transport.timeout,
|
|
713
|
+
authorization_headers=self._transport._authorization_headers,
|
|
714
|
+
)
|
|
715
|
+
try:
|
|
716
|
+
upload_url = uploader.create_url()
|
|
717
|
+
if upload_url.rstrip("/") != expected_resource_url:
|
|
718
|
+
raise TusCommunicationError(
|
|
719
|
+
"NuFi API returned an unexpected Dataset upload URL."
|
|
720
|
+
)
|
|
721
|
+
recovered_offset = None
|
|
722
|
+
except TusCommunicationError as creation_error:
|
|
723
|
+
if creation_error.status_code != 409:
|
|
724
|
+
raise
|
|
725
|
+
upload_url = expected_resource_url
|
|
726
|
+
uploader.set_url(upload_url)
|
|
727
|
+
recovered_offset = uploader.get_offset()
|
|
728
|
+
if recovered_offset != size:
|
|
729
|
+
raise creation_error
|
|
730
|
+
uploader.set_url(upload_url)
|
|
731
|
+
uploader.offset = (
|
|
732
|
+
uploader.get_offset() if recovered_offset is None else recovered_offset
|
|
733
|
+
)
|
|
734
|
+
if not 0 <= uploader.offset <= size:
|
|
735
|
+
raise TusCommunicationError(
|
|
736
|
+
"Dataset upload offset exceeds file bounds."
|
|
737
|
+
)
|
|
738
|
+
uploader.upload()
|
|
739
|
+
except TusCommunicationError as exc:
|
|
740
|
+
_raise_upload_error(exc, import_id, public_upload_url, self._transport)
|
|
741
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
742
|
+
raise DatasetUploadError(import_id, public_upload_url) from exc
|
|
743
|
+
try:
|
|
744
|
+
return self.get(dataset_id, import_id, project=project_id)
|
|
745
|
+
except NuFiConnectionError as exc:
|
|
746
|
+
raise DatasetUploadError(import_id, public_upload_url) from exc
|
|
747
|
+
except APIError as exc:
|
|
748
|
+
if exc.status_code is not None and exc.status_code >= 500:
|
|
749
|
+
raise DatasetUploadError(import_id, public_upload_url) from exc
|
|
750
|
+
raise
|
|
751
|
+
|
|
752
|
+
def cancel(
|
|
753
|
+
self,
|
|
754
|
+
dataset_id: str,
|
|
755
|
+
import_id: str,
|
|
756
|
+
*,
|
|
757
|
+
project: ProjectRef | None = None,
|
|
758
|
+
) -> DatasetImport:
|
|
759
|
+
project_id = self._resolve_project_id(project)
|
|
760
|
+
response = self._transport.post(
|
|
761
|
+
f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
|
|
762
|
+
f"/imports/{escape_path(import_id)}/cancel"
|
|
763
|
+
)
|
|
764
|
+
return DatasetImport.model_validate(response_json(response))
|
|
765
|
+
|
|
766
|
+
def wait(
|
|
767
|
+
self,
|
|
768
|
+
dataset_id: str,
|
|
769
|
+
import_id: str,
|
|
770
|
+
*,
|
|
771
|
+
project: ProjectRef | None = None,
|
|
772
|
+
timeout: float = 1800,
|
|
773
|
+
interval: float = 5,
|
|
774
|
+
) -> DatasetImport:
|
|
775
|
+
deadline = time.monotonic() + timeout
|
|
776
|
+
last = self.get(dataset_id, import_id, project=project)
|
|
777
|
+
while last.state not in {"ready", "failed"}:
|
|
778
|
+
remaining = deadline - time.monotonic()
|
|
779
|
+
if remaining <= 0:
|
|
780
|
+
raise TimeoutError(
|
|
781
|
+
f"Timed out waiting for dataset import {import_id}; "
|
|
782
|
+
f"last state: {last.state}"
|
|
783
|
+
)
|
|
784
|
+
time.sleep(min(interval, remaining))
|
|
785
|
+
last = self.get(dataset_id, import_id, project=project)
|
|
786
|
+
return last
|