nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,786 @@
1
+ from __future__ import annotations
2
+
3
+ import time
4
+ from collections.abc import Callable, Iterator
5
+ from pathlib import Path
6
+ from typing import Any
7
+ from urllib.parse import urljoin
8
+
9
+ import httpx
10
+ import requests
11
+ from tusclient.client import TusClient
12
+ from tusclient.exceptions import TusCommunicationError, TusUploadFailed
13
+ from tusclient.request import TusRequest, catch_requests_error
14
+ from tusclient.uploader import Uploader
15
+
16
+ from nufi.exceptions import (
17
+ APIError,
18
+ ConnectionError as NuFiConnectionError,
19
+ DatasetUploadError,
20
+ ValidationError,
21
+ )
22
+ from nufi.http_utils import local_dev_url_and_headers
23
+ from nufi.models import (
24
+ Dataset,
25
+ DatasetImport,
26
+ DatasetImportListResponse,
27
+ DatasetListResponse,
28
+ Page,
29
+ DatasetVersion,
30
+ DatasetVersionListResponse,
31
+ )
32
+ from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
33
+ from nufi.transport import SyncTransport
34
+
35
+ MAX_DATASET_UPLOAD_SIZE = 100 * 1024**3
36
+ TUS_CHUNK_SIZE = 64 * 1024**2
37
+ TUS_RETRIES = 3
38
+ TUS_RETRY_DELAY = 1
39
+ TUS_ERROR_BODY_LIMIT = 4096
40
+
41
+
42
+ def _without_none(payload: dict[str, Any]) -> dict[str, Any]:
43
+ return {key: value for key, value in payload.items() if value is not None}
44
+
45
+
46
+ def _validated_jsonl(path: Path) -> int:
47
+ if not path.is_file():
48
+ raise FileNotFoundError(path)
49
+ if path.suffix.lower() != ".jsonl":
50
+ raise ValueError("Dataset upload file must use the .jsonl extension.")
51
+ size = path.stat().st_size
52
+ if size == 0:
53
+ raise ValueError("Dataset upload file must not be empty.")
54
+ if size > MAX_DATASET_UPLOAD_SIZE:
55
+ raise ValueError("Dataset upload file must not exceed 100 GiB.")
56
+ return size
57
+
58
+
59
+ def _raise_upload_error(
60
+ error: TusCommunicationError,
61
+ import_id: str,
62
+ upload_url: str,
63
+ transport: SyncTransport,
64
+ ) -> None:
65
+ status_code = error.status_code
66
+ if status_code is not None and 400 <= status_code < 500:
67
+ response = httpx.Response(
68
+ status_code,
69
+ headers=getattr(error, "response_headers", {}),
70
+ content=error.response_content or b"",
71
+ )
72
+ mapped = transport._map_http_error(response)
73
+ if status_code in {409, 412, 413, 415} and type(mapped) is APIError:
74
+ mapped = ValidationError(
75
+ str(mapped),
76
+ status_code=mapped.status_code,
77
+ response_body=mapped.response_body,
78
+ request_id=mapped.request_id,
79
+ )
80
+ raise mapped from error
81
+ raise DatasetUploadError(import_id, upload_url) from error
82
+
83
+
84
+ def _response_prefix(response: requests.Response) -> bytes:
85
+ if response.raw is None:
86
+ return b""
87
+ try:
88
+ try:
89
+ return response.raw.read(TUS_ERROR_BODY_LIMIT, decode_content=True)
90
+ except TypeError:
91
+ return response.raw.read(TUS_ERROR_BODY_LIMIT)
92
+ finally:
93
+ response.close()
94
+
95
+
96
+ class _TimedTusRequest(TusRequest):
97
+ def __init__(self, uploader: "_TimedTusUploader"):
98
+ self.uploader = uploader
99
+ super().__init__(uploader)
100
+
101
+ def perform(self) -> None:
102
+ try:
103
+ chunk = self.file.read(self._content_length)
104
+ self.add_checksum(chunk)
105
+
106
+ def send() -> requests.Response:
107
+ return requests.patch(
108
+ self._url,
109
+ data=chunk,
110
+ headers=self._request_headers,
111
+ verify=self.verify_tls_cert,
112
+ stream=True,
113
+ cert=self.client_cert,
114
+ timeout=self.uploader.request_timeout,
115
+ allow_redirects=False,
116
+ )
117
+
118
+ response = send()
119
+ refreshed_headers = (
120
+ self.uploader.refresh_authorization()
121
+ if response.status_code == 401
122
+ else None
123
+ )
124
+ if refreshed_headers is not None:
125
+ _response_prefix(response)
126
+ self._request_headers.update(self.uploader.get_headers())
127
+ self._request_headers.update(refreshed_headers)
128
+ response = send()
129
+ self.status_code = response.status_code
130
+ self.response_headers = {
131
+ key.lower(): value for key, value in response.headers.items()
132
+ }
133
+ self.response_content = _response_prefix(response)
134
+ except requests.RequestException as exc:
135
+ raise TusUploadFailed(exc) from exc
136
+
137
+
138
+ class _TimedTusUploader(Uploader):
139
+ def __init__(
140
+ self,
141
+ *args: Any,
142
+ request_timeout: float,
143
+ authorization_headers: Callable[[bool], dict[str, str] | None],
144
+ **kwargs: Any,
145
+ ):
146
+ self.request_timeout = request_timeout
147
+ self.authorization_headers = authorization_headers
148
+ super().__init__(*args, **kwargs)
149
+
150
+ def get_headers(self) -> dict[str, str]:
151
+ if headers := self.authorization_headers(False):
152
+ self.client.set_headers(headers)
153
+ return super().get_headers()
154
+
155
+ def refresh_authorization(self) -> dict[str, str] | None:
156
+ headers = self.authorization_headers(True)
157
+ if headers is None:
158
+ return None
159
+ self.client.set_headers(headers)
160
+ return headers
161
+
162
+ @catch_requests_error
163
+ def create_url(self) -> str:
164
+ def send(
165
+ refreshed_headers: dict[str, str] | None = None,
166
+ ) -> requests.Response:
167
+ headers = self.get_url_creation_headers()
168
+ headers.update(refreshed_headers or {})
169
+ return requests.post(
170
+ self.client.url,
171
+ headers=headers,
172
+ verify=self.verify_tls_cert,
173
+ cert=self.client_cert,
174
+ timeout=self.request_timeout,
175
+ stream=True,
176
+ allow_redirects=False,
177
+ )
178
+
179
+ response = send()
180
+ refreshed_headers = (
181
+ self.refresh_authorization() if response.status_code == 401 else None
182
+ )
183
+ if refreshed_headers is not None:
184
+ _response_prefix(response)
185
+ response = send(refreshed_headers)
186
+ location = response.headers.get("location")
187
+ response_content = _response_prefix(response)
188
+ if not 200 <= response.status_code < 300 or location is None:
189
+ error = TusCommunicationError(
190
+ "Dataset upload creation failed.",
191
+ response.status_code,
192
+ response_content,
193
+ )
194
+ error.response_headers = dict(response.headers)
195
+ raise error
196
+ return urljoin(self.client.url, location)
197
+
198
+ @catch_requests_error
199
+ def get_offset(self) -> int:
200
+ def send(
201
+ refreshed_headers: dict[str, str] | None = None,
202
+ ) -> requests.Response:
203
+ headers = self.get_headers()
204
+ headers.update(refreshed_headers or {})
205
+ return requests.head(
206
+ self.url,
207
+ headers=headers,
208
+ verify=self.verify_tls_cert,
209
+ cert=self.client_cert,
210
+ timeout=self.request_timeout,
211
+ stream=True,
212
+ allow_redirects=False,
213
+ )
214
+
215
+ response = send()
216
+ refreshed_headers = (
217
+ self.refresh_authorization() if response.status_code == 401 else None
218
+ )
219
+ if refreshed_headers is not None:
220
+ _response_prefix(response)
221
+ response = send(refreshed_headers)
222
+ offset = response.headers.get("upload-offset")
223
+ response_content = _response_prefix(response)
224
+ if not 200 <= response.status_code < 300 or offset is None:
225
+ error = TusCommunicationError(
226
+ "Dataset upload offset recovery failed.",
227
+ response.status_code,
228
+ response_content,
229
+ )
230
+ error.response_headers = dict(response.headers)
231
+ raise error
232
+ return int(offset)
233
+
234
+ def _do_request(self) -> None:
235
+ self.request = _TimedTusRequest(self)
236
+ try:
237
+ self.request.perform()
238
+ if not 200 <= self.request.status_code < 300:
239
+ error = TusUploadFailed(
240
+ "",
241
+ self.request.status_code,
242
+ self.request.response_content,
243
+ )
244
+ error.response_headers = self.request.response_headers
245
+ raise error
246
+ except TusUploadFailed as exc:
247
+ self._retry_or_cry(exc)
248
+
249
+ def _retry_or_cry(self, error: TusCommunicationError) -> None:
250
+ if error.status_code is not None and 400 <= error.status_code < 500:
251
+ raise error
252
+ if self.retries <= self._retried:
253
+ raise error
254
+ time.sleep(self.retry_delay)
255
+ self._retried += 1
256
+ try:
257
+ self.offset = self.get_offset()
258
+ except TusCommunicationError as exc:
259
+ self._retry_or_cry(exc)
260
+ return
261
+ if not 0 <= self.offset <= self.stop_at:
262
+ raise TusCommunicationError("Dataset upload offset exceeds file bounds.")
263
+ if self.offset == self.stop_at:
264
+ self.request.response_headers["upload-offset"] = str(self.offset)
265
+ return
266
+ self._do_request()
267
+
268
+
269
+ class DatasetsResource(BaseResource):
270
+ def __init__(self, *args: Any, **kwargs: Any):
271
+ super().__init__(*args, **kwargs)
272
+ self.versions = DatasetVersionsResource(self._transport, self._default_project)
273
+ self.imports = DatasetImportsResource(self._transport, self._default_project)
274
+
275
+ def list(
276
+ self,
277
+ *,
278
+ project: ProjectRef | None = None,
279
+ page: int | None = None,
280
+ limit: int | None = None,
281
+ ) -> list[Dataset]:
282
+ project_id = self._resolve_project_id(project)
283
+ params, _, _ = self._pagination_params(page, limit)
284
+ response = self._transport.get(
285
+ f"/api/v1/projects/{project_id}/datasets", params=params
286
+ )
287
+ return DatasetListResponse.model_validate(response_json(response)).items
288
+
289
+ def list_page(
290
+ self,
291
+ *,
292
+ project: ProjectRef | None = None,
293
+ page: int | None = None,
294
+ limit: int | None = None,
295
+ ) -> Page[Dataset]:
296
+ """Return one Dataset page with server pagination metadata."""
297
+ project_id = self._resolve_project_id(project)
298
+ params, resolved_page, resolved_limit = self._pagination_params(
299
+ page, limit, default_page=1, default_limit=20
300
+ )
301
+ response = self._transport.get(
302
+ f"/api/v1/projects/{project_id}/datasets", params=params
303
+ )
304
+ return self._page(
305
+ response_json(response),
306
+ Dataset,
307
+ page=resolved_page,
308
+ limit=resolved_limit,
309
+ )
310
+
311
+ def iter_all(
312
+ self,
313
+ *,
314
+ project: ProjectRef | None = None,
315
+ limit: int = 100,
316
+ ) -> Iterator[Dataset]:
317
+ """Iterate all Datasets in server order."""
318
+ return self._iter_all(
319
+ lambda page, page_limit: self.list_page(
320
+ project=project, page=page, limit=page_limit
321
+ ),
322
+ limit=limit,
323
+ )
324
+
325
+ def create(
326
+ self,
327
+ name: str,
328
+ *,
329
+ project: ProjectRef | None = None,
330
+ description: str | None = None,
331
+ ) -> Dataset:
332
+ project_id = self._resolve_project_id(project)
333
+ payload = {"name": name}
334
+ if description is not None:
335
+ payload["description"] = description
336
+ response = self._transport.post(
337
+ f"/api/v1/projects/{project_id}/datasets",
338
+ json=payload,
339
+ )
340
+ return Dataset.model_validate(response_json(response))
341
+
342
+ def get(self, dataset_id: str, *, project: ProjectRef | None = None) -> Dataset:
343
+ project_id = self._resolve_project_id(project)
344
+ response = self._transport.get(
345
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
346
+ )
347
+ return Dataset.model_validate(response_json(response))
348
+
349
+ def archive(
350
+ self,
351
+ dataset_id: str,
352
+ *,
353
+ project: ProjectRef | None = None,
354
+ reason: str | None = None,
355
+ ) -> None:
356
+ project_id = self._resolve_project_id(project)
357
+ payload = {}
358
+ if reason is not None:
359
+ payload["reason"] = reason
360
+ self._transport.post(
361
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/archive",
362
+ json=payload,
363
+ )
364
+
365
+ def restore(self, dataset_id: str, *, project: ProjectRef | None = None) -> None:
366
+ project_id = self._resolve_project_id(project)
367
+ self._transport.post(
368
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/restore"
369
+ )
370
+
371
+
372
+ class DatasetVersionsResource(BaseResource):
373
+ def list(
374
+ self,
375
+ dataset_id: str,
376
+ *,
377
+ project: ProjectRef | None = None,
378
+ page: int | None = None,
379
+ limit: int | None = None,
380
+ ) -> list[DatasetVersion]:
381
+ project_id = self._resolve_project_id(project)
382
+ params, _, _ = self._pagination_params(page, limit)
383
+ response = self._transport.get(
384
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
385
+ params=params,
386
+ )
387
+ return DatasetVersionListResponse.model_validate(response_json(response)).items
388
+
389
+ def list_page(
390
+ self,
391
+ dataset_id: str,
392
+ *,
393
+ project: ProjectRef | None = None,
394
+ page: int | None = None,
395
+ limit: int | None = None,
396
+ ) -> Page[DatasetVersion]:
397
+ """Return one Dataset Version page with pagination metadata."""
398
+ project_id = self._resolve_project_id(project)
399
+ params, resolved_page, resolved_limit = self._pagination_params(
400
+ page, limit, default_page=1, default_limit=20
401
+ )
402
+ response = self._transport.get(
403
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
404
+ params=params,
405
+ )
406
+ return self._page(
407
+ response_json(response),
408
+ DatasetVersion,
409
+ page=resolved_page,
410
+ limit=resolved_limit,
411
+ )
412
+
413
+ def iter_all(
414
+ self,
415
+ dataset_id: str,
416
+ *,
417
+ project: ProjectRef | None = None,
418
+ limit: int = 100,
419
+ ) -> Iterator[DatasetVersion]:
420
+ """Iterate all versions of a Dataset in server order."""
421
+ return self._iter_all(
422
+ lambda page, page_limit: self.list_page(
423
+ dataset_id,
424
+ project=project,
425
+ page=page,
426
+ limit=page_limit,
427
+ ),
428
+ limit=limit,
429
+ )
430
+
431
+ def get(
432
+ self,
433
+ dataset_id: str,
434
+ version_id: str,
435
+ *,
436
+ project: ProjectRef | None = None,
437
+ ) -> DatasetVersion:
438
+ project_id = self._resolve_project_id(project)
439
+ response = self._transport.get(
440
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
441
+ f"/versions/{escape_path(version_id)}"
442
+ )
443
+ return DatasetVersion.model_validate(response_json(response))
444
+
445
+ def archive(
446
+ self,
447
+ dataset_id: str,
448
+ version_id: str,
449
+ *,
450
+ project: ProjectRef | None = None,
451
+ reason: str | None = None,
452
+ ) -> None:
453
+ project_id = self._resolve_project_id(project)
454
+ self._transport.post(
455
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
456
+ f"/versions/{escape_path(version_id)}/archive",
457
+ json=_without_none({"reason": reason}),
458
+ )
459
+
460
+ def restore(
461
+ self,
462
+ dataset_id: str,
463
+ version_id: str,
464
+ *,
465
+ project: ProjectRef | None = None,
466
+ ) -> None:
467
+ project_id = self._resolve_project_id(project)
468
+ self._transport.post(
469
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
470
+ f"/versions/{escape_path(version_id)}/restore"
471
+ )
472
+
473
+ def create_from_pvc(
474
+ self,
475
+ dataset_id: str,
476
+ name: str,
477
+ *,
478
+ project: ProjectRef | None = None,
479
+ data_uri: str,
480
+ description: str | None = None,
481
+ ) -> DatasetVersion:
482
+ project_id = self._resolve_project_id(project)
483
+ response = self._transport.post(
484
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/versions",
485
+ json=_without_none(
486
+ {"name": name, "description": description, "dataUri": data_uri}
487
+ ),
488
+ )
489
+ return DatasetVersion.model_validate(response_json(response))
490
+
491
+ def create_from_file(
492
+ self,
493
+ dataset_id: str,
494
+ name: str,
495
+ *,
496
+ project: ProjectRef | None = None,
497
+ file: str | Path,
498
+ description: str | None = None,
499
+ storage_pvc: str | None = None,
500
+ wait: bool = False,
501
+ timeout: float = 1800,
502
+ interval: float = 5,
503
+ ) -> DatasetImport:
504
+ path = Path(file)
505
+ size = _validated_jsonl(path)
506
+ source_config = _without_none(
507
+ {
508
+ "filename": path.name,
509
+ "sizeBytes": size,
510
+ "description": description,
511
+ "storagePvc": storage_pvc,
512
+ }
513
+ )
514
+ created = self._parent_imports().create(
515
+ dataset_id,
516
+ project=project,
517
+ requested_version_name=name,
518
+ kind="file_upload",
519
+ source_config=source_config,
520
+ )
521
+ uploaded = self._parent_imports().upload(
522
+ dataset_id, created.id, path, project=project
523
+ )
524
+ if wait:
525
+ return self._parent_imports().wait(
526
+ dataset_id,
527
+ uploaded.id,
528
+ project=project,
529
+ timeout=timeout,
530
+ interval=interval,
531
+ )
532
+ return uploaded
533
+
534
+ def create_from_huggingface(
535
+ self,
536
+ dataset_id: str,
537
+ name: str,
538
+ *,
539
+ project: ProjectRef | None = None,
540
+ repository: str,
541
+ description: str | None = None,
542
+ subset: str | None = None,
543
+ split: str | None = None,
544
+ revision: str | None = None,
545
+ storage_pvc: str | None = None,
546
+ wait: bool = False,
547
+ timeout: float = 1800,
548
+ interval: float = 5,
549
+ ) -> DatasetImport:
550
+ source_config = _without_none(
551
+ {
552
+ "repository": repository,
553
+ "description": description,
554
+ "subset": subset,
555
+ "split": split,
556
+ "revision": revision,
557
+ "storagePvc": storage_pvc,
558
+ }
559
+ )
560
+ created = self._parent_imports().create(
561
+ dataset_id,
562
+ project=project,
563
+ requested_version_name=name,
564
+ kind="huggingface",
565
+ source_config=source_config,
566
+ )
567
+ if wait:
568
+ return self._parent_imports().wait(
569
+ dataset_id,
570
+ created.id,
571
+ project=project,
572
+ timeout=timeout,
573
+ interval=interval,
574
+ )
575
+ return created
576
+
577
+ def _parent_imports(self) -> "DatasetImportsResource":
578
+ return DatasetImportsResource(self._transport, self._default_project)
579
+
580
+
581
+ class DatasetImportsResource(BaseResource):
582
+ def list(
583
+ self,
584
+ dataset_id: str,
585
+ *,
586
+ project: ProjectRef | None = None,
587
+ page: int | None = None,
588
+ limit: int | None = None,
589
+ ) -> list[DatasetImport]:
590
+ project_id = self._resolve_project_id(project)
591
+ params, _, _ = self._pagination_params(page, limit)
592
+ response = self._transport.get(
593
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
594
+ params=params,
595
+ )
596
+ return DatasetImportListResponse.model_validate(response_json(response)).items
597
+
598
+ def list_page(
599
+ self,
600
+ dataset_id: str,
601
+ *,
602
+ project: ProjectRef | None = None,
603
+ page: int | None = None,
604
+ limit: int | None = None,
605
+ ) -> Page[DatasetImport]:
606
+ """Return one Dataset import page with pagination metadata."""
607
+ project_id = self._resolve_project_id(project)
608
+ params, resolved_page, resolved_limit = self._pagination_params(
609
+ page, limit, default_page=1, default_limit=20
610
+ )
611
+ response = self._transport.get(
612
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
613
+ params=params,
614
+ )
615
+ return self._page(
616
+ response_json(response),
617
+ DatasetImport,
618
+ page=resolved_page,
619
+ limit=resolved_limit,
620
+ )
621
+
622
+ def iter_all(
623
+ self,
624
+ dataset_id: str,
625
+ *,
626
+ project: ProjectRef | None = None,
627
+ limit: int = 100,
628
+ ) -> Iterator[DatasetImport]:
629
+ """Iterate all imports for a Dataset in server order."""
630
+ return self._iter_all(
631
+ lambda page, page_limit: self.list_page(
632
+ dataset_id,
633
+ project=project,
634
+ page=page,
635
+ limit=page_limit,
636
+ ),
637
+ limit=limit,
638
+ )
639
+
640
+ def create(
641
+ self,
642
+ dataset_id: str,
643
+ *,
644
+ project: ProjectRef | None = None,
645
+ requested_version_name: str | None = None,
646
+ kind: str,
647
+ source_config: dict[str, Any] | None = None,
648
+ ) -> DatasetImport:
649
+ project_id = self._resolve_project_id(project)
650
+ payload = _without_none(
651
+ {
652
+ "requestedVersionName": requested_version_name,
653
+ "kind": kind,
654
+ "sourceConfig": source_config,
655
+ }
656
+ )
657
+ response = self._transport.post(
658
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}/imports",
659
+ json=payload,
660
+ )
661
+ return DatasetImport.model_validate(response_json(response))
662
+
663
+ def get(
664
+ self,
665
+ dataset_id: str,
666
+ import_id: str,
667
+ *,
668
+ project: ProjectRef | None = None,
669
+ ) -> DatasetImport:
670
+ project_id = self._resolve_project_id(project)
671
+ response = self._transport.get(
672
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
673
+ f"/imports/{escape_path(import_id)}"
674
+ )
675
+ return DatasetImport.model_validate(response_json(response))
676
+
677
+ def upload(
678
+ self,
679
+ dataset_id: str,
680
+ import_id: str,
681
+ file: str | Path,
682
+ *,
683
+ project: ProjectRef | None = None,
684
+ ) -> DatasetImport:
685
+ project_id = self._resolve_project_id(project)
686
+ path = Path(file)
687
+ size = _validated_jsonl(path)
688
+ upload_path = (
689
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
690
+ f"/imports/{escape_path(import_id)}/upload"
691
+ )
692
+ resource_path = f"{upload_path}/{escape_path(import_id)}"
693
+ public_upload_url = f"{self._transport.base_url}{resource_path}"
694
+ headers = {
695
+ key: value
696
+ for key, value in self._transport.headers.items()
697
+ if key.lower() != "content-type"
698
+ }
699
+ headers["Accept-Encoding"] = "identity"
700
+ request_base_url, headers = local_dev_url_and_headers(
701
+ self._transport.base_url, headers
702
+ )
703
+ endpoint = f"{request_base_url}{upload_path}"
704
+ expected_resource_url = f"{request_base_url}{resource_path}"
705
+ try:
706
+ uploader = _TimedTusUploader(
707
+ client=TusClient(endpoint, headers=headers),
708
+ file_path=str(path),
709
+ chunk_size=TUS_CHUNK_SIZE,
710
+ retries=TUS_RETRIES,
711
+ retry_delay=TUS_RETRY_DELAY,
712
+ request_timeout=self._transport.timeout,
713
+ authorization_headers=self._transport._authorization_headers,
714
+ )
715
+ try:
716
+ upload_url = uploader.create_url()
717
+ if upload_url.rstrip("/") != expected_resource_url:
718
+ raise TusCommunicationError(
719
+ "NuFi API returned an unexpected Dataset upload URL."
720
+ )
721
+ recovered_offset = None
722
+ except TusCommunicationError as creation_error:
723
+ if creation_error.status_code != 409:
724
+ raise
725
+ upload_url = expected_resource_url
726
+ uploader.set_url(upload_url)
727
+ recovered_offset = uploader.get_offset()
728
+ if recovered_offset != size:
729
+ raise creation_error
730
+ uploader.set_url(upload_url)
731
+ uploader.offset = (
732
+ uploader.get_offset() if recovered_offset is None else recovered_offset
733
+ )
734
+ if not 0 <= uploader.offset <= size:
735
+ raise TusCommunicationError(
736
+ "Dataset upload offset exceeds file bounds."
737
+ )
738
+ uploader.upload()
739
+ except TusCommunicationError as exc:
740
+ _raise_upload_error(exc, import_id, public_upload_url, self._transport)
741
+ except (OSError, TypeError, ValueError) as exc:
742
+ raise DatasetUploadError(import_id, public_upload_url) from exc
743
+ try:
744
+ return self.get(dataset_id, import_id, project=project_id)
745
+ except NuFiConnectionError as exc:
746
+ raise DatasetUploadError(import_id, public_upload_url) from exc
747
+ except APIError as exc:
748
+ if exc.status_code is not None and exc.status_code >= 500:
749
+ raise DatasetUploadError(import_id, public_upload_url) from exc
750
+ raise
751
+
752
+ def cancel(
753
+ self,
754
+ dataset_id: str,
755
+ import_id: str,
756
+ *,
757
+ project: ProjectRef | None = None,
758
+ ) -> DatasetImport:
759
+ project_id = self._resolve_project_id(project)
760
+ response = self._transport.post(
761
+ f"/api/v1/projects/{project_id}/datasets/{escape_path(dataset_id)}"
762
+ f"/imports/{escape_path(import_id)}/cancel"
763
+ )
764
+ return DatasetImport.model_validate(response_json(response))
765
+
766
+ def wait(
767
+ self,
768
+ dataset_id: str,
769
+ import_id: str,
770
+ *,
771
+ project: ProjectRef | None = None,
772
+ timeout: float = 1800,
773
+ interval: float = 5,
774
+ ) -> DatasetImport:
775
+ deadline = time.monotonic() + timeout
776
+ last = self.get(dataset_id, import_id, project=project)
777
+ while last.state not in {"ready", "failed"}:
778
+ remaining = deadline - time.monotonic()
779
+ if remaining <= 0:
780
+ raise TimeoutError(
781
+ f"Timed out waiting for dataset import {import_id}; "
782
+ f"last state: {last.state}"
783
+ )
784
+ time.sleep(min(interval, remaining))
785
+ last = self.get(dataset_id, import_id, project=project)
786
+ return last