reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,589 @@
1
+ """Hand-written extension of the generated DatasetSchemaResource classes.
2
+
3
+ Adds endpoints that are hidden from Swagger via `@ApiOperation(hidden = true)`
4
+ in the live backend and therefore never appear in spec/api-docs.json — see
5
+ the "known spec-snapshot gap" (hidden endpoints). This file is never touched by
6
+ scripts/generate_resources.py since it lives outside dataset_schema.py.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import hashlib
13
+ import json
14
+ import os
15
+ import re
16
+ import tempfile
17
+ import time
18
+ import zipfile
19
+ from collections.abc import Awaitable, Callable
20
+ from dataclasses import dataclass
21
+ from pathlib import Path
22
+ from typing import Any, TypeVar
23
+
24
+ import httpx
25
+
26
+ from .._core import _NEVER_SENT_ERRORS, _TRANSIENT_ERRORS, ACCEPT_ANY, stream_upload_file
27
+ from .._exceptions import ReportnetAPIError, ServerError
28
+ from ..jobs import JobError
29
+ from .dataset_schema import AsyncDatasetSchemaResource as _GeneratedAsyncDatasetSchemaResource
30
+ from .dataset_schema import DatasetSchemaResource as _GeneratedDatasetSchemaResource
31
+
32
+ _IMPORT_PATH = "/dataschema/import"
33
+ _IMPORTED_PREFIX = "IMPORTED_"
34
+
35
+ # Indirection so tests can drive the clock without patching `time`/`asyncio`.
36
+ _monotonic = time.monotonic
37
+ _sleep = time.sleep
38
+ _async_sleep = asyncio.sleep
39
+
40
+ # Poll failures that don't stop the wait (the import keeps running server side):
41
+ # 5xx plus the transport errors the client's retry policy also treats as transient.
42
+ _POLL_RETRY_ERRORS = (ServerError, *_TRANSIENT_ERRORS)
43
+ # Export answers counted as "still changing" while waiting for an import to settle.
44
+ _EXPORT_RETRY_ERRORS = (*_POLL_RETRY_ERRORS, zipfile.BadZipFile)
45
+ _MAY_HAVE_STARTED = (
46
+ "the import may have started and still be running on the server: check the "
47
+ "dataflow's design datasets before retrying, or it is imported twice"
48
+ )
49
+
50
+
51
+ class SchemaImportError(JobError):
52
+ """A schema import did not create its datasets or settle in time."""
53
+
54
+
55
+ def _check_zip_name(file: str | os.PathLike[str]) -> None:
56
+ # ZipUtils#unZipImportSchema reads nothing unless the upload's file name
57
+ # ends in .zip, and that failure only shows up in the async part.
58
+ if not str(file).lower().endswith(".zip"):
59
+ raise ValueError(f"the schema import file name must end in .zip, got {str(file)!r}")
60
+
61
+
62
+ def _source_shapes(file: str | os.PathLike[str]) -> dict[str, _Shape]:
63
+ """The upload's schemas by name, with what each should end up holding."""
64
+ _check_zip_name(file)
65
+ shapes = _zip_shapes(Path(file), strict=True) or {}
66
+ if not shapes:
67
+ raise ValueError(f"{str(file)!r} holds no .schema file to import")
68
+ return shapes
69
+
70
+
71
+ def _describe(error: BaseException) -> str:
72
+ if isinstance(error, ReportnetAPIError):
73
+ return f"{error.status_code} {str(error)[:200]}"
74
+ return f"{type(error).__name__}: {str(error)[:200]}"
75
+
76
+
77
+ def _upload_failed(dataflow_id: int, error: Exception) -> Exception:
78
+ """A sent upload that broke off may still have started the import."""
79
+ if isinstance(error, _TRANSIENT_ERRORS) and not isinstance(error, _NEVER_SENT_ERRORS):
80
+ message = f"schema upload to dataflow {dataflow_id} broke off ({_describe(error)}); "
81
+ return SchemaImportError(message + _MAY_HAVE_STARTED)
82
+ return error
83
+
84
+
85
+ def _design_ids(dataflow_id: int, dataflow: dict[str, Any] | None) -> set[int]:
86
+ if dataflow is None:
87
+ # The backend answers an empty 200 when it swallows its own error.
88
+ raise SchemaImportError(f"dataflow {dataflow_id}: no dataflow details to start from")
89
+ return {d["id"] for d in dataflow.get("designDatasets") or []}
90
+
91
+
92
+ def _zip_digest(path: Path) -> str:
93
+ # Entry names and contents only: the export stamps entries with the current time.
94
+ digest = hashlib.sha256()
95
+ with zipfile.ZipFile(path) as zf:
96
+ for name in sorted(zf.namelist()):
97
+ digest.update(name.encode() + b"\0")
98
+ with zf.open(name) as entry:
99
+ for chunk in iter(lambda: entry.read(1 << 16), b""):
100
+ digest.update(chunk)
101
+ digest.update(b"\0")
102
+ return digest.hexdigest()
103
+
104
+
105
+ @dataclass(frozen=True)
106
+ class _Shape:
107
+ """What a dataset schema's export files hold, as counts (the import's completion test).
108
+
109
+ The backend writes each file as Jackson JSON (`ZipUtils#zipSchema`): `.schema` is the
110
+ `DataSetSchema` (`tableSchemas[].recordSchema.fieldSchema[]`), `.qcrules` a
111
+ `RulesSchema` (`rules[]`), the other three a plain list.
112
+ """
113
+
114
+ tables: tuple[tuple[str, int], ...] # (table name, field count), sorted by name
115
+ uniques: int
116
+ rules: int
117
+ integrity: int
118
+ ext_integrations: int
119
+
120
+
121
+ def _json_count(raw: bytes | None, key: str | None = None) -> int:
122
+ if raw is None:
123
+ return 0
124
+ data = json.loads(raw)
125
+ if key is not None:
126
+ data = (data or {}).get(key)
127
+ return len(data or [])
128
+
129
+
130
+ def _zip_shapes(path: Path, *, strict: bool) -> dict[str, _Shape] | None:
131
+ """Shape of every dataset schema in a schema ZIP, by dataset name (the file's base name).
132
+
133
+ `strict` raises `ValueError` on content that is not the expected JSON (the upload);
134
+ otherwise None is returned (an export caught half written counts as "not done").
135
+ """
136
+ try:
137
+ with zipfile.ZipFile(path) as zf:
138
+ files = {name: zf.read(name) for name in zf.namelist()}
139
+ shapes: dict[str, _Shape] = {}
140
+ for name, raw in files.items():
141
+ if not name.lower().endswith(".schema"):
142
+ continue
143
+ base = name[: -len(".schema")]
144
+ schema = json.loads(raw) or {}
145
+ tables = tuple(
146
+ sorted(
147
+ (
148
+ str(t.get("nameTableSchema")),
149
+ len((t.get("recordSchema") or {}).get("fieldSchema") or []),
150
+ )
151
+ for t in schema.get("tableSchemas") or []
152
+ )
153
+ )
154
+ shapes[base] = _Shape(
155
+ tables=tables,
156
+ uniques=_json_count(files.get(f"{base}.unique")),
157
+ rules=_json_count(files.get(f"{base}.qcrules"), "rules"),
158
+ integrity=_json_count(files.get(f"{base}.integrity")),
159
+ ext_integrations=_json_count(files.get(f"{base}.extintegrations")),
160
+ )
161
+ except (ValueError, AttributeError, TypeError, zipfile.BadZipFile) as error:
162
+ if strict:
163
+ raise ValueError(
164
+ f"{str(path)!r} is not a readable schema export ZIP: {error}"
165
+ ) from error
166
+ return None
167
+ return shapes
168
+
169
+
170
+ _IMPORTED_NAME = re.compile(rf"^{_IMPORTED_PREFIX}(.*?)(?: \(\d+\))?$", re.DOTALL)
171
+
172
+
173
+ def _source_name(imported_name: str) -> str:
174
+ """`IMPORTED_<name>` or `IMPORTED_<name> (1)` -> `<name>`."""
175
+ match = _IMPORTED_NAME.match(imported_name)
176
+ return match.group(1) if match else imported_name
177
+
178
+
179
+ def _differences(want: _Shape, got: _Shape | None) -> list[str]:
180
+ if got is None:
181
+ return ["no files in the export"]
182
+ out: list[str] = []
183
+ got_tables = dict(got.tables)
184
+ missing = [t for t, _ in want.tables if t not in got_tables]
185
+ if len(got.tables) != len(want.tables) or missing:
186
+ out.append(f"{len(got.tables)}/{len(want.tables)} tables")
187
+ wrong = [t for t, n in want.tables if t in got_tables and got_tables[t] != n]
188
+ if wrong:
189
+ have = sum(got_tables[t] for t in wrong)
190
+ need = sum(n for t, n in want.tables if t in wrong)
191
+ out.append(f"{have}/{need} fields in {len(wrong)} table(s)")
192
+ for label, a, b in (
193
+ ("uniques", got.uniques, want.uniques),
194
+ ("rules", got.rules, want.rules),
195
+ ("integrity rules", got.integrity, want.integrity),
196
+ ("external integrations", got.ext_integrations, want.ext_integrations),
197
+ ):
198
+ if a != b:
199
+ out.append(f"{a}/{b} {label}")
200
+ return out
201
+
202
+
203
+ class _ImportWait:
204
+ """The wait's state; the resource methods only do the I/O and sleeping."""
205
+
206
+ def __init__(
207
+ self,
208
+ dataflow_id: int,
209
+ source: dict[str, _Shape],
210
+ before: set[int],
211
+ settle: float,
212
+ timeout: float,
213
+ quiet: float = 0.0,
214
+ ) -> None:
215
+ self.dataflow_id = dataflow_id
216
+ self.source = source
217
+ self.expected = len(source)
218
+ self.before = before
219
+ self.settle = settle
220
+ self.quiet = quiet
221
+ self.timeout = timeout
222
+ self.deadline = _monotonic() + timeout
223
+ self.new: list[dict[str, Any]] = []
224
+ self._digest: str | None = None
225
+ self._stable_since = 0.0
226
+ self._pairs: list[tuple[dict[str, Any], _Shape]] | None = None
227
+ self.last_error: Exception | None = None
228
+
229
+ def _imported(self, dataflow: dict[str, Any] | None) -> list[dict[str, Any]]:
230
+ # Only the import's own: new ids named IMPORTED_..., not datasets others add.
231
+ new = [
232
+ d
233
+ for d in (dataflow or {}).get("designDatasets") or []
234
+ if d.get("id") not in self.before
235
+ and str(d.get("dataSetName", "")).startswith(_IMPORTED_PREFIX)
236
+ ]
237
+ return sorted(new, key=lambda d: d["id"])
238
+
239
+ def _check_deadline(self, now: float, what: str) -> None:
240
+ if now >= self.deadline:
241
+ message = (
242
+ f"schema import into dataflow {self.dataflow_id}: {what} after {self.timeout:g}s"
243
+ )
244
+ if self.last_error is not None:
245
+ message += f" (last poll error: {_describe(self.last_error)})"
246
+ raise SchemaImportError(message) from self.last_error
247
+
248
+ def answered(self) -> None:
249
+ self.last_error = None
250
+
251
+ def retrying(self, error: Exception) -> None:
252
+ self.last_error = error
253
+
254
+ def interrupted(self, error: Exception) -> SchemaImportError:
255
+ """A poll error that ends the wait (e.g. an expired login) while the import runs."""
256
+ return SchemaImportError(
257
+ f"waiting for the schema import into dataflow {self.dataflow_id} failed "
258
+ f"({_describe(error)}); {_MAY_HAVE_STARTED}"
259
+ )
260
+
261
+ def datasets_seen(self, dataflow: dict[str, Any] | None) -> bool:
262
+ """Step 1: True once all the import's datasets exist."""
263
+ self.new = self._imported(dataflow) or self.new
264
+ if len(self.new) >= self.expected:
265
+ return True
266
+ self._check_deadline(_monotonic(), f"{len(self.new)} of {self.expected} datasets created")
267
+ return False
268
+
269
+ def _pair(self) -> list[tuple[dict[str, Any], _Shape]]:
270
+ """Each new dataset with the shape its source schema should end up with.
271
+
272
+ By name first (`IMPORTED_<name>` <- `<name>`), then what is left in id order
273
+ against what is left of the ZIP's schemas in file order.
274
+ """
275
+ if self._pairs is None:
276
+ left = dict(self.source)
277
+ pairs: list[tuple[dict[str, Any], _Shape]] = []
278
+ rest: list[dict[str, Any]] = []
279
+ for dataset in self.new:
280
+ name = _source_name(str(dataset.get("dataSetName", "")))
281
+ if name in left:
282
+ pairs.append((dataset, left.pop(name)))
283
+ else:
284
+ rest.append(dataset)
285
+ pairs.extend(zip(rest, left.values(), strict=False))
286
+ self._pairs = pairs
287
+ return self._pairs
288
+
289
+ def mismatches(self, shapes: dict[str, _Shape] | None) -> list[str]:
290
+ """What the export still lacks compared with the uploaded ZIP (empty: all there)."""
291
+ if shapes is None:
292
+ return ["export unreadable"]
293
+ out: list[str] = []
294
+ for dataset, want in self._pair():
295
+ name = str(dataset.get("dataSetName", ""))
296
+ out.extend(f"{name} has {d}" for d in _differences(want, shapes.get(name)))
297
+ return out
298
+
299
+ def export_seen(self, state: tuple[str, dict[str, _Shape] | None] | None) -> bool:
300
+ """Step 2: True once the export matches the ZIP and then stopped changing.
301
+
302
+ The match alone is not enough (the backend still runs its SQL rule check
303
+ afterwards, which edits rules), so it must also hold for `quiet` seconds. An
304
+ export that stays unchanged for `settle` seconds while it still lacks
305
+ content means the import stalled or failed: that raises.
306
+ """
307
+ now = _monotonic()
308
+ digest, shapes = state if state is not None else (None, None)
309
+ if digest is None or digest != self._digest:
310
+ self._digest, self._stable_since = digest, now
311
+ missing = self.mismatches(shapes) if digest is not None else ["no export yet"]
312
+ unchanged = now - self._stable_since
313
+ if not missing and unchanged >= self.quiet:
314
+ return True
315
+ if missing and digest is not None and unchanged >= self.settle:
316
+ raise SchemaImportError(
317
+ f"schema import into dataflow {self.dataflow_id} seems stalled or failed: "
318
+ f"the export has not changed for {unchanged:g}s and still lacks content "
319
+ f"({'; '.join(missing[:6])})"
320
+ )
321
+ self._check_deadline(
322
+ now,
323
+ "export does not match the uploaded ZIP yet (" + "; ".join(missing[:6]) + ")"
324
+ if missing
325
+ else "export still changing",
326
+ )
327
+ return False
328
+
329
+ def result(self, dataflow: dict[str, Any] | None) -> list[dict[str, Any]]:
330
+ return self._imported(dataflow) or self.new
331
+
332
+
333
+ _T = TypeVar("_T")
334
+
335
+
336
+ def _poll(
337
+ wait: _ImportWait, call: Callable[[], _T], retry: tuple[type[Exception], ...]
338
+ ) -> _T | None:
339
+ """`call()`, or None after an error worth polling again (kept for the timeout message)."""
340
+ try:
341
+ result = call()
342
+ except retry as error:
343
+ wait.retrying(error)
344
+ return None
345
+ except (ReportnetAPIError, httpx.HTTPError) as error:
346
+ raise wait.interrupted(error) from error
347
+ wait.answered()
348
+ return result
349
+
350
+
351
+ async def _async_poll(
352
+ wait: _ImportWait, call: Callable[[], Awaitable[_T]], retry: tuple[type[Exception], ...]
353
+ ) -> _T | None:
354
+ """Async counterpart of `_poll`."""
355
+ try:
356
+ result = await call()
357
+ except retry as error:
358
+ wait.retrying(error)
359
+ return None
360
+ except (ReportnetAPIError, httpx.HTTPError) as error:
361
+ raise wait.interrupted(error) from error
362
+ wait.answered()
363
+ return result
364
+
365
+
366
+ _IMPORT_AND_WAIT_DOC = """Import a schema ZIP into a dataflow and wait until it looks done.
367
+
368
+ Returns the design datasets the import created: the `designDatasets`
369
+ entries of `dataflow_datasets()` that are new and named
370
+ `IMPORTED_<name>`, sorted by id. Raises `ValueError` if the ZIP holds
371
+ no `.schema` file, and `SchemaImportError` (a `JobError`) if the
372
+ dataflow details can't be read first, if the import stalls, or after
373
+ `timeout` seconds.
374
+
375
+ The backend gives no completion signal through the API (its
376
+ `IMPORT_DATASET_SCHEMA_COMPLETED_EVENT` / `_FAILED_EVENT` only reach
377
+ the web app's websocket, and the import lock is only on an internal
378
+ route), so this is a heuristic in two steps:
379
+
380
+ 1. until the dataflow has as many new `IMPORTED_` design datasets as
381
+ the ZIP has `.schema` files (the import creates them first, ~4 s
382
+ apart);
383
+ 2. until the dataflow's schema export (`export_dataflow_schemas_zip`)
384
+ matches the uploaded ZIP and has stopped changing. The import fills
385
+ the datasets one after another (tables ~2 s each, a dataset's
386
+ schema document only after its last table), then the unique
387
+ constraints, QC and integrity rules, external integrations and the
388
+ SQL rule check, so a quiet export is not a finished one. For each
389
+ new `IMPORTED_<name>` dataset (paired with `<name>` in the ZIP) the
390
+ export must have the same tables, the same number of fields per
391
+ table, and the same number of unique constraints, QC rules,
392
+ integrity rules and external integrations (counts only: the SQL
393
+ rule check disables and un-verifies rules that fail it, so their
394
+ `enabled`/`verified` flags are not compared). Once it matches, it
395
+ must also stay unchanged for two polls. A 5xx, a timeout or a broken
396
+ ZIP from the export counts as "not done yet".
397
+
398
+ If the export stays unchanged for `settle` seconds while it still lacks
399
+ content, the import has stalled or failed and `SchemaImportError`
400
+ says what is missing (for example "IMPORTED_X has 0/14 tables").
401
+
402
+ A 5xx or a transient transport error (timeout, dropped connection)
403
+ while polling does not end the wait: the import carries on server side,
404
+ and a retry would import a second set. The timeout's `SchemaImportError`
405
+ names the last such error and is chained to it. Any other error while
406
+ waiting (e.g. `401` once a login token expires) raises a chained
407
+ `SchemaImportError` saying the import may still be running: check the
408
+ dataflow's design datasets before retrying. `timeout` counts from the
409
+ end of the upload.
410
+
411
+ Limits: the backend reports completion only on the web app's
412
+ websocket, so this is still a heuristic. An item the import legitimately
413
+ leaves out (a rule it cannot recreate, an integration it cannot
414
+ create) shows up as a stall error naming the count, not as a silent
415
+ success. An import that fails before creating datasets (invalid names
416
+ in the ZIP, ...) ends in the timeout. `ValueError` if the ZIP holds no
417
+ readable `.schema` file. Same roles as `import_dataflow_schemas_zip()`.
418
+ Polls every `poll_interval` seconds.
419
+ """
420
+
421
+
422
+ class DatasetSchemaResource(_GeneratedDatasetSchemaResource):
423
+ def export_dataflow_schemas_zip(
424
+ self, dataflow_id: int, *, dest: str | os.PathLike[str]
425
+ ) -> None:
426
+ """GET /dataschema/export?dataflowId=...
427
+
428
+ Downloads the full dataflow schema export ZIP: one `.schema`/
429
+ `.qcrules`/`.unique`/`.integrity`/`.extintegrations` file set per
430
+ dataset schema in the dataflow, plus the global
431
+ `datasetSchemaNames.names` and `datasetSchemaIds.ids` files.
432
+
433
+ `DatasetSchemaControllerImpl#exportSchemas` (controller method
434
+ mapped to this route) is annotated `@ApiOperation(hidden = true)`,
435
+ so it never reaches spec/api-docs.json and can't come from the
436
+ generator — this is a permanent hand-written addition, not a stopgap.
437
+ Requires `DATAFLOW_CUSTODIAN` or `DATAFLOW_STEWARD` on the dataflow.
438
+
439
+ The `Accept: */*` override is required: `RequestCore.build_headers()`'s
440
+ default `Accept: application/json` gets rejected before auth is even
441
+ checked — confirmed against the live production API, this specific
442
+ route 406s behind what looks like a gateway/proxy layer in front of
443
+ the actual controller. Oddly, `Accept: application/octet-stream`
444
+ (matching the controller's declared `produces`) 406s too with an
445
+ empty body, while `Accept: */*` passes through cleanly to the real
446
+ auth/role check.
447
+ """
448
+ self._client.download(
449
+ "GET",
450
+ "/dataschema/export",
451
+ params={"dataflowId": dataflow_id},
452
+ headers=dict(ACCEPT_ANY),
453
+ dest=dest,
454
+ )
455
+
456
+ def import_dataflow_schemas_zip(self, dataflow_id: int, file: str | os.PathLike[str]) -> None:
457
+ """POST /dataschema/import?dataflowId=...: add the dataset schemas of an
458
+ export ZIP (`export_dataflow_schemas_zip()`'s file set) to a dataflow.
459
+
460
+ `DatasetSchemaControllerImpl#importSchemas` (hidden), multipart `file`.
461
+ Requires `DATAFLOW_CUSTODIAN` or `DATAFLOW_STEWARD` on the dataflow, and
462
+ the dataflow in `DESIGN` status (`403` "Invalid dataflow status"
463
+ otherwise). The file name must end in `.zip` (checked here, raises
464
+ `ValueError`): the backend ignores anything else.
465
+
466
+ Asynchronous: answers 200 once the work is started. The import
467
+ (`DataschemaServiceImpl#importSchemas`) adds one new design dataset
468
+ per schema, named `IMPORTED_<name>` (`IMPORTED_<name> (1)`, ... when
469
+ taken), never replaces existing ones, gives every schema, table,
470
+ record and field a new id (remapping links, unique constraints,
471
+ integrity checks, QC rules and external integrations to them), and
472
+ then runs the SQL QC rule check. A second import into the same
473
+ dataflow while one runs answers `423`. Use
474
+ `import_dataflow_schemas_zip_and_wait()` to wait for it.
475
+
476
+ Not retried: if the connection times out or drops after the upload was
477
+ sent, this raises `SchemaImportError` (chained to the transport error),
478
+ because the import may have started anyway; check the dataflow's design
479
+ datasets before trying again.
480
+ """
481
+ _check_zip_name(file)
482
+ try:
483
+ with stream_upload_file("file", file) as files:
484
+ self._client.request(
485
+ "POST", _IMPORT_PATH, params={"dataflowId": dataflow_id}, files=files
486
+ )
487
+ except httpx.HTTPError as error:
488
+ raise _upload_failed(dataflow_id, error) from error
489
+
490
+ def import_dataflow_schemas_zip_and_wait(
491
+ self,
492
+ dataflow_id: int,
493
+ file: str | os.PathLike[str],
494
+ *,
495
+ poll_interval: float = 5.0,
496
+ settle: float = 120.0,
497
+ timeout: float = 900.0,
498
+ ) -> list[dict[str, Any]]:
499
+ source = _source_shapes(file)
500
+ dataflow = self._client.dataflow
501
+ before = _design_ids(dataflow_id, dataflow.dataflow_datasets(dataflow_id))
502
+ self.import_dataflow_schemas_zip(dataflow_id, file)
503
+ # Started after the upload: `timeout` bounds the waiting, not the upload.
504
+ wait = _ImportWait(dataflow_id, source, before, settle, timeout, quiet=2 * poll_interval)
505
+
506
+ def datasets() -> dict[str, Any] | None:
507
+ return _poll(wait, lambda: dataflow.dataflow_datasets(dataflow_id), _POLL_RETRY_ERRORS)
508
+
509
+ while True:
510
+ _sleep(poll_interval)
511
+ if wait.datasets_seen(datasets()):
512
+ break
513
+ with tempfile.TemporaryDirectory() as tmp:
514
+ dest = Path(tmp) / "export.zip"
515
+
516
+ def export_state() -> tuple[str, dict[str, _Shape] | None]:
517
+ self.export_dataflow_schemas_zip(dataflow_id, dest=dest)
518
+ return _zip_digest(dest), _zip_shapes(dest, strict=False)
519
+
520
+ while not wait.export_seen(_poll(wait, export_state, _EXPORT_RETRY_ERRORS)):
521
+ _sleep(poll_interval)
522
+ return wait.result(datasets())
523
+
524
+ import_dataflow_schemas_zip_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
525
+
526
+
527
+ class AsyncDatasetSchemaResource(_GeneratedAsyncDatasetSchemaResource):
528
+ async def export_dataflow_schemas_zip(
529
+ self, dataflow_id: int, *, dest: str | os.PathLike[str]
530
+ ) -> None:
531
+ """Async counterpart of DatasetSchemaResource.export_dataflow_schemas_zip."""
532
+ await self._client.download(
533
+ "GET",
534
+ "/dataschema/export",
535
+ params={"dataflowId": dataflow_id},
536
+ headers=dict(ACCEPT_ANY),
537
+ dest=dest,
538
+ )
539
+
540
+ async def import_dataflow_schemas_zip(
541
+ self, dataflow_id: int, file: str | os.PathLike[str]
542
+ ) -> None:
543
+ """Async counterpart of DatasetSchemaResource.import_dataflow_schemas_zip."""
544
+ _check_zip_name(file)
545
+ try:
546
+ with stream_upload_file("file", file) as files:
547
+ await self._client.request(
548
+ "POST", _IMPORT_PATH, params={"dataflowId": dataflow_id}, files=files
549
+ )
550
+ except httpx.HTTPError as error:
551
+ raise _upload_failed(dataflow_id, error) from error
552
+
553
+ async def import_dataflow_schemas_zip_and_wait(
554
+ self,
555
+ dataflow_id: int,
556
+ file: str | os.PathLike[str],
557
+ *,
558
+ poll_interval: float = 5.0,
559
+ settle: float = 120.0,
560
+ timeout: float = 900.0,
561
+ ) -> list[dict[str, Any]]:
562
+ source = _source_shapes(file)
563
+ dataflow = self._client.dataflow
564
+ before = _design_ids(dataflow_id, await dataflow.dataflow_datasets(dataflow_id))
565
+ await self.import_dataflow_schemas_zip(dataflow_id, file)
566
+ # Started after the upload: `timeout` bounds the waiting, not the upload.
567
+ wait = _ImportWait(dataflow_id, source, before, settle, timeout, quiet=2 * poll_interval)
568
+
569
+ async def datasets() -> dict[str, Any] | None:
570
+ return await _async_poll(
571
+ wait, lambda: dataflow.dataflow_datasets(dataflow_id), _POLL_RETRY_ERRORS
572
+ )
573
+
574
+ while True:
575
+ await _async_sleep(poll_interval)
576
+ if wait.datasets_seen(await datasets()):
577
+ break
578
+ with tempfile.TemporaryDirectory() as tmp:
579
+ dest = Path(tmp) / "export.zip"
580
+
581
+ async def export_state() -> tuple[str, dict[str, _Shape] | None]:
582
+ await self.export_dataflow_schemas_zip(dataflow_id, dest=dest)
583
+ return _zip_digest(dest), _zip_shapes(dest, strict=False)
584
+
585
+ while not wait.export_seen(await _async_poll(wait, export_state, _EXPORT_RETRY_ERRORS)):
586
+ await _async_sleep(poll_interval)
587
+ return wait.result(await datasets())
588
+
589
+ import_dataflow_schemas_zip_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC