reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
"""Hand-written extension of the generated DatasetSchemaResource classes.
|
|
2
|
+
|
|
3
|
+
Adds endpoints that are hidden from Swagger via `@ApiOperation(hidden = true)`
|
|
4
|
+
in the live backend and therefore never appear in spec/api-docs.json — see
|
|
5
|
+
the "known spec-snapshot gap" (hidden endpoints). This file is never touched by
|
|
6
|
+
scripts/generate_resources.py since it lives outside dataset_schema.py.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import tempfile
|
|
17
|
+
import time
|
|
18
|
+
import zipfile
|
|
19
|
+
from collections.abc import Awaitable, Callable
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, TypeVar
|
|
23
|
+
|
|
24
|
+
import httpx
|
|
25
|
+
|
|
26
|
+
from .._core import _NEVER_SENT_ERRORS, _TRANSIENT_ERRORS, ACCEPT_ANY, stream_upload_file
|
|
27
|
+
from .._exceptions import ReportnetAPIError, ServerError
|
|
28
|
+
from ..jobs import JobError
|
|
29
|
+
from .dataset_schema import AsyncDatasetSchemaResource as _GeneratedAsyncDatasetSchemaResource
|
|
30
|
+
from .dataset_schema import DatasetSchemaResource as _GeneratedDatasetSchemaResource
|
|
31
|
+
|
|
32
|
+
_IMPORT_PATH = "/dataschema/import"
|
|
33
|
+
_IMPORTED_PREFIX = "IMPORTED_"
|
|
34
|
+
|
|
35
|
+
# Indirection so tests can drive the clock without patching `time`/`asyncio`.
|
|
36
|
+
_monotonic = time.monotonic
|
|
37
|
+
_sleep = time.sleep
|
|
38
|
+
_async_sleep = asyncio.sleep
|
|
39
|
+
|
|
40
|
+
# Poll failures that don't stop the wait (the import keeps running server side):
|
|
41
|
+
# 5xx plus the transport errors the client's retry policy also treats as transient.
|
|
42
|
+
_POLL_RETRY_ERRORS = (ServerError, *_TRANSIENT_ERRORS)
|
|
43
|
+
# Export answers counted as "still changing" while waiting for an import to settle.
|
|
44
|
+
_EXPORT_RETRY_ERRORS = (*_POLL_RETRY_ERRORS, zipfile.BadZipFile)
|
|
45
|
+
_MAY_HAVE_STARTED = (
|
|
46
|
+
"the import may have started and still be running on the server: check the "
|
|
47
|
+
"dataflow's design datasets before retrying, or it is imported twice"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class SchemaImportError(JobError):
|
|
52
|
+
"""A schema import did not create its datasets or settle in time."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _check_zip_name(file: str | os.PathLike[str]) -> None:
|
|
56
|
+
# ZipUtils#unZipImportSchema reads nothing unless the upload's file name
|
|
57
|
+
# ends in .zip, and that failure only shows up in the async part.
|
|
58
|
+
if not str(file).lower().endswith(".zip"):
|
|
59
|
+
raise ValueError(f"the schema import file name must end in .zip, got {str(file)!r}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _source_shapes(file: str | os.PathLike[str]) -> dict[str, _Shape]:
|
|
63
|
+
"""The upload's schemas by name, with what each should end up holding."""
|
|
64
|
+
_check_zip_name(file)
|
|
65
|
+
shapes = _zip_shapes(Path(file), strict=True) or {}
|
|
66
|
+
if not shapes:
|
|
67
|
+
raise ValueError(f"{str(file)!r} holds no .schema file to import")
|
|
68
|
+
return shapes
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _describe(error: BaseException) -> str:
|
|
72
|
+
if isinstance(error, ReportnetAPIError):
|
|
73
|
+
return f"{error.status_code} {str(error)[:200]}"
|
|
74
|
+
return f"{type(error).__name__}: {str(error)[:200]}"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _upload_failed(dataflow_id: int, error: Exception) -> Exception:
|
|
78
|
+
"""A sent upload that broke off may still have started the import."""
|
|
79
|
+
if isinstance(error, _TRANSIENT_ERRORS) and not isinstance(error, _NEVER_SENT_ERRORS):
|
|
80
|
+
message = f"schema upload to dataflow {dataflow_id} broke off ({_describe(error)}); "
|
|
81
|
+
return SchemaImportError(message + _MAY_HAVE_STARTED)
|
|
82
|
+
return error
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _design_ids(dataflow_id: int, dataflow: dict[str, Any] | None) -> set[int]:
|
|
86
|
+
if dataflow is None:
|
|
87
|
+
# The backend answers an empty 200 when it swallows its own error.
|
|
88
|
+
raise SchemaImportError(f"dataflow {dataflow_id}: no dataflow details to start from")
|
|
89
|
+
return {d["id"] for d in dataflow.get("designDatasets") or []}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _zip_digest(path: Path) -> str:
|
|
93
|
+
# Entry names and contents only: the export stamps entries with the current time.
|
|
94
|
+
digest = hashlib.sha256()
|
|
95
|
+
with zipfile.ZipFile(path) as zf:
|
|
96
|
+
for name in sorted(zf.namelist()):
|
|
97
|
+
digest.update(name.encode() + b"\0")
|
|
98
|
+
with zf.open(name) as entry:
|
|
99
|
+
for chunk in iter(lambda: entry.read(1 << 16), b""):
|
|
100
|
+
digest.update(chunk)
|
|
101
|
+
digest.update(b"\0")
|
|
102
|
+
return digest.hexdigest()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass(frozen=True)
|
|
106
|
+
class _Shape:
|
|
107
|
+
"""What a dataset schema's export files hold, as counts (the import's completion test).
|
|
108
|
+
|
|
109
|
+
The backend writes each file as Jackson JSON (`ZipUtils#zipSchema`): `.schema` is the
|
|
110
|
+
`DataSetSchema` (`tableSchemas[].recordSchema.fieldSchema[]`), `.qcrules` a
|
|
111
|
+
`RulesSchema` (`rules[]`), the other three a plain list.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
tables: tuple[tuple[str, int], ...] # (table name, field count), sorted by name
|
|
115
|
+
uniques: int
|
|
116
|
+
rules: int
|
|
117
|
+
integrity: int
|
|
118
|
+
ext_integrations: int
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _json_count(raw: bytes | None, key: str | None = None) -> int:
|
|
122
|
+
if raw is None:
|
|
123
|
+
return 0
|
|
124
|
+
data = json.loads(raw)
|
|
125
|
+
if key is not None:
|
|
126
|
+
data = (data or {}).get(key)
|
|
127
|
+
return len(data or [])
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _zip_shapes(path: Path, *, strict: bool) -> dict[str, _Shape] | None:
|
|
131
|
+
"""Shape of every dataset schema in a schema ZIP, by dataset name (the file's base name).
|
|
132
|
+
|
|
133
|
+
`strict` raises `ValueError` on content that is not the expected JSON (the upload);
|
|
134
|
+
otherwise None is returned (an export caught half written counts as "not done").
|
|
135
|
+
"""
|
|
136
|
+
try:
|
|
137
|
+
with zipfile.ZipFile(path) as zf:
|
|
138
|
+
files = {name: zf.read(name) for name in zf.namelist()}
|
|
139
|
+
shapes: dict[str, _Shape] = {}
|
|
140
|
+
for name, raw in files.items():
|
|
141
|
+
if not name.lower().endswith(".schema"):
|
|
142
|
+
continue
|
|
143
|
+
base = name[: -len(".schema")]
|
|
144
|
+
schema = json.loads(raw) or {}
|
|
145
|
+
tables = tuple(
|
|
146
|
+
sorted(
|
|
147
|
+
(
|
|
148
|
+
str(t.get("nameTableSchema")),
|
|
149
|
+
len((t.get("recordSchema") or {}).get("fieldSchema") or []),
|
|
150
|
+
)
|
|
151
|
+
for t in schema.get("tableSchemas") or []
|
|
152
|
+
)
|
|
153
|
+
)
|
|
154
|
+
shapes[base] = _Shape(
|
|
155
|
+
tables=tables,
|
|
156
|
+
uniques=_json_count(files.get(f"{base}.unique")),
|
|
157
|
+
rules=_json_count(files.get(f"{base}.qcrules"), "rules"),
|
|
158
|
+
integrity=_json_count(files.get(f"{base}.integrity")),
|
|
159
|
+
ext_integrations=_json_count(files.get(f"{base}.extintegrations")),
|
|
160
|
+
)
|
|
161
|
+
except (ValueError, AttributeError, TypeError, zipfile.BadZipFile) as error:
|
|
162
|
+
if strict:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
f"{str(path)!r} is not a readable schema export ZIP: {error}"
|
|
165
|
+
) from error
|
|
166
|
+
return None
|
|
167
|
+
return shapes
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
_IMPORTED_NAME = re.compile(rf"^{_IMPORTED_PREFIX}(.*?)(?: \(\d+\))?$", re.DOTALL)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _source_name(imported_name: str) -> str:
|
|
174
|
+
"""`IMPORTED_<name>` or `IMPORTED_<name> (1)` -> `<name>`."""
|
|
175
|
+
match = _IMPORTED_NAME.match(imported_name)
|
|
176
|
+
return match.group(1) if match else imported_name
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _differences(want: _Shape, got: _Shape | None) -> list[str]:
|
|
180
|
+
if got is None:
|
|
181
|
+
return ["no files in the export"]
|
|
182
|
+
out: list[str] = []
|
|
183
|
+
got_tables = dict(got.tables)
|
|
184
|
+
missing = [t for t, _ in want.tables if t not in got_tables]
|
|
185
|
+
if len(got.tables) != len(want.tables) or missing:
|
|
186
|
+
out.append(f"{len(got.tables)}/{len(want.tables)} tables")
|
|
187
|
+
wrong = [t for t, n in want.tables if t in got_tables and got_tables[t] != n]
|
|
188
|
+
if wrong:
|
|
189
|
+
have = sum(got_tables[t] for t in wrong)
|
|
190
|
+
need = sum(n for t, n in want.tables if t in wrong)
|
|
191
|
+
out.append(f"{have}/{need} fields in {len(wrong)} table(s)")
|
|
192
|
+
for label, a, b in (
|
|
193
|
+
("uniques", got.uniques, want.uniques),
|
|
194
|
+
("rules", got.rules, want.rules),
|
|
195
|
+
("integrity rules", got.integrity, want.integrity),
|
|
196
|
+
("external integrations", got.ext_integrations, want.ext_integrations),
|
|
197
|
+
):
|
|
198
|
+
if a != b:
|
|
199
|
+
out.append(f"{a}/{b} {label}")
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class _ImportWait:
|
|
204
|
+
"""The wait's state; the resource methods only do the I/O and sleeping."""
|
|
205
|
+
|
|
206
|
+
def __init__(
|
|
207
|
+
self,
|
|
208
|
+
dataflow_id: int,
|
|
209
|
+
source: dict[str, _Shape],
|
|
210
|
+
before: set[int],
|
|
211
|
+
settle: float,
|
|
212
|
+
timeout: float,
|
|
213
|
+
quiet: float = 0.0,
|
|
214
|
+
) -> None:
|
|
215
|
+
self.dataflow_id = dataflow_id
|
|
216
|
+
self.source = source
|
|
217
|
+
self.expected = len(source)
|
|
218
|
+
self.before = before
|
|
219
|
+
self.settle = settle
|
|
220
|
+
self.quiet = quiet
|
|
221
|
+
self.timeout = timeout
|
|
222
|
+
self.deadline = _monotonic() + timeout
|
|
223
|
+
self.new: list[dict[str, Any]] = []
|
|
224
|
+
self._digest: str | None = None
|
|
225
|
+
self._stable_since = 0.0
|
|
226
|
+
self._pairs: list[tuple[dict[str, Any], _Shape]] | None = None
|
|
227
|
+
self.last_error: Exception | None = None
|
|
228
|
+
|
|
229
|
+
def _imported(self, dataflow: dict[str, Any] | None) -> list[dict[str, Any]]:
|
|
230
|
+
# Only the import's own: new ids named IMPORTED_..., not datasets others add.
|
|
231
|
+
new = [
|
|
232
|
+
d
|
|
233
|
+
for d in (dataflow or {}).get("designDatasets") or []
|
|
234
|
+
if d.get("id") not in self.before
|
|
235
|
+
and str(d.get("dataSetName", "")).startswith(_IMPORTED_PREFIX)
|
|
236
|
+
]
|
|
237
|
+
return sorted(new, key=lambda d: d["id"])
|
|
238
|
+
|
|
239
|
+
def _check_deadline(self, now: float, what: str) -> None:
|
|
240
|
+
if now >= self.deadline:
|
|
241
|
+
message = (
|
|
242
|
+
f"schema import into dataflow {self.dataflow_id}: {what} after {self.timeout:g}s"
|
|
243
|
+
)
|
|
244
|
+
if self.last_error is not None:
|
|
245
|
+
message += f" (last poll error: {_describe(self.last_error)})"
|
|
246
|
+
raise SchemaImportError(message) from self.last_error
|
|
247
|
+
|
|
248
|
+
def answered(self) -> None:
|
|
249
|
+
self.last_error = None
|
|
250
|
+
|
|
251
|
+
def retrying(self, error: Exception) -> None:
|
|
252
|
+
self.last_error = error
|
|
253
|
+
|
|
254
|
+
def interrupted(self, error: Exception) -> SchemaImportError:
|
|
255
|
+
"""A poll error that ends the wait (e.g. an expired login) while the import runs."""
|
|
256
|
+
return SchemaImportError(
|
|
257
|
+
f"waiting for the schema import into dataflow {self.dataflow_id} failed "
|
|
258
|
+
f"({_describe(error)}); {_MAY_HAVE_STARTED}"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
def datasets_seen(self, dataflow: dict[str, Any] | None) -> bool:
|
|
262
|
+
"""Step 1: True once all the import's datasets exist."""
|
|
263
|
+
self.new = self._imported(dataflow) or self.new
|
|
264
|
+
if len(self.new) >= self.expected:
|
|
265
|
+
return True
|
|
266
|
+
self._check_deadline(_monotonic(), f"{len(self.new)} of {self.expected} datasets created")
|
|
267
|
+
return False
|
|
268
|
+
|
|
269
|
+
def _pair(self) -> list[tuple[dict[str, Any], _Shape]]:
|
|
270
|
+
"""Each new dataset with the shape its source schema should end up with.
|
|
271
|
+
|
|
272
|
+
By name first (`IMPORTED_<name>` <- `<name>`), then what is left in id order
|
|
273
|
+
against what is left of the ZIP's schemas in file order.
|
|
274
|
+
"""
|
|
275
|
+
if self._pairs is None:
|
|
276
|
+
left = dict(self.source)
|
|
277
|
+
pairs: list[tuple[dict[str, Any], _Shape]] = []
|
|
278
|
+
rest: list[dict[str, Any]] = []
|
|
279
|
+
for dataset in self.new:
|
|
280
|
+
name = _source_name(str(dataset.get("dataSetName", "")))
|
|
281
|
+
if name in left:
|
|
282
|
+
pairs.append((dataset, left.pop(name)))
|
|
283
|
+
else:
|
|
284
|
+
rest.append(dataset)
|
|
285
|
+
pairs.extend(zip(rest, left.values(), strict=False))
|
|
286
|
+
self._pairs = pairs
|
|
287
|
+
return self._pairs
|
|
288
|
+
|
|
289
|
+
def mismatches(self, shapes: dict[str, _Shape] | None) -> list[str]:
|
|
290
|
+
"""What the export still lacks compared with the uploaded ZIP (empty: all there)."""
|
|
291
|
+
if shapes is None:
|
|
292
|
+
return ["export unreadable"]
|
|
293
|
+
out: list[str] = []
|
|
294
|
+
for dataset, want in self._pair():
|
|
295
|
+
name = str(dataset.get("dataSetName", ""))
|
|
296
|
+
out.extend(f"{name} has {d}" for d in _differences(want, shapes.get(name)))
|
|
297
|
+
return out
|
|
298
|
+
|
|
299
|
+
def export_seen(self, state: tuple[str, dict[str, _Shape] | None] | None) -> bool:
|
|
300
|
+
"""Step 2: True once the export matches the ZIP and then stopped changing.
|
|
301
|
+
|
|
302
|
+
The match alone is not enough (the backend still runs its SQL rule check
|
|
303
|
+
afterwards, which edits rules), so it must also hold for `quiet` seconds. An
|
|
304
|
+
export that stays unchanged for `settle` seconds while it still lacks
|
|
305
|
+
content means the import stalled or failed: that raises.
|
|
306
|
+
"""
|
|
307
|
+
now = _monotonic()
|
|
308
|
+
digest, shapes = state if state is not None else (None, None)
|
|
309
|
+
if digest is None or digest != self._digest:
|
|
310
|
+
self._digest, self._stable_since = digest, now
|
|
311
|
+
missing = self.mismatches(shapes) if digest is not None else ["no export yet"]
|
|
312
|
+
unchanged = now - self._stable_since
|
|
313
|
+
if not missing and unchanged >= self.quiet:
|
|
314
|
+
return True
|
|
315
|
+
if missing and digest is not None and unchanged >= self.settle:
|
|
316
|
+
raise SchemaImportError(
|
|
317
|
+
f"schema import into dataflow {self.dataflow_id} seems stalled or failed: "
|
|
318
|
+
f"the export has not changed for {unchanged:g}s and still lacks content "
|
|
319
|
+
f"({'; '.join(missing[:6])})"
|
|
320
|
+
)
|
|
321
|
+
self._check_deadline(
|
|
322
|
+
now,
|
|
323
|
+
"export does not match the uploaded ZIP yet (" + "; ".join(missing[:6]) + ")"
|
|
324
|
+
if missing
|
|
325
|
+
else "export still changing",
|
|
326
|
+
)
|
|
327
|
+
return False
|
|
328
|
+
|
|
329
|
+
def result(self, dataflow: dict[str, Any] | None) -> list[dict[str, Any]]:
|
|
330
|
+
return self._imported(dataflow) or self.new
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
_T = TypeVar("_T")
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _poll(
|
|
337
|
+
wait: _ImportWait, call: Callable[[], _T], retry: tuple[type[Exception], ...]
|
|
338
|
+
) -> _T | None:
|
|
339
|
+
"""`call()`, or None after an error worth polling again (kept for the timeout message)."""
|
|
340
|
+
try:
|
|
341
|
+
result = call()
|
|
342
|
+
except retry as error:
|
|
343
|
+
wait.retrying(error)
|
|
344
|
+
return None
|
|
345
|
+
except (ReportnetAPIError, httpx.HTTPError) as error:
|
|
346
|
+
raise wait.interrupted(error) from error
|
|
347
|
+
wait.answered()
|
|
348
|
+
return result
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
async def _async_poll(
|
|
352
|
+
wait: _ImportWait, call: Callable[[], Awaitable[_T]], retry: tuple[type[Exception], ...]
|
|
353
|
+
) -> _T | None:
|
|
354
|
+
"""Async counterpart of `_poll`."""
|
|
355
|
+
try:
|
|
356
|
+
result = await call()
|
|
357
|
+
except retry as error:
|
|
358
|
+
wait.retrying(error)
|
|
359
|
+
return None
|
|
360
|
+
except (ReportnetAPIError, httpx.HTTPError) as error:
|
|
361
|
+
raise wait.interrupted(error) from error
|
|
362
|
+
wait.answered()
|
|
363
|
+
return result
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
_IMPORT_AND_WAIT_DOC = """Import a schema ZIP into a dataflow and wait until it looks done.
|
|
367
|
+
|
|
368
|
+
Returns the design datasets the import created: the `designDatasets`
|
|
369
|
+
entries of `dataflow_datasets()` that are new and named
|
|
370
|
+
`IMPORTED_<name>`, sorted by id. Raises `ValueError` if the ZIP holds
|
|
371
|
+
no `.schema` file, and `SchemaImportError` (a `JobError`) if the
|
|
372
|
+
dataflow details can't be read first, if the import stalls, or after
|
|
373
|
+
`timeout` seconds.
|
|
374
|
+
|
|
375
|
+
The backend gives no completion signal through the API (its
|
|
376
|
+
`IMPORT_DATASET_SCHEMA_COMPLETED_EVENT` / `_FAILED_EVENT` only reach
|
|
377
|
+
the web app's websocket, and the import lock is only on an internal
|
|
378
|
+
route), so this is a heuristic in two steps:
|
|
379
|
+
|
|
380
|
+
1. until the dataflow has as many new `IMPORTED_` design datasets as
|
|
381
|
+
the ZIP has `.schema` files (the import creates them first, ~4 s
|
|
382
|
+
apart);
|
|
383
|
+
2. until the dataflow's schema export (`export_dataflow_schemas_zip`)
|
|
384
|
+
matches the uploaded ZIP and has stopped changing. The import fills
|
|
385
|
+
the datasets one after another (tables ~2 s each, a dataset's
|
|
386
|
+
schema document only after its last table), then the unique
|
|
387
|
+
constraints, QC and integrity rules, external integrations and the
|
|
388
|
+
SQL rule check, so a quiet export is not a finished one. For each
|
|
389
|
+
new `IMPORTED_<name>` dataset (paired with `<name>` in the ZIP) the
|
|
390
|
+
export must have the same tables, the same number of fields per
|
|
391
|
+
table, and the same number of unique constraints, QC rules,
|
|
392
|
+
integrity rules and external integrations (counts only: the SQL
|
|
393
|
+
rule check disables and un-verifies rules that fail it, so their
|
|
394
|
+
`enabled`/`verified` flags are not compared). Once it matches, it
|
|
395
|
+
must also stay unchanged for two polls. A 5xx, a timeout or a broken
|
|
396
|
+
ZIP from the export counts as "not done yet".
|
|
397
|
+
|
|
398
|
+
If the export stays unchanged for `settle` seconds while it still lacks
|
|
399
|
+
content, the import has stalled or failed and `SchemaImportError`
|
|
400
|
+
says what is missing (for example "IMPORTED_X has 0/14 tables").
|
|
401
|
+
|
|
402
|
+
A 5xx or a transient transport error (timeout, dropped connection)
|
|
403
|
+
while polling does not end the wait: the import carries on server side,
|
|
404
|
+
and a retry would import a second set. The timeout's `SchemaImportError`
|
|
405
|
+
names the last such error and is chained to it. Any other error while
|
|
406
|
+
waiting (e.g. `401` once a login token expires) raises a chained
|
|
407
|
+
`SchemaImportError` saying the import may still be running: check the
|
|
408
|
+
dataflow's design datasets before retrying. `timeout` counts from the
|
|
409
|
+
end of the upload.
|
|
410
|
+
|
|
411
|
+
Limits: the backend reports completion only on the web app's
|
|
412
|
+
websocket, so this is still a heuristic. An item the import legitimately
|
|
413
|
+
leaves out (a rule it cannot recreate, an integration it cannot
|
|
414
|
+
create) shows up as a stall error naming the count, not as a silent
|
|
415
|
+
success. An import that fails before creating datasets (invalid names
|
|
416
|
+
in the ZIP, ...) ends in the timeout. `ValueError` if the ZIP holds no
|
|
417
|
+
readable `.schema` file. Same roles as `import_dataflow_schemas_zip()`.
|
|
418
|
+
Polls every `poll_interval` seconds.
|
|
419
|
+
"""
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
class DatasetSchemaResource(_GeneratedDatasetSchemaResource):
|
|
423
|
+
def export_dataflow_schemas_zip(
|
|
424
|
+
self, dataflow_id: int, *, dest: str | os.PathLike[str]
|
|
425
|
+
) -> None:
|
|
426
|
+
"""GET /dataschema/export?dataflowId=...
|
|
427
|
+
|
|
428
|
+
Downloads the full dataflow schema export ZIP: one `.schema`/
|
|
429
|
+
`.qcrules`/`.unique`/`.integrity`/`.extintegrations` file set per
|
|
430
|
+
dataset schema in the dataflow, plus the global
|
|
431
|
+
`datasetSchemaNames.names` and `datasetSchemaIds.ids` files.
|
|
432
|
+
|
|
433
|
+
`DatasetSchemaControllerImpl#exportSchemas` (controller method
|
|
434
|
+
mapped to this route) is annotated `@ApiOperation(hidden = true)`,
|
|
435
|
+
so it never reaches spec/api-docs.json and can't come from the
|
|
436
|
+
generator — this is a permanent hand-written addition, not a stopgap.
|
|
437
|
+
Requires `DATAFLOW_CUSTODIAN` or `DATAFLOW_STEWARD` on the dataflow.
|
|
438
|
+
|
|
439
|
+
The `Accept: */*` override is required: `RequestCore.build_headers()`'s
|
|
440
|
+
default `Accept: application/json` gets rejected before auth is even
|
|
441
|
+
checked — confirmed against the live production API, this specific
|
|
442
|
+
route 406s behind what looks like a gateway/proxy layer in front of
|
|
443
|
+
the actual controller. Oddly, `Accept: application/octet-stream`
|
|
444
|
+
(matching the controller's declared `produces`) 406s too with an
|
|
445
|
+
empty body, while `Accept: */*` passes through cleanly to the real
|
|
446
|
+
auth/role check.
|
|
447
|
+
"""
|
|
448
|
+
self._client.download(
|
|
449
|
+
"GET",
|
|
450
|
+
"/dataschema/export",
|
|
451
|
+
params={"dataflowId": dataflow_id},
|
|
452
|
+
headers=dict(ACCEPT_ANY),
|
|
453
|
+
dest=dest,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
def import_dataflow_schemas_zip(self, dataflow_id: int, file: str | os.PathLike[str]) -> None:
|
|
457
|
+
"""POST /dataschema/import?dataflowId=...: add the dataset schemas of an
|
|
458
|
+
export ZIP (`export_dataflow_schemas_zip()`'s file set) to a dataflow.
|
|
459
|
+
|
|
460
|
+
`DatasetSchemaControllerImpl#importSchemas` (hidden), multipart `file`.
|
|
461
|
+
Requires `DATAFLOW_CUSTODIAN` or `DATAFLOW_STEWARD` on the dataflow, and
|
|
462
|
+
the dataflow in `DESIGN` status (`403` "Invalid dataflow status"
|
|
463
|
+
otherwise). The file name must end in `.zip` (checked here, raises
|
|
464
|
+
`ValueError`): the backend ignores anything else.
|
|
465
|
+
|
|
466
|
+
Asynchronous: answers 200 once the work is started. The import
|
|
467
|
+
(`DataschemaServiceImpl#importSchemas`) adds one new design dataset
|
|
468
|
+
per schema, named `IMPORTED_<name>` (`IMPORTED_<name> (1)`, ... when
|
|
469
|
+
taken), never replaces existing ones, gives every schema, table,
|
|
470
|
+
record and field a new id (remapping links, unique constraints,
|
|
471
|
+
integrity checks, QC rules and external integrations to them), and
|
|
472
|
+
then runs the SQL QC rule check. A second import into the same
|
|
473
|
+
dataflow while one runs answers `423`. Use
|
|
474
|
+
`import_dataflow_schemas_zip_and_wait()` to wait for it.
|
|
475
|
+
|
|
476
|
+
Not retried: if the connection times out or drops after the upload was
|
|
477
|
+
sent, this raises `SchemaImportError` (chained to the transport error),
|
|
478
|
+
because the import may have started anyway; check the dataflow's design
|
|
479
|
+
datasets before trying again.
|
|
480
|
+
"""
|
|
481
|
+
_check_zip_name(file)
|
|
482
|
+
try:
|
|
483
|
+
with stream_upload_file("file", file) as files:
|
|
484
|
+
self._client.request(
|
|
485
|
+
"POST", _IMPORT_PATH, params={"dataflowId": dataflow_id}, files=files
|
|
486
|
+
)
|
|
487
|
+
except httpx.HTTPError as error:
|
|
488
|
+
raise _upload_failed(dataflow_id, error) from error
|
|
489
|
+
|
|
490
|
+
def import_dataflow_schemas_zip_and_wait(
|
|
491
|
+
self,
|
|
492
|
+
dataflow_id: int,
|
|
493
|
+
file: str | os.PathLike[str],
|
|
494
|
+
*,
|
|
495
|
+
poll_interval: float = 5.0,
|
|
496
|
+
settle: float = 120.0,
|
|
497
|
+
timeout: float = 900.0,
|
|
498
|
+
) -> list[dict[str, Any]]:
|
|
499
|
+
source = _source_shapes(file)
|
|
500
|
+
dataflow = self._client.dataflow
|
|
501
|
+
before = _design_ids(dataflow_id, dataflow.dataflow_datasets(dataflow_id))
|
|
502
|
+
self.import_dataflow_schemas_zip(dataflow_id, file)
|
|
503
|
+
# Started after the upload: `timeout` bounds the waiting, not the upload.
|
|
504
|
+
wait = _ImportWait(dataflow_id, source, before, settle, timeout, quiet=2 * poll_interval)
|
|
505
|
+
|
|
506
|
+
def datasets() -> dict[str, Any] | None:
|
|
507
|
+
return _poll(wait, lambda: dataflow.dataflow_datasets(dataflow_id), _POLL_RETRY_ERRORS)
|
|
508
|
+
|
|
509
|
+
while True:
|
|
510
|
+
_sleep(poll_interval)
|
|
511
|
+
if wait.datasets_seen(datasets()):
|
|
512
|
+
break
|
|
513
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
514
|
+
dest = Path(tmp) / "export.zip"
|
|
515
|
+
|
|
516
|
+
def export_state() -> tuple[str, dict[str, _Shape] | None]:
|
|
517
|
+
self.export_dataflow_schemas_zip(dataflow_id, dest=dest)
|
|
518
|
+
return _zip_digest(dest), _zip_shapes(dest, strict=False)
|
|
519
|
+
|
|
520
|
+
while not wait.export_seen(_poll(wait, export_state, _EXPORT_RETRY_ERRORS)):
|
|
521
|
+
_sleep(poll_interval)
|
|
522
|
+
return wait.result(datasets())
|
|
523
|
+
|
|
524
|
+
import_dataflow_schemas_zip_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
class AsyncDatasetSchemaResource(_GeneratedAsyncDatasetSchemaResource):
|
|
528
|
+
async def export_dataflow_schemas_zip(
|
|
529
|
+
self, dataflow_id: int, *, dest: str | os.PathLike[str]
|
|
530
|
+
) -> None:
|
|
531
|
+
"""Async counterpart of DatasetSchemaResource.export_dataflow_schemas_zip."""
|
|
532
|
+
await self._client.download(
|
|
533
|
+
"GET",
|
|
534
|
+
"/dataschema/export",
|
|
535
|
+
params={"dataflowId": dataflow_id},
|
|
536
|
+
headers=dict(ACCEPT_ANY),
|
|
537
|
+
dest=dest,
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
async def import_dataflow_schemas_zip(
|
|
541
|
+
self, dataflow_id: int, file: str | os.PathLike[str]
|
|
542
|
+
) -> None:
|
|
543
|
+
"""Async counterpart of DatasetSchemaResource.import_dataflow_schemas_zip."""
|
|
544
|
+
_check_zip_name(file)
|
|
545
|
+
try:
|
|
546
|
+
with stream_upload_file("file", file) as files:
|
|
547
|
+
await self._client.request(
|
|
548
|
+
"POST", _IMPORT_PATH, params={"dataflowId": dataflow_id}, files=files
|
|
549
|
+
)
|
|
550
|
+
except httpx.HTTPError as error:
|
|
551
|
+
raise _upload_failed(dataflow_id, error) from error
|
|
552
|
+
|
|
553
|
+
async def import_dataflow_schemas_zip_and_wait(
|
|
554
|
+
self,
|
|
555
|
+
dataflow_id: int,
|
|
556
|
+
file: str | os.PathLike[str],
|
|
557
|
+
*,
|
|
558
|
+
poll_interval: float = 5.0,
|
|
559
|
+
settle: float = 120.0,
|
|
560
|
+
timeout: float = 900.0,
|
|
561
|
+
) -> list[dict[str, Any]]:
|
|
562
|
+
source = _source_shapes(file)
|
|
563
|
+
dataflow = self._client.dataflow
|
|
564
|
+
before = _design_ids(dataflow_id, await dataflow.dataflow_datasets(dataflow_id))
|
|
565
|
+
await self.import_dataflow_schemas_zip(dataflow_id, file)
|
|
566
|
+
# Started after the upload: `timeout` bounds the waiting, not the upload.
|
|
567
|
+
wait = _ImportWait(dataflow_id, source, before, settle, timeout, quiet=2 * poll_interval)
|
|
568
|
+
|
|
569
|
+
async def datasets() -> dict[str, Any] | None:
|
|
570
|
+
return await _async_poll(
|
|
571
|
+
wait, lambda: dataflow.dataflow_datasets(dataflow_id), _POLL_RETRY_ERRORS
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
while True:
|
|
575
|
+
await _async_sleep(poll_interval)
|
|
576
|
+
if wait.datasets_seen(await datasets()):
|
|
577
|
+
break
|
|
578
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
579
|
+
dest = Path(tmp) / "export.zip"
|
|
580
|
+
|
|
581
|
+
async def export_state() -> tuple[str, dict[str, _Shape] | None]:
|
|
582
|
+
await self.export_dataflow_schemas_zip(dataflow_id, dest=dest)
|
|
583
|
+
return _zip_digest(dest), _zip_shapes(dest, strict=False)
|
|
584
|
+
|
|
585
|
+
while not wait.export_seen(await _async_poll(wait, export_state, _EXPORT_RETRY_ERRORS)):
|
|
586
|
+
await _async_sleep(poll_interval)
|
|
587
|
+
return wait.result(await datasets())
|
|
588
|
+
|
|
589
|
+
import_dataflow_schemas_zip_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
|