reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
"""Hand-written Validation resource (run validation, read and export its results).
|
|
2
|
+
|
|
3
|
+
The Validation tag is absent from spec/api-docs.json ("Known spec-
|
|
4
|
+
snapshot gap") and every route here is `@ApiOperation(hidden = true)` in
|
|
5
|
+
`ValidationControllerImpl` (`validation-service`) / `JobControllerImpl`
|
|
6
|
+
(`orchestrator-service`). Paths are the ones the Reportnet UI calls. This file
|
|
7
|
+
is never touched by the generators.
|
|
8
|
+
|
|
9
|
+
Not covered: `PUT /validation/dataset/{id}` (the older direct start the job route
|
|
10
|
+
replaced) and the admin-only task routes. A "DL" (data-lake) variant of the
|
|
11
|
+
group listing was suggested by a third-party survey but exists in neither the
|
|
12
|
+
`develop` nor the `MasterDataLakes` branch of the Reportnet 3 backend repository, so it is not
|
|
13
|
+
implemented. All of this is mocked-test only: nothing has been run live.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import os
|
|
20
|
+
import time
|
|
21
|
+
import uuid
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import TYPE_CHECKING, Any
|
|
24
|
+
|
|
25
|
+
from .._exceptions import NotFoundError
|
|
26
|
+
from ..jobs import JobError
|
|
27
|
+
from .dataflow_ext import _json_or_none
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from ..async_client import AsyncReportnetClient
|
|
31
|
+
from ..client import ReportnetClient
|
|
32
|
+
|
|
33
|
+
_LEVELS = ("CORRECT", "WARNING", "ERROR", "INFO", "BLOCKER")
|
|
34
|
+
_ENTITIES = ("TABLE", "DATASET", "FIELD", "RECORD")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _file_name(dataset_id: int) -> str:
|
|
38
|
+
# ValidationServiceImpl#exportValidationFile composes exactly this name.
|
|
39
|
+
return f"dataset-{dataset_id}-validations.csv"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _check(values: list[str] | None, allowed: tuple[str, ...], what: str) -> list[str] | None:
|
|
43
|
+
if values is None:
|
|
44
|
+
return None
|
|
45
|
+
bad = [v for v in values if v not in allowed]
|
|
46
|
+
if bad:
|
|
47
|
+
raise ValueError(f"{what} must be from {list(allowed)}, got {bad}")
|
|
48
|
+
return list(values)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _list_params(
|
|
52
|
+
page_num: int,
|
|
53
|
+
page_size: int,
|
|
54
|
+
sort_by: str | None,
|
|
55
|
+
asc: bool,
|
|
56
|
+
level_errors: list[str] | None,
|
|
57
|
+
type_entities: list[str] | None,
|
|
58
|
+
table_filter: str | None,
|
|
59
|
+
field_value_filter: str | None,
|
|
60
|
+
) -> dict[str, Any]:
|
|
61
|
+
if page_num < 0 or page_size < 1:
|
|
62
|
+
raise ValueError("page_num must be >= 0 and page_size >= 1")
|
|
63
|
+
params: dict[str, Any] = {
|
|
64
|
+
"pageNum": page_num,
|
|
65
|
+
"pageSize": page_size,
|
|
66
|
+
"headers": sort_by,
|
|
67
|
+
"asc": str(asc).lower(),
|
|
68
|
+
"levelErrorsFilter": _check(level_errors, _LEVELS, "level_errors"),
|
|
69
|
+
"typeEntitiesFilter": _check(type_entities, _ENTITIES, "type_entities"),
|
|
70
|
+
"tableFilter": table_filter,
|
|
71
|
+
"fieldValueFilter": field_value_filter,
|
|
72
|
+
}
|
|
73
|
+
return {k: v for k, v in params.items() if v is not None}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ValidationResource:
|
|
77
|
+
def __init__(self, client: ReportnetClient) -> None:
|
|
78
|
+
self._client = client
|
|
79
|
+
|
|
80
|
+
def start_validation(self, dataset_id: int, *, released: bool = False) -> None:
|
|
81
|
+
"""PUT /orchestrator/jobs/addValidationJob/{datasetId}: queue a validation.
|
|
82
|
+
|
|
83
|
+
The route the Reportnet UI's "Validate" button calls. It only queues an
|
|
84
|
+
Orchestrator job and returns None (live, 2026-11-06: the body was a bare
|
|
85
|
+
number, `44570`, not understood and not used here). No status is exposed on
|
|
86
|
+
a data-lake dataset's listing entry; completion shows as a
|
|
87
|
+
`VALIDATION_FINISHED_EVENT` user notification
|
|
88
|
+
(`GET /notification/findUserNotifications`), which `easy.validate_and_wait()`
|
|
89
|
+
polls. Read the results with `list_group_validations()` (or `_dl()`). This WRITES: it replaces the dataset's
|
|
90
|
+
existing validation results and takes a dataset lock (a second start while
|
|
91
|
+
one runs may get `423`). Roles: stewards, custodians, lead reporter,
|
|
92
|
+
reporter write and similar (see `JobControllerImpl`), or ADMIN. `released`
|
|
93
|
+
is for validating a dataset that is being released.
|
|
94
|
+
"""
|
|
95
|
+
self._client.request(
|
|
96
|
+
"PUT",
|
|
97
|
+
f"/orchestrator/jobs/addValidationJob/{dataset_id}",
|
|
98
|
+
params={"released": "true"} if released else None,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def list_validations(
|
|
102
|
+
self,
|
|
103
|
+
dataset_id: int,
|
|
104
|
+
*,
|
|
105
|
+
page_num: int = 0,
|
|
106
|
+
page_size: int = 20,
|
|
107
|
+
sort_by: str | None = None,
|
|
108
|
+
asc: bool = True,
|
|
109
|
+
level_errors: list[str] | None = None,
|
|
110
|
+
type_entities: list[str] | None = None,
|
|
111
|
+
table_filter: str | None = None,
|
|
112
|
+
field_value_filter: str | None = None,
|
|
113
|
+
) -> dict[str, Any] | None:
|
|
114
|
+
"""GET /validation/listValidations/{datasetId}: failed validations, one per failure.
|
|
115
|
+
|
|
116
|
+
Returns the backend's `FailedValidationsDatasetVO`: `idDataset`,
|
|
117
|
+
`idDatasetSchema`, `nameDataSetSchema`, `errors` (a list), `totalRecords`,
|
|
118
|
+
`totalErrors` and `totalFilteredRecords`. Each error is one failure: `idValidation`, `idObject`, `idTableSchema`,
|
|
119
|
+
`nameTableSchema`, `message`, `levelError`, `typeEntity`, `validationDate`,
|
|
120
|
+
`nameFieldSchema`, `shortCode`.
|
|
121
|
+
|
|
122
|
+
`page_num` is a 0-based page number; `page_size` defaults to 20 on the
|
|
123
|
+
backend. `sort_by` is a column name (the backend maps `tableSchemaName`,
|
|
124
|
+
`fieldSchemaName` and `entityType` to its own columns). `level_errors`
|
|
125
|
+
takes `CORRECT`/`WARNING`/`ERROR`/`INFO`/`BLOCKER`, `type_entities`
|
|
126
|
+
`TABLE`/`DATASET`/`FIELD`/`RECORD`. The backend swallows its own
|
|
127
|
+
`EEAException` and answers an empty 200, so this can return `None`.
|
|
128
|
+
Roles: any dataset role incl. observer/requester, custodians, stewards.
|
|
129
|
+
For a data-lake dataflow (`bigData: true`) this answers empty even when the
|
|
130
|
+
UI shows failures; there is no `DL` variant of this route (see
|
|
131
|
+
`list_group_validations_dl()`).
|
|
132
|
+
"""
|
|
133
|
+
params = _list_params(
|
|
134
|
+
page_num,
|
|
135
|
+
page_size,
|
|
136
|
+
sort_by,
|
|
137
|
+
asc,
|
|
138
|
+
level_errors,
|
|
139
|
+
type_entities,
|
|
140
|
+
table_filter,
|
|
141
|
+
field_value_filter,
|
|
142
|
+
)
|
|
143
|
+
response = self._client.request(
|
|
144
|
+
"GET", f"/validation/listValidations/{dataset_id}", params=params
|
|
145
|
+
)
|
|
146
|
+
return _json_or_none(response)
|
|
147
|
+
|
|
148
|
+
def list_group_validations(
|
|
149
|
+
self,
|
|
150
|
+
dataset_id: int,
|
|
151
|
+
*,
|
|
152
|
+
page_num: int = 0,
|
|
153
|
+
page_size: int = 20,
|
|
154
|
+
sort_by: str | None = None,
|
|
155
|
+
asc: bool = True,
|
|
156
|
+
level_errors: list[str] | None = None,
|
|
157
|
+
type_entities: list[str] | None = None,
|
|
158
|
+
table_filter: str | None = None,
|
|
159
|
+
field_value_filter: str | None = None,
|
|
160
|
+
) -> dict[str, Any] | None:
|
|
161
|
+
"""GET /validation/listGroupValidations/{datasetId}: failed validations grouped by
|
|
162
|
+
rule (what the UI shows).
|
|
163
|
+
|
|
164
|
+
Returns the backend's `FailedValidationsDatasetVO`: `idDataset`,
|
|
165
|
+
`idDatasetSchema`, `nameDataSetSchema`, `errors` (a list), `totalRecords`,
|
|
166
|
+
`totalErrors` and `totalFilteredRecords`. Each error is one rule's failures grouped by code: `message`, `idRule`,
|
|
167
|
+
`levelError`, `typeEntity`, `numberOfRecords`, `nameTableSchema`, `shortCode`,
|
|
168
|
+
`nameFieldSchema` (what the UI's validation table shows).
|
|
169
|
+
|
|
170
|
+
`page_num` is a 0-based page number; `page_size` defaults to 20 on the
|
|
171
|
+
backend. `sort_by` is a column name (the backend maps `tableSchemaName`,
|
|
172
|
+
`fieldSchemaName` and `entityType` to its own columns). `level_errors`
|
|
173
|
+
takes `CORRECT`/`WARNING`/`ERROR`/`INFO`/`BLOCKER`, `type_entities`
|
|
174
|
+
`TABLE`/`DATASET`/`FIELD`/`RECORD`. The backend swallows its own
|
|
175
|
+
`EEAException` and answers an empty 200, so this can return `None`.
|
|
176
|
+
Backend rule `checkAccessSuperUser('DATASET', id)`, not the role list of
|
|
177
|
+
`list_validations`. For a data-lake dataflow (`bigData: true`) use
|
|
178
|
+
`list_group_validations_dl()`: this one answers `errors: []` and
|
|
179
|
+
`totalRecords: 0` there even when the UI shows failures.
|
|
180
|
+
"""
|
|
181
|
+
params = _list_params(
|
|
182
|
+
page_num,
|
|
183
|
+
page_size,
|
|
184
|
+
sort_by,
|
|
185
|
+
asc,
|
|
186
|
+
level_errors,
|
|
187
|
+
type_entities,
|
|
188
|
+
table_filter,
|
|
189
|
+
field_value_filter,
|
|
190
|
+
)
|
|
191
|
+
response = self._client.request(
|
|
192
|
+
"GET", f"/validation/listGroupValidations/{dataset_id}", params=params
|
|
193
|
+
)
|
|
194
|
+
return _json_or_none(response)
|
|
195
|
+
|
|
196
|
+
def list_group_validations_dl(
|
|
197
|
+
self,
|
|
198
|
+
dataset_id: int,
|
|
199
|
+
*,
|
|
200
|
+
page_num: int = 0,
|
|
201
|
+
page_size: int = 20,
|
|
202
|
+
sort_by: str | None = None,
|
|
203
|
+
asc: bool = True,
|
|
204
|
+
level_errors: list[str] | None = None,
|
|
205
|
+
type_entities: list[str] | None = None,
|
|
206
|
+
table_filter: str | None = None,
|
|
207
|
+
field_value_filter: str | None = None,
|
|
208
|
+
) -> dict[str, Any] | None:
|
|
209
|
+
"""GET /validation/listGroupValidationsDL/{datasetId}: grouped failures, data-lake datasets.
|
|
210
|
+
|
|
211
|
+
For a dataflow whose `dataflow_datasets()` answer has `bigData: true`
|
|
212
|
+
(data lake), validation results live in the data lake and
|
|
213
|
+
`list_group_validations()` reads the empty old per-dataset database, so it
|
|
214
|
+
answers `errors: []`, `totalRecords: 0`. This is the route the Reportnet UI
|
|
215
|
+
calls for those datasets. Same arguments and answer shape as
|
|
216
|
+
`list_group_validations()`. Live-tested on the sandbox (2026-10-27): it
|
|
217
|
+
returned the same two rule groups and totals the UI shows. The backend source
|
|
218
|
+
(`MasterDataLakes`) has no `DL` variant of `list_validations()` (one entry per
|
|
219
|
+
failing record), so that one stays empty for data-lake datasets. The UI also
|
|
220
|
+
sends a `shortCode` filter, not wrapped here.
|
|
221
|
+
"""
|
|
222
|
+
params = _list_params(
|
|
223
|
+
page_num,
|
|
224
|
+
page_size,
|
|
225
|
+
sort_by,
|
|
226
|
+
asc,
|
|
227
|
+
level_errors,
|
|
228
|
+
type_entities,
|
|
229
|
+
table_filter,
|
|
230
|
+
field_value_filter,
|
|
231
|
+
)
|
|
232
|
+
response = self._client.request(
|
|
233
|
+
"GET", f"/validation/listGroupValidationsDL/{dataset_id}", params=params
|
|
234
|
+
)
|
|
235
|
+
return _json_or_none(response)
|
|
236
|
+
|
|
237
|
+
def start_validations_export(self, dataset_id: int) -> None:
|
|
238
|
+
"""POST /validation/export/{datasetId}: generate the validations CSV.
|
|
239
|
+
|
|
240
|
+
Not a job: the backend writes `dataset-{id}-validations.csv` on the server
|
|
241
|
+
and only notifies the Reportnet UI, so there is no id or status to poll.
|
|
242
|
+
Fetch it with `download_validations_file()`, or use
|
|
243
|
+
`export_validations_to_file()` which does both. Leaves a notification in the
|
|
244
|
+
user's Reportnet UI.
|
|
245
|
+
"""
|
|
246
|
+
self._client.request("POST", f"/validation/export/{dataset_id}")
|
|
247
|
+
|
|
248
|
+
def download_validations_file(self, dataset_id: int, dest: str | os.PathLike[str]) -> bool:
|
|
249
|
+
"""GET /validation/downloadFile/{datasetId}: fetch the generated CSV to `dest`.
|
|
250
|
+
|
|
251
|
+
Returns False (writing nothing) while the file does not exist yet (`404`),
|
|
252
|
+
True once it was saved. The backend deletes the file after serving it, so a
|
|
253
|
+
second call returns False again.
|
|
254
|
+
"""
|
|
255
|
+
part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
|
|
256
|
+
try:
|
|
257
|
+
self._client.download(
|
|
258
|
+
"GET",
|
|
259
|
+
f"/validation/downloadFile/{dataset_id}",
|
|
260
|
+
params={"fileName": _file_name(dataset_id)},
|
|
261
|
+
dest=part,
|
|
262
|
+
)
|
|
263
|
+
if not part.exists() or part.stat().st_size == 0:
|
|
264
|
+
return False
|
|
265
|
+
part.replace(dest)
|
|
266
|
+
return True
|
|
267
|
+
except NotFoundError:
|
|
268
|
+
return False
|
|
269
|
+
finally:
|
|
270
|
+
part.unlink(missing_ok=True)
|
|
271
|
+
|
|
272
|
+
def export_validations_to_file(
|
|
273
|
+
self,
|
|
274
|
+
dataset_id: int,
|
|
275
|
+
dest: str | os.PathLike[str],
|
|
276
|
+
*,
|
|
277
|
+
poll_interval: float = 2.0,
|
|
278
|
+
timeout: float = 300.0,
|
|
279
|
+
) -> None:
|
|
280
|
+
"""Generate the validations CSV and wait for it, saving it at `dest`.
|
|
281
|
+
|
|
282
|
+
Calls `start_validations_export()` then polls `download_validations_file()`
|
|
283
|
+
every `poll_interval` seconds; raises `JobError` after `timeout` seconds.
|
|
284
|
+
Columns (from `ValidationServiceImpl`): entity, table, field, code, code
|
|
285
|
+
name, code description, level error, message, number of records. A
|
|
286
|
+
leftover file with the same name from an abandoned export could be served
|
|
287
|
+
instead of a fresh one. The server path has no per-user part and the file
|
|
288
|
+
is deleted when served, so two simultaneous exports of one dataset can
|
|
289
|
+
take each other's file, and a dropped download loses it (export again).
|
|
290
|
+
A failed export is not reported (the backend still answers 200), so it
|
|
291
|
+
ends as a timeout.
|
|
292
|
+
"""
|
|
293
|
+
self.start_validations_export(dataset_id)
|
|
294
|
+
deadline = time.monotonic() + timeout
|
|
295
|
+
while not self.download_validations_file(dataset_id, dest):
|
|
296
|
+
remaining = deadline - time.monotonic()
|
|
297
|
+
if remaining <= 0:
|
|
298
|
+
raise JobError(
|
|
299
|
+
f"Validations file not ready within {timeout:g} seconds (the backend "
|
|
300
|
+
"answers 200 even when its export fails, so a failed export looks like this)"
|
|
301
|
+
)
|
|
302
|
+
time.sleep(min(poll_interval, remaining))
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class AsyncValidationResource:
|
|
306
|
+
def __init__(self, client: AsyncReportnetClient) -> None:
|
|
307
|
+
self._client = client
|
|
308
|
+
|
|
309
|
+
async def start_validation(self, dataset_id: int, *, released: bool = False) -> None:
|
|
310
|
+
"""Async counterpart of ValidationResource.start_validation."""
|
|
311
|
+
await self._client.request(
|
|
312
|
+
"PUT",
|
|
313
|
+
f"/orchestrator/jobs/addValidationJob/{dataset_id}",
|
|
314
|
+
params={"released": "true"} if released else None,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
async def list_validations(
|
|
318
|
+
self,
|
|
319
|
+
dataset_id: int,
|
|
320
|
+
*,
|
|
321
|
+
page_num: int = 0,
|
|
322
|
+
page_size: int = 20,
|
|
323
|
+
sort_by: str | None = None,
|
|
324
|
+
asc: bool = True,
|
|
325
|
+
level_errors: list[str] | None = None,
|
|
326
|
+
type_entities: list[str] | None = None,
|
|
327
|
+
table_filter: str | None = None,
|
|
328
|
+
field_value_filter: str | None = None,
|
|
329
|
+
) -> dict[str, Any] | None:
|
|
330
|
+
"""Async counterpart of ValidationResource.list_validations."""
|
|
331
|
+
params = _list_params(
|
|
332
|
+
page_num,
|
|
333
|
+
page_size,
|
|
334
|
+
sort_by,
|
|
335
|
+
asc,
|
|
336
|
+
level_errors,
|
|
337
|
+
type_entities,
|
|
338
|
+
table_filter,
|
|
339
|
+
field_value_filter,
|
|
340
|
+
)
|
|
341
|
+
response = await self._client.request(
|
|
342
|
+
"GET", f"/validation/listValidations/{dataset_id}", params=params
|
|
343
|
+
)
|
|
344
|
+
return _json_or_none(response)
|
|
345
|
+
|
|
346
|
+
async def list_group_validations(
|
|
347
|
+
self,
|
|
348
|
+
dataset_id: int,
|
|
349
|
+
*,
|
|
350
|
+
page_num: int = 0,
|
|
351
|
+
page_size: int = 20,
|
|
352
|
+
sort_by: str | None = None,
|
|
353
|
+
asc: bool = True,
|
|
354
|
+
level_errors: list[str] | None = None,
|
|
355
|
+
type_entities: list[str] | None = None,
|
|
356
|
+
table_filter: str | None = None,
|
|
357
|
+
field_value_filter: str | None = None,
|
|
358
|
+
) -> dict[str, Any] | None:
|
|
359
|
+
"""Async counterpart of ValidationResource.list_group_validations."""
|
|
360
|
+
params = _list_params(
|
|
361
|
+
page_num,
|
|
362
|
+
page_size,
|
|
363
|
+
sort_by,
|
|
364
|
+
asc,
|
|
365
|
+
level_errors,
|
|
366
|
+
type_entities,
|
|
367
|
+
table_filter,
|
|
368
|
+
field_value_filter,
|
|
369
|
+
)
|
|
370
|
+
response = await self._client.request(
|
|
371
|
+
"GET", f"/validation/listGroupValidations/{dataset_id}", params=params
|
|
372
|
+
)
|
|
373
|
+
return _json_or_none(response)
|
|
374
|
+
|
|
375
|
+
async def list_group_validations_dl(
|
|
376
|
+
self,
|
|
377
|
+
dataset_id: int,
|
|
378
|
+
*,
|
|
379
|
+
page_num: int = 0,
|
|
380
|
+
page_size: int = 20,
|
|
381
|
+
sort_by: str | None = None,
|
|
382
|
+
asc: bool = True,
|
|
383
|
+
level_errors: list[str] | None = None,
|
|
384
|
+
type_entities: list[str] | None = None,
|
|
385
|
+
table_filter: str | None = None,
|
|
386
|
+
field_value_filter: str | None = None,
|
|
387
|
+
) -> dict[str, Any] | None:
|
|
388
|
+
"""Async counterpart of ValidationResource.list_group_validations_dl."""
|
|
389
|
+
params = _list_params(
|
|
390
|
+
page_num,
|
|
391
|
+
page_size,
|
|
392
|
+
sort_by,
|
|
393
|
+
asc,
|
|
394
|
+
level_errors,
|
|
395
|
+
type_entities,
|
|
396
|
+
table_filter,
|
|
397
|
+
field_value_filter,
|
|
398
|
+
)
|
|
399
|
+
response = await self._client.request(
|
|
400
|
+
"GET", f"/validation/listGroupValidationsDL/{dataset_id}", params=params
|
|
401
|
+
)
|
|
402
|
+
return _json_or_none(response)
|
|
403
|
+
|
|
404
|
+
async def start_validations_export(self, dataset_id: int) -> None:
|
|
405
|
+
"""Async counterpart of ValidationResource.start_validations_export."""
|
|
406
|
+
await self._client.request("POST", f"/validation/export/{dataset_id}")
|
|
407
|
+
|
|
408
|
+
async def download_validations_file(
|
|
409
|
+
self, dataset_id: int, dest: str | os.PathLike[str]
|
|
410
|
+
) -> bool:
|
|
411
|
+
"""Async counterpart of ValidationResource.download_validations_file."""
|
|
412
|
+
part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
|
|
413
|
+
try:
|
|
414
|
+
await self._client.download(
|
|
415
|
+
"GET",
|
|
416
|
+
f"/validation/downloadFile/{dataset_id}",
|
|
417
|
+
params={"fileName": _file_name(dataset_id)},
|
|
418
|
+
dest=part,
|
|
419
|
+
)
|
|
420
|
+
if not part.exists() or part.stat().st_size == 0:
|
|
421
|
+
return False
|
|
422
|
+
part.replace(dest)
|
|
423
|
+
return True
|
|
424
|
+
except NotFoundError:
|
|
425
|
+
return False
|
|
426
|
+
finally:
|
|
427
|
+
part.unlink(missing_ok=True)
|
|
428
|
+
|
|
429
|
+
async def export_validations_to_file(
|
|
430
|
+
self,
|
|
431
|
+
dataset_id: int,
|
|
432
|
+
dest: str | os.PathLike[str],
|
|
433
|
+
*,
|
|
434
|
+
poll_interval: float = 2.0,
|
|
435
|
+
timeout: float = 300.0,
|
|
436
|
+
) -> None:
|
|
437
|
+
"""Async counterpart of ValidationResource.export_validations_to_file."""
|
|
438
|
+
await self.start_validations_export(dataset_id)
|
|
439
|
+
deadline = time.monotonic() + timeout
|
|
440
|
+
while not await self.download_validations_file(dataset_id, dest):
|
|
441
|
+
remaining = deadline - time.monotonic()
|
|
442
|
+
if remaining <= 0:
|
|
443
|
+
raise JobError(
|
|
444
|
+
f"Validations file not ready within {timeout:g} seconds (the backend "
|
|
445
|
+
"answers 200 even when its export fails, so a failed export looks like this)"
|
|
446
|
+
)
|
|
447
|
+
await asyncio.sleep(min(poll_interval, remaining))
|