reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,447 @@
1
+ """Hand-written Validation resource (run validation, read and export its results).
2
+
3
+ The Validation tag is absent from spec/api-docs.json ("Known spec-
4
+ snapshot gap") and every route here is `@ApiOperation(hidden = true)` in
5
+ `ValidationControllerImpl` (`validation-service`) / `JobControllerImpl`
6
+ (`orchestrator-service`). Paths are the ones the Reportnet UI calls. This file
7
+ is never touched by the generators.
8
+
9
+ Not covered: `PUT /validation/dataset/{id}` (the older direct start the job route
10
+ replaced) and the admin-only task routes. A "DL" (data-lake) variant of the
11
+ group listing was suggested by a third-party survey but exists in neither the
12
+ `develop` nor the `MasterDataLakes` branch of the Reportnet 3 backend repository, so it is not
13
+ implemented. All of this is mocked-test only: nothing has been run live.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import asyncio
19
+ import os
20
+ import time
21
+ import uuid
22
+ from pathlib import Path
23
+ from typing import TYPE_CHECKING, Any
24
+
25
+ from .._exceptions import NotFoundError
26
+ from ..jobs import JobError
27
+ from .dataflow_ext import _json_or_none
28
+
29
+ if TYPE_CHECKING:
30
+ from ..async_client import AsyncReportnetClient
31
+ from ..client import ReportnetClient
32
+
33
+ _LEVELS = ("CORRECT", "WARNING", "ERROR", "INFO", "BLOCKER")
34
+ _ENTITIES = ("TABLE", "DATASET", "FIELD", "RECORD")
35
+
36
+
37
+ def _file_name(dataset_id: int) -> str:
38
+ # ValidationServiceImpl#exportValidationFile composes exactly this name.
39
+ return f"dataset-{dataset_id}-validations.csv"
40
+
41
+
42
+ def _check(values: list[str] | None, allowed: tuple[str, ...], what: str) -> list[str] | None:
43
+ if values is None:
44
+ return None
45
+ bad = [v for v in values if v not in allowed]
46
+ if bad:
47
+ raise ValueError(f"{what} must be from {list(allowed)}, got {bad}")
48
+ return list(values)
49
+
50
+
51
+ def _list_params(
52
+ page_num: int,
53
+ page_size: int,
54
+ sort_by: str | None,
55
+ asc: bool,
56
+ level_errors: list[str] | None,
57
+ type_entities: list[str] | None,
58
+ table_filter: str | None,
59
+ field_value_filter: str | None,
60
+ ) -> dict[str, Any]:
61
+ if page_num < 0 or page_size < 1:
62
+ raise ValueError("page_num must be >= 0 and page_size >= 1")
63
+ params: dict[str, Any] = {
64
+ "pageNum": page_num,
65
+ "pageSize": page_size,
66
+ "headers": sort_by,
67
+ "asc": str(asc).lower(),
68
+ "levelErrorsFilter": _check(level_errors, _LEVELS, "level_errors"),
69
+ "typeEntitiesFilter": _check(type_entities, _ENTITIES, "type_entities"),
70
+ "tableFilter": table_filter,
71
+ "fieldValueFilter": field_value_filter,
72
+ }
73
+ return {k: v for k, v in params.items() if v is not None}
74
+
75
+
76
+ class ValidationResource:
77
+ def __init__(self, client: ReportnetClient) -> None:
78
+ self._client = client
79
+
80
+ def start_validation(self, dataset_id: int, *, released: bool = False) -> None:
81
+ """PUT /orchestrator/jobs/addValidationJob/{datasetId}: queue a validation.
82
+
83
+ The route the Reportnet UI's "Validate" button calls. It only queues an
84
+ Orchestrator job and returns None (live, 2026-11-06: the body was a bare
85
+ number, `44570`, not understood and not used here). No status is exposed on
86
+ a data-lake dataset's listing entry; completion shows as a
87
+ `VALIDATION_FINISHED_EVENT` user notification
88
+ (`GET /notification/findUserNotifications`), which `easy.validate_and_wait()`
89
+ polls. Read the results with `list_group_validations()` (or `_dl()`). This WRITES: it replaces the dataset's
90
+ existing validation results and takes a dataset lock (a second start while
91
+ one runs may get `423`). Roles: stewards, custodians, lead reporter,
92
+ reporter write and similar (see `JobControllerImpl`), or ADMIN. `released`
93
+ is for validating a dataset that is being released.
94
+ """
95
+ self._client.request(
96
+ "PUT",
97
+ f"/orchestrator/jobs/addValidationJob/{dataset_id}",
98
+ params={"released": "true"} if released else None,
99
+ )
100
+
101
+ def list_validations(
102
+ self,
103
+ dataset_id: int,
104
+ *,
105
+ page_num: int = 0,
106
+ page_size: int = 20,
107
+ sort_by: str | None = None,
108
+ asc: bool = True,
109
+ level_errors: list[str] | None = None,
110
+ type_entities: list[str] | None = None,
111
+ table_filter: str | None = None,
112
+ field_value_filter: str | None = None,
113
+ ) -> dict[str, Any] | None:
114
+ """GET /validation/listValidations/{datasetId}: failed validations, one per failure.
115
+
116
+ Returns the backend's `FailedValidationsDatasetVO`: `idDataset`,
117
+ `idDatasetSchema`, `nameDataSetSchema`, `errors` (a list), `totalRecords`,
118
+ `totalErrors` and `totalFilteredRecords`. Each error is one failure: `idValidation`, `idObject`, `idTableSchema`,
119
+ `nameTableSchema`, `message`, `levelError`, `typeEntity`, `validationDate`,
120
+ `nameFieldSchema`, `shortCode`.
121
+
122
+ `page_num` is a 0-based page number; `page_size` defaults to 20 on the
123
+ backend. `sort_by` is a column name (the backend maps `tableSchemaName`,
124
+ `fieldSchemaName` and `entityType` to its own columns). `level_errors`
125
+ takes `CORRECT`/`WARNING`/`ERROR`/`INFO`/`BLOCKER`, `type_entities`
126
+ `TABLE`/`DATASET`/`FIELD`/`RECORD`. The backend swallows its own
127
+ `EEAException` and answers an empty 200, so this can return `None`.
128
+ Roles: any dataset role incl. observer/requester, custodians, stewards.
129
+ For a data-lake dataflow (`bigData: true`) this answers empty even when the
130
+ UI shows failures; there is no `DL` variant of this route (see
131
+ `list_group_validations_dl()`).
132
+ """
133
+ params = _list_params(
134
+ page_num,
135
+ page_size,
136
+ sort_by,
137
+ asc,
138
+ level_errors,
139
+ type_entities,
140
+ table_filter,
141
+ field_value_filter,
142
+ )
143
+ response = self._client.request(
144
+ "GET", f"/validation/listValidations/{dataset_id}", params=params
145
+ )
146
+ return _json_or_none(response)
147
+
148
+ def list_group_validations(
149
+ self,
150
+ dataset_id: int,
151
+ *,
152
+ page_num: int = 0,
153
+ page_size: int = 20,
154
+ sort_by: str | None = None,
155
+ asc: bool = True,
156
+ level_errors: list[str] | None = None,
157
+ type_entities: list[str] | None = None,
158
+ table_filter: str | None = None,
159
+ field_value_filter: str | None = None,
160
+ ) -> dict[str, Any] | None:
161
+ """GET /validation/listGroupValidations/{datasetId}: failed validations grouped by
162
+ rule (what the UI shows).
163
+
164
+ Returns the backend's `FailedValidationsDatasetVO`: `idDataset`,
165
+ `idDatasetSchema`, `nameDataSetSchema`, `errors` (a list), `totalRecords`,
166
+ `totalErrors` and `totalFilteredRecords`. Each error is one rule's failures grouped by code: `message`, `idRule`,
167
+ `levelError`, `typeEntity`, `numberOfRecords`, `nameTableSchema`, `shortCode`,
168
+ `nameFieldSchema` (what the UI's validation table shows).
169
+
170
+ `page_num` is a 0-based page number; `page_size` defaults to 20 on the
171
+ backend. `sort_by` is a column name (the backend maps `tableSchemaName`,
172
+ `fieldSchemaName` and `entityType` to its own columns). `level_errors`
173
+ takes `CORRECT`/`WARNING`/`ERROR`/`INFO`/`BLOCKER`, `type_entities`
174
+ `TABLE`/`DATASET`/`FIELD`/`RECORD`. The backend swallows its own
175
+ `EEAException` and answers an empty 200, so this can return `None`.
176
+ Backend rule `checkAccessSuperUser('DATASET', id)`, not the role list of
177
+ `list_validations`. For a data-lake dataflow (`bigData: true`) use
178
+ `list_group_validations_dl()`: this one answers `errors: []` and
179
+ `totalRecords: 0` there even when the UI shows failures.
180
+ """
181
+ params = _list_params(
182
+ page_num,
183
+ page_size,
184
+ sort_by,
185
+ asc,
186
+ level_errors,
187
+ type_entities,
188
+ table_filter,
189
+ field_value_filter,
190
+ )
191
+ response = self._client.request(
192
+ "GET", f"/validation/listGroupValidations/{dataset_id}", params=params
193
+ )
194
+ return _json_or_none(response)
195
+
196
+ def list_group_validations_dl(
197
+ self,
198
+ dataset_id: int,
199
+ *,
200
+ page_num: int = 0,
201
+ page_size: int = 20,
202
+ sort_by: str | None = None,
203
+ asc: bool = True,
204
+ level_errors: list[str] | None = None,
205
+ type_entities: list[str] | None = None,
206
+ table_filter: str | None = None,
207
+ field_value_filter: str | None = None,
208
+ ) -> dict[str, Any] | None:
209
+ """GET /validation/listGroupValidationsDL/{datasetId}: grouped failures, data-lake datasets.
210
+
211
+ For a dataflow whose `dataflow_datasets()` answer has `bigData: true`
212
+ (data lake), validation results live in the data lake and
213
+ `list_group_validations()` reads the empty old per-dataset database, so it
214
+ answers `errors: []`, `totalRecords: 0`. This is the route the Reportnet UI
215
+ calls for those datasets. Same arguments and answer shape as
216
+ `list_group_validations()`. Live-tested on the sandbox (2026-10-27): it
217
+ returned the same two rule groups and totals the UI shows. The backend source
218
+ (`MasterDataLakes`) has no `DL` variant of `list_validations()` (one entry per
219
+ failing record), so that one stays empty for data-lake datasets. The UI also
220
+ sends a `shortCode` filter, not wrapped here.
221
+ """
222
+ params = _list_params(
223
+ page_num,
224
+ page_size,
225
+ sort_by,
226
+ asc,
227
+ level_errors,
228
+ type_entities,
229
+ table_filter,
230
+ field_value_filter,
231
+ )
232
+ response = self._client.request(
233
+ "GET", f"/validation/listGroupValidationsDL/{dataset_id}", params=params
234
+ )
235
+ return _json_or_none(response)
236
+
237
+ def start_validations_export(self, dataset_id: int) -> None:
238
+ """POST /validation/export/{datasetId}: generate the validations CSV.
239
+
240
+ Not a job: the backend writes `dataset-{id}-validations.csv` on the server
241
+ and only notifies the Reportnet UI, so there is no id or status to poll.
242
+ Fetch it with `download_validations_file()`, or use
243
+ `export_validations_to_file()` which does both. Leaves a notification in the
244
+ user's Reportnet UI.
245
+ """
246
+ self._client.request("POST", f"/validation/export/{dataset_id}")
247
+
248
+ def download_validations_file(self, dataset_id: int, dest: str | os.PathLike[str]) -> bool:
249
+ """GET /validation/downloadFile/{datasetId}: fetch the generated CSV to `dest`.
250
+
251
+ Returns False (writing nothing) while the file does not exist yet (`404`),
252
+ True once it was saved. The backend deletes the file after serving it, so a
253
+ second call returns False again.
254
+ """
255
+ part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
256
+ try:
257
+ self._client.download(
258
+ "GET",
259
+ f"/validation/downloadFile/{dataset_id}",
260
+ params={"fileName": _file_name(dataset_id)},
261
+ dest=part,
262
+ )
263
+ if not part.exists() or part.stat().st_size == 0:
264
+ return False
265
+ part.replace(dest)
266
+ return True
267
+ except NotFoundError:
268
+ return False
269
+ finally:
270
+ part.unlink(missing_ok=True)
271
+
272
+ def export_validations_to_file(
273
+ self,
274
+ dataset_id: int,
275
+ dest: str | os.PathLike[str],
276
+ *,
277
+ poll_interval: float = 2.0,
278
+ timeout: float = 300.0,
279
+ ) -> None:
280
+ """Generate the validations CSV and wait for it, saving it at `dest`.
281
+
282
+ Calls `start_validations_export()` then polls `download_validations_file()`
283
+ every `poll_interval` seconds; raises `JobError` after `timeout` seconds.
284
+ Columns (from `ValidationServiceImpl`): entity, table, field, code, code
285
+ name, code description, level error, message, number of records. A
286
+ leftover file with the same name from an abandoned export could be served
287
+ instead of a fresh one. The server path has no per-user part and the file
288
+ is deleted when served, so two simultaneous exports of one dataset can
289
+ take each other's file, and a dropped download loses it (export again).
290
+ A failed export is not reported (the backend still answers 200), so it
291
+ ends as a timeout.
292
+ """
293
+ self.start_validations_export(dataset_id)
294
+ deadline = time.monotonic() + timeout
295
+ while not self.download_validations_file(dataset_id, dest):
296
+ remaining = deadline - time.monotonic()
297
+ if remaining <= 0:
298
+ raise JobError(
299
+ f"Validations file not ready within {timeout:g} seconds (the backend "
300
+ "answers 200 even when its export fails, so a failed export looks like this)"
301
+ )
302
+ time.sleep(min(poll_interval, remaining))
303
+
304
+
305
+ class AsyncValidationResource:
306
+ def __init__(self, client: AsyncReportnetClient) -> None:
307
+ self._client = client
308
+
309
+ async def start_validation(self, dataset_id: int, *, released: bool = False) -> None:
310
+ """Async counterpart of ValidationResource.start_validation."""
311
+ await self._client.request(
312
+ "PUT",
313
+ f"/orchestrator/jobs/addValidationJob/{dataset_id}",
314
+ params={"released": "true"} if released else None,
315
+ )
316
+
317
+ async def list_validations(
318
+ self,
319
+ dataset_id: int,
320
+ *,
321
+ page_num: int = 0,
322
+ page_size: int = 20,
323
+ sort_by: str | None = None,
324
+ asc: bool = True,
325
+ level_errors: list[str] | None = None,
326
+ type_entities: list[str] | None = None,
327
+ table_filter: str | None = None,
328
+ field_value_filter: str | None = None,
329
+ ) -> dict[str, Any] | None:
330
+ """Async counterpart of ValidationResource.list_validations."""
331
+ params = _list_params(
332
+ page_num,
333
+ page_size,
334
+ sort_by,
335
+ asc,
336
+ level_errors,
337
+ type_entities,
338
+ table_filter,
339
+ field_value_filter,
340
+ )
341
+ response = await self._client.request(
342
+ "GET", f"/validation/listValidations/{dataset_id}", params=params
343
+ )
344
+ return _json_or_none(response)
345
+
346
+ async def list_group_validations(
347
+ self,
348
+ dataset_id: int,
349
+ *,
350
+ page_num: int = 0,
351
+ page_size: int = 20,
352
+ sort_by: str | None = None,
353
+ asc: bool = True,
354
+ level_errors: list[str] | None = None,
355
+ type_entities: list[str] | None = None,
356
+ table_filter: str | None = None,
357
+ field_value_filter: str | None = None,
358
+ ) -> dict[str, Any] | None:
359
+ """Async counterpart of ValidationResource.list_group_validations."""
360
+ params = _list_params(
361
+ page_num,
362
+ page_size,
363
+ sort_by,
364
+ asc,
365
+ level_errors,
366
+ type_entities,
367
+ table_filter,
368
+ field_value_filter,
369
+ )
370
+ response = await self._client.request(
371
+ "GET", f"/validation/listGroupValidations/{dataset_id}", params=params
372
+ )
373
+ return _json_or_none(response)
374
+
375
+ async def list_group_validations_dl(
376
+ self,
377
+ dataset_id: int,
378
+ *,
379
+ page_num: int = 0,
380
+ page_size: int = 20,
381
+ sort_by: str | None = None,
382
+ asc: bool = True,
383
+ level_errors: list[str] | None = None,
384
+ type_entities: list[str] | None = None,
385
+ table_filter: str | None = None,
386
+ field_value_filter: str | None = None,
387
+ ) -> dict[str, Any] | None:
388
+ """Async counterpart of ValidationResource.list_group_validations_dl."""
389
+ params = _list_params(
390
+ page_num,
391
+ page_size,
392
+ sort_by,
393
+ asc,
394
+ level_errors,
395
+ type_entities,
396
+ table_filter,
397
+ field_value_filter,
398
+ )
399
+ response = await self._client.request(
400
+ "GET", f"/validation/listGroupValidationsDL/{dataset_id}", params=params
401
+ )
402
+ return _json_or_none(response)
403
+
404
+ async def start_validations_export(self, dataset_id: int) -> None:
405
+ """Async counterpart of ValidationResource.start_validations_export."""
406
+ await self._client.request("POST", f"/validation/export/{dataset_id}")
407
+
408
+ async def download_validations_file(
409
+ self, dataset_id: int, dest: str | os.PathLike[str]
410
+ ) -> bool:
411
+ """Async counterpart of ValidationResource.download_validations_file."""
412
+ part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
413
+ try:
414
+ await self._client.download(
415
+ "GET",
416
+ f"/validation/downloadFile/{dataset_id}",
417
+ params={"fileName": _file_name(dataset_id)},
418
+ dest=part,
419
+ )
420
+ if not part.exists() or part.stat().st_size == 0:
421
+ return False
422
+ part.replace(dest)
423
+ return True
424
+ except NotFoundError:
425
+ return False
426
+ finally:
427
+ part.unlink(missing_ok=True)
428
+
429
+ async def export_validations_to_file(
430
+ self,
431
+ dataset_id: int,
432
+ dest: str | os.PathLike[str],
433
+ *,
434
+ poll_interval: float = 2.0,
435
+ timeout: float = 300.0,
436
+ ) -> None:
437
+ """Async counterpart of ValidationResource.export_validations_to_file."""
438
+ await self.start_validations_export(dataset_id)
439
+ deadline = time.monotonic() + timeout
440
+ while not await self.download_validations_file(dataset_id, dest):
441
+ remaining = deadline - time.monotonic()
442
+ if remaining <= 0:
443
+ raise JobError(
444
+ f"Validations file not ready within {timeout:g} seconds (the backend "
445
+ "answers 200 even when its export fails, so a failed export looks like this)"
446
+ )
447
+ await asyncio.sleep(min(poll_interval, remaining))