reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1001 @@
1
+ """Hand-written extension of the generated DatasetsResource classes.
2
+
3
+ Adds endpoints hidden from Swagger via `@ApiOperation(hidden = true)` in the
4
+ live backend, so they never appear in spec/api-docs.json ("Known
5
+ spec-snapshot gap"), and helpers that wait for the async Orchestrator jobs
6
+ behind the `etl_export_*` v3/v4/v5 and `import_big_file_data` methods (polling
7
+ lives in `reportnet_api/jobs.py`). This file is never touched by
8
+ scripts/generate_resources.py since it lives outside datasets.py.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import asyncio
14
+ import os
15
+ import tempfile
16
+ import time
17
+ import uuid
18
+ import warnings
19
+ from pathlib import Path
20
+ from typing import Any, cast
21
+
22
+ import httpx
23
+
24
+ from .._core import stream_upload_file
25
+ from .._exceptions import NotFoundError
26
+ from ..jobs import (
27
+ JobError,
28
+ Progress,
29
+ async_wait_for_job,
30
+ download_url_of,
31
+ import_statistics_changed,
32
+ progress_event,
33
+ progress_scope,
34
+ wait_for_job,
35
+ )
36
+ from .datasets import AsyncDatasetsResource as _GeneratedAsyncDatasetsResource
37
+ from .datasets import DatasetsResource as _GeneratedDatasetsResource
38
+
39
+ ExportJobError = JobError # name from before the shared poller; same class
40
+
41
+ _EXPORT_TO_FILE_DOC = """{summary}
42
+
43
+ `{starter}()` (`{route}`) only starts an async job. This polls its
44
+ `pollingUrl` (`/orchestrator/jobs/pollForJobStatus/{{jobId}}`, answers
45
+ `{{"status": ...}}`; `QUEUED` / `IN_PROGRESS` while it runs, see
46
+ `reportnet_api.jobs`) every `poll_interval` seconds until `FINISHED`, then
47
+ downloads the `downloadUrl` it returns
48
+ (`/orchestrator/jobs/downloadEtlExportedFile/{{jobId}}`) to `dest`.
49
+ Raises `JobError` (`ExportJobError`) on a failed/refused/cancelled job, on
50
+ an `error` answer, or after `timeout` seconds. Same roles as `{starter}()`.
51
+ `progress=True` shows one updating status line, a callable gets event dicts
52
+ (see `reportnet_api.jobs.progress_event`).
53
+
54
+ Written from the `MasterDataLakes` branch of the Reportnet 3 backend repository (the job
55
+ and the orchestrator's poll/download routes); untested live.
56
+ """
57
+
58
+ _WAIT_FOR_JOB_DOC = """Wait for an async job to finish and return its last status answer.
59
+
60
+ Pass what `import_big_file_data()` (or any job-starting method) returned,
61
+ or its `pollingUrl`. Polls every `poll_interval` seconds; raises `JobError`
62
+ on a failed/refused/cancelled job or after `timeout` seconds. For an import,
63
+ `FINISHED` is the job's own completion; `check_import_process()` is not a
64
+ reliable signal (live-tested). `progress=True` shows one updating status
65
+ line, a callable gets event dicts (`reportnet_api.jobs.progress_event`).
66
+ Untested live.
67
+ """
68
+
69
+ # After a big-file job reports FINISHED it is authoritative; statistics only get
70
+ # this long to show the change (they can lag, or stay equal for an empty/identical file).
71
+ _STATS_GRACE = 10.0
72
+
73
+ # Indirection so tests can drive import_file_and_wait's clock without patching `time`/`asyncio`.
74
+ _monotonic = time.monotonic
75
+ _sleep = time.sleep
76
+ _async_sleep = asyncio.sleep
77
+
78
+ _NO_JOB_MESSAGE = (
79
+ "The file was uploaded but the answer carried no pollingUrl to wait on ({job!r}); "
80
+ "the import may still complete, so check get_import_related_statistics() before retrying"
81
+ )
82
+
83
+ _IMPORT_AND_WAIT_DOC = """Upload a file into a dataset and wait until the import is visible.
84
+
85
+ Reads `get_import_related_statistics()` as a baseline, uploads `file` with
86
+ `import_big_file_data()` (`big_file=True`, the default; async job) or
87
+ `import_file_data()` (`big_file=False`; returns nothing and merges in the
88
+ background), then waits:
89
+
90
+ - big file: polls the answer's `pollingUrl` until the job is `FINISHED`
91
+ (`wait_for_job()`); if the answer has no `pollingUrl` this step is skipped.
92
+ Once the job is `FINISHED` it is authoritative: statistics then get only a
93
+ short grace period to show the change, after which the latest statistics
94
+ are returned anyway (an empty or identical file may change nothing).
95
+ - otherwise (small file, or no `pollingUrl`): polls
96
+ `get_import_related_statistics()` until a table's
97
+ `(lastImportDate, numberOfRecordsImported)` differs from the baseline
98
+ (only `table_schema_id`'s table when given), else raises on timeout.
99
+ Pass `confirm_statistics=False` to trust the job alone (big file only; a
100
+ missing `pollingUrl` then raises, though the upload was already sent).
101
+
102
+ `check_import_process()` is never used: its lock stayed set long after
103
+ the data was in (live-tested). `list_imported_files()` is not used
104
+ either: a file is listed before its data is merged. Returns the final
105
+ statistics answer. Raises `JobError` on a failed/refused/cancelled job,
106
+ or when `timeout` seconds pass while waiting (the upload may still complete
107
+ afterwards; nothing is rolled back, so check before retrying). `timeout`
108
+ bounds the waiting, not the baseline read or the upload itself, and
109
+ `poll_interval` / `timeout` apply to both polling phases combined. Roles as
110
+ for the underlying import method. `progress=True` shows one updating status
111
+ line (a callable gets event dicts); its elapsed time counts from the start
112
+ of the waiting.
113
+
114
+ Limits of the statistics check: without `table_schema_id` any table's
115
+ change counts, so a concurrent import into the same dataset can be
116
+ mistaken for yours; and a `replace` that makes a table drop out of the
117
+ statistics is not seen as a change.
118
+
119
+ Verified live (2026-10-23, Bearer, scratch dataset): `import_big_file_data`
120
+ returns a `pollingUrl` carrying `datasetId`/`dataflowId`, the job reached
121
+ `FINISHED` in ~20s, and `lastImportDate` changed (the count stayed at 1:
122
+ `numberOfRecordsImported` looks like the last import's rows, not a table
123
+ total, so don't compare the count alone). The file name must match a table
124
+ name or the job is `CANCELED`. The imported row was confirmed in the table
125
+ via the v4 CSV export. An ApiKey import also worked (job
126
+ FINISHED, statistics changed). NOT verified live: the `big_file=False`
127
+ path with this helper.
128
+ """
129
+
130
+ _EXPORT_DOCS = {
131
+ "zip_csv": _EXPORT_TO_FILE_DOC.format(
132
+ summary="Export a dataset as a ZIP of CSV files into `dest`, waiting for the job.",
133
+ starter="etl_export_zip_csv",
134
+ route="/dataset/v4/etlExport/{id}",
135
+ ),
136
+ "zip_parquet": _EXPORT_TO_FILE_DOC.format(
137
+ summary="Export a dataset as a ZIP of Parquet files into `dest`, waiting for the job.",
138
+ starter="etl_export_zip_parquet",
139
+ route="/dataset/v5/etlExport/{id}",
140
+ ),
141
+ "with_job": _EXPORT_TO_FILE_DOC.format(
142
+ summary="Export a dataset (v3 job variant) into `dest`, waiting for the job.",
143
+ starter="etl_export_dataset_with_job",
144
+ route="/dataset/v3/etlExport/{id}",
145
+ ),
146
+ }
147
+
148
+ _DELETE_IMPORT_TABLE_DOC = """DELETE /dataset/{dataset_id}/deleteImportTable/{table_schema_id}
149
+
150
+ Deletes all data in one table. Hidden legacy (unversioned) alias of
151
+ `delete_table_data()` (`/dataset/v1/{id}/deleteTableData/{tableId}`):
152
+ `DatasetControllerImpl#deleteImportTableLegacy` just calls
153
+ `deleteTableData`. The Reportnet FME integration uses this route.
154
+ Destructive and not undoable. Returns `None` (the controller is `void`).
155
+
156
+ Role rules are the same as `delete_table_data()`; the hidden route
157
+ also accepts an ApiKey (`dataflow_id`/`provider_id` feed that check).
158
+ Untested live: the backend source audited is older than production.
159
+ """
160
+
161
+
162
+ _FILE_EXPORT_NOTE = """
163
+
164
+ Not job-based: the backend starts generating the file in the background
165
+ and tells only the Reportnet UI (a user notification); there is no job
166
+ id or polling URL. `fetch_export_file()` / the `*_to_file()` helpers
167
+ therefore poll the download until a non-empty body arrives. A file left
168
+ from an earlier export with the same name can be served instead of the
169
+ new one (the normal download deletes the file, the `big_data` one does not).
170
+ Read from `DatasetControllerImpl` / `FileTreatmentHelper` (`MasterDataLakes`
171
+ branch; `develop` has no `big_data` variants). Live (2026-10-23, Bearer only: the
172
+ routes reject an ApiKey with 403) production behaves like the `big_data`
173
+ (`...DL`) routes, which is what the Reportnet UI calls; the plain routes
174
+ accepted a request but never produced a file. Hence `big_data=True` is the
175
+ default; for the dataset export use `zip csv` (plain `csv` produced no file).
176
+ """
177
+
178
+
179
+ def _file_route(name: str, big_data: bool) -> str:
180
+ return f"{name}DL" if big_data else name
181
+
182
+
183
+ def _table_export_request(
184
+ dataset_id: int,
185
+ mime_type: str,
186
+ table_schema_id: str | None,
187
+ level_error: list[str] | None,
188
+ id_rules: str | None,
189
+ field_value: str | None,
190
+ big_data: bool,
191
+ ) -> tuple[str, dict[str, Any], dict[str, Any]]:
192
+ params: dict[str, Any] = {"datasetId": dataset_id, "mimeType": mime_type}
193
+ if table_schema_id is not None:
194
+ params["tableSchemaId"] = table_schema_id
195
+ # The big-data route reads these without a null check, so always send them.
196
+ body: dict[str, Any] = {
197
+ "levelError": level_error or [],
198
+ "idRules": id_rules or "",
199
+ "fieldValue": field_value or "",
200
+ }
201
+ if big_data:
202
+ body["qcCodes"] = "" # a String in the backend's ExportFilterVO (MasterDataLakes)
203
+ return "/dataset/" + _file_route("exportFile", big_data), params, body
204
+
205
+
206
+ def _check_extension(mime_type: str) -> None:
207
+ """A table export is saved as `<table>.<mime_type>`, so it must be a bare extension."""
208
+ if (
209
+ not mime_type
210
+ or mime_type != mime_type.strip()
211
+ or " " in mime_type
212
+ or mime_type.startswith(".")
213
+ ):
214
+ raise ValueError(
215
+ f"mime_type must be a bare file extension such as 'csv' or 'xlsx', got {mime_type!r}"
216
+ )
217
+
218
+
219
+ _STALE_WARNING = (
220
+ "An export file named {name!r} already exists on the server and may be returned instead "
221
+ "of the new export; compare the result with the dataset if the data changed."
222
+ )
223
+
224
+
225
+ def _dataset_file_name(dataset_name: str, mime_type: str) -> str:
226
+ """File the dataset export writes: `<name>.zip` for `zip csv`/`zip xlsx`, else `<name>.<ext>`."""
227
+ parts = mime_type.split()
228
+ return (
229
+ f"{dataset_name}.zip"
230
+ if len(parts) > 1 and parts[0] == "zip"
231
+ else f"{dataset_name}.{parts[-1]}"
232
+ )
233
+
234
+
235
+ class DatasetsResource(_GeneratedDatasetsResource):
236
+ def import_big_file_data(
237
+ self,
238
+ dataset_id: int,
239
+ file: str | os.PathLike[str],
240
+ dataflow_id: int | None = None,
241
+ delimiter: str | None = None,
242
+ fme_job_id: str | None = None,
243
+ integration_id: int | None = None,
244
+ job_id: int | None = None,
245
+ provider_id: int | None = None,
246
+ replace: bool | None = None,
247
+ table_schema_id: str | None = None,
248
+ preparation_code: str | None = None,
249
+ *,
250
+ timeout: float | httpx.Timeout | None = None,
251
+ ) -> dict[str, Any]:
252
+ """Import file to dataset data (Large files)
253
+
254
+ Allowed roles:
255
+
256
+ Reporting dataset: LEAD REPORTER, REPORTER WRITE, NATIONAL COORDINATOR
257
+
258
+ Data collection: CUSTODIAN, STEWARD
259
+
260
+ Test dataset: CUSTODIAN, STEWARD, STEWARD SUPPORT
261
+
262
+ Reference dataset: CUSTODIAN, STEWARD
263
+
264
+ Design dataset: CUSTODIAN, STEWARD, EDITOR WRITE
265
+
266
+ EU dataset: CUSTODIAN, STEWARD
267
+
268
+ Overrides the generated method. An empty response body returns `{}` (data lake
269
+ datasets answer `void`: the import is done inside the request, MasterDataLakes `DatasetControllerImpl:391-422`;
270
+ other datasets answer a job map), and `timeout` overrides the client's timeout for
271
+ this call only (a lake import can run for minutes). `preparation_code` is sent as
272
+ `code`."""
273
+ params = {
274
+ "dataflowId": dataflow_id,
275
+ "delimiter": delimiter,
276
+ "fmeJobId": fme_job_id,
277
+ "integrationId": integration_id,
278
+ "jobId": job_id,
279
+ "providerId": provider_id,
280
+ "replace": replace,
281
+ "tableSchemaId": table_schema_id,
282
+ "code": preparation_code,
283
+ }
284
+ params = {k: v for k, v in params.items() if v is not None}
285
+ options: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
286
+ url = f"/dataset/v2/importFileData/{dataset_id}"
287
+ with stream_upload_file("file", file) as files:
288
+ response = self._client.request("POST", url, params=params, files=files, **options)
289
+ return cast(dict[str, Any], response.json()) if response.content.strip() else {}
290
+
291
+ def delete_import_table(
292
+ self,
293
+ dataset_id: int,
294
+ table_schema_id: str,
295
+ dataflow_id: int | None = None,
296
+ provider_id: int | None = None,
297
+ ) -> None:
298
+ params = {"dataflowId": dataflow_id, "providerId": provider_id}
299
+ params = {k: v for k, v in params.items() if v is not None}
300
+ self._client.request(
301
+ "DELETE", f"/dataset/{dataset_id}/deleteImportTable/{table_schema_id}", params=params
302
+ )
303
+
304
+ delete_import_table.__doc__ = _DELETE_IMPORT_TABLE_DOC
305
+
306
+ def update_dataset_name(self, dataset_id: int, name: str) -> None:
307
+ """PUT /datasetmetabase/updateDatasetName: rename a dataset (as the web
308
+ app renames a design dataset, i.e. its schema).
309
+
310
+ `DatasetMetabaseControllerImpl#updateDatasetName` (hidden). Requires
311
+ `DATASCHEMA_STEWARD`, `DATASCHEMA_CUSTODIAN` or `DATASCHEMA_EDITOR_WRITE`
312
+ on the dataset. The backend trims `name` and answers `400` unless it
313
+ matches `[a-zA-Z0-9\\s()_-]+`, or for an unknown dataset. It does not
314
+ check that the name is unique in the dataflow.
315
+ """
316
+ self._client.request(
317
+ "PUT",
318
+ "/datasetmetabase/updateDatasetName",
319
+ params={"datasetId": dataset_id, "datasetName": name},
320
+ )
321
+
322
+ def wait_for_job(
323
+ self,
324
+ job: dict[str, Any] | str,
325
+ *,
326
+ poll_interval: float = 2.0,
327
+ timeout: float = 300.0,
328
+ progress: Progress = None,
329
+ ) -> dict[str, Any]:
330
+ return wait_for_job(
331
+ self._client, job, poll_interval=poll_interval, timeout=timeout, progress=progress
332
+ )
333
+
334
+ wait_for_job.__doc__ = _WAIT_FOR_JOB_DOC
335
+
336
+ def import_file_and_wait(
337
+ self,
338
+ dataset_id: int,
339
+ file: str | os.PathLike[str],
340
+ *,
341
+ dataflow_id: int | None = None,
342
+ delimiter: str | None = None,
343
+ integration_id: int | None = None,
344
+ provider_id: int | None = None,
345
+ replace: bool | None = None,
346
+ table_schema_id: str | None = None,
347
+ big_file: bool = True,
348
+ confirm_statistics: bool = True,
349
+ poll_interval: float = 2.0,
350
+ timeout: float = 300.0,
351
+ progress: Progress = None,
352
+ ) -> dict[str, Any]:
353
+ before = self.get_import_related_statistics(dataset_id)
354
+ options: dict[str, Any] = {
355
+ "dataflow_id": dataflow_id,
356
+ "delimiter": delimiter,
357
+ "integration_id": integration_id,
358
+ "provider_id": provider_id,
359
+ "replace": replace,
360
+ "table_schema_id": table_schema_id,
361
+ }
362
+ job_finished = False
363
+ with progress_scope(progress) as report:
364
+ job: Any = None # only a big-file upload answers with a job to wait on
365
+ if big_file:
366
+ job = self.import_big_file_data(dataset_id, file, **options)
367
+ else:
368
+ self.import_file_data(dataset_id, file, **options)
369
+ # `timeout` bounds the waiting, not the upload.
370
+ started = _monotonic()
371
+ deadline = started + timeout
372
+ if isinstance(job, dict) and job.get("pollingUrl"):
373
+ remaining = max(0.0, deadline - _monotonic())
374
+ wait_for_job(
375
+ self._client,
376
+ job,
377
+ poll_interval=poll_interval,
378
+ timeout=remaining,
379
+ progress=report,
380
+ )
381
+ job_finished = True
382
+ deadline = min(deadline, _monotonic() + _STATS_GRACE)
383
+ elif big_file and not confirm_statistics:
384
+ raise JobError(_NO_JOB_MESSAGE.format(job=job))
385
+ while True:
386
+ after = self.get_import_related_statistics(dataset_id)
387
+ if (big_file and not confirm_statistics) or import_statistics_changed(
388
+ before, after, table_schema_id
389
+ ):
390
+ return after
391
+ if _monotonic() >= deadline:
392
+ if job_finished:
393
+ return after
394
+ raise JobError(f"Import not visible in statistics within {timeout:g} seconds")
395
+ if report:
396
+ elapsed = _monotonic() - started
397
+ report(progress_event("confirm", elapsed=elapsed))
398
+ _sleep(poll_interval)
399
+
400
+ import_file_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
401
+
402
+ def _finish_export(
403
+ self,
404
+ job: dict[str, Any],
405
+ dest: str | os.PathLike[str],
406
+ poll_interval: float,
407
+ timeout: float,
408
+ progress: Progress = None,
409
+ ) -> None:
410
+ answer = wait_for_job(
411
+ self._client, job, poll_interval=poll_interval, timeout=timeout, progress=progress
412
+ )
413
+ self._client.download("GET", download_url_of(answer), dest=dest)
414
+
415
+ def export_zip_csv_to_file(
416
+ self,
417
+ dataset_id: int,
418
+ *,
419
+ dataflow_id: int,
420
+ dest: str | os.PathLike[str],
421
+ table_schema_id: str | None = None,
422
+ poll_interval: float = 2.0,
423
+ timeout: float = 300.0,
424
+ progress: Progress = None,
425
+ ) -> None:
426
+ job = self.etl_export_zip_csv(
427
+ dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
428
+ )
429
+ self._finish_export(job, dest, poll_interval, timeout, progress)
430
+
431
+ export_zip_csv_to_file.__doc__ = _EXPORT_DOCS["zip_csv"]
432
+
433
+ def export_zip_parquet_to_file(
434
+ self,
435
+ dataset_id: int,
436
+ *,
437
+ dataflow_id: int,
438
+ dest: str | os.PathLike[str],
439
+ table_schema_id: str | None = None,
440
+ poll_interval: float = 2.0,
441
+ timeout: float = 300.0,
442
+ progress: Progress = None,
443
+ ) -> None:
444
+ job = self.etl_export_zip_parquet(
445
+ dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
446
+ )
447
+ self._finish_export(job, dest, poll_interval, timeout, progress)
448
+
449
+ export_zip_parquet_to_file.__doc__ = _EXPORT_DOCS["zip_parquet"]
450
+
451
+ def export_dataset_with_job_to_file(
452
+ self,
453
+ dataset_id: int,
454
+ *,
455
+ dataflow_id: int,
456
+ dest: str | os.PathLike[str],
457
+ table_schema_id: str | None = None,
458
+ provider_id: int | None = None,
459
+ poll_interval: float = 2.0,
460
+ timeout: float = 300.0,
461
+ progress: Progress = None,
462
+ ) -> None:
463
+ job = self.etl_export_dataset_with_job(
464
+ dataset_id,
465
+ table_schema_id=table_schema_id,
466
+ provider_id=provider_id,
467
+ dataflow_id=dataflow_id,
468
+ )
469
+ self._finish_export(job, dest, poll_interval, timeout, progress)
470
+
471
+ export_dataset_with_job_to_file.__doc__ = _EXPORT_DOCS["with_job"]
472
+
473
+ def start_table_export(
474
+ self,
475
+ dataset_id: int,
476
+ *,
477
+ mime_type: str = "csv",
478
+ table_schema_id: str | None = None,
479
+ level_error: list[str] | None = None,
480
+ id_rules: str | None = None,
481
+ field_value: str | None = None,
482
+ big_data: bool = True,
483
+ ) -> None:
484
+ path, params, body = _table_export_request(
485
+ dataset_id, mime_type, table_schema_id, level_error, id_rules, field_value, big_data
486
+ )
487
+ self._client.request("POST", path, params=params, json=body)
488
+
489
+ def start_dataset_export(
490
+ self, dataset_id: int, *, mime_type: str = "csv", big_data: bool = True
491
+ ) -> None:
492
+ self._client.request(
493
+ "GET",
494
+ f"/dataset/{dataset_id}/" + _file_route("exportDatasetFile", big_data),
495
+ params={"mimeType": mime_type},
496
+ )
497
+
498
+ def fetch_export_file(
499
+ self,
500
+ dataset_id: int,
501
+ file_name: str,
502
+ *,
503
+ dest: str | os.PathLike[str],
504
+ big_data: bool = True,
505
+ ) -> bool:
506
+ part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
507
+ try:
508
+ self._client.download(
509
+ "GET",
510
+ f"/dataset/{dataset_id}/" + _file_route("downloadFile", big_data),
511
+ params={"fileName": file_name},
512
+ dest=part,
513
+ )
514
+ if not part.exists() or part.stat().st_size == 0:
515
+ return False
516
+ part.replace(dest)
517
+ return True
518
+ except NotFoundError:
519
+ return False # not generated yet
520
+ finally:
521
+ part.unlink(missing_ok=True)
522
+
523
+ def _warn_if_stale(self, dataset_id: int, file_name: str, big_data: bool) -> None:
524
+ with tempfile.TemporaryDirectory() as tmp:
525
+ probe = Path(tmp) / "probe"
526
+ if self.fetch_export_file(dataset_id, file_name, dest=probe, big_data=big_data):
527
+ warnings.warn(_STALE_WARNING.format(name=file_name), RuntimeWarning, stacklevel=4)
528
+
529
+ def _wait_for_export_file(
530
+ self,
531
+ dataset_id: int,
532
+ file_name: str,
533
+ dest: str | os.PathLike[str],
534
+ big_data: bool,
535
+ poll_interval: float,
536
+ timeout: float,
537
+ progress: Progress = None,
538
+ ) -> None:
539
+ started = time.monotonic()
540
+ deadline = started + timeout
541
+ with progress_scope(progress) as report:
542
+ while not self.fetch_export_file(dataset_id, file_name, dest=dest, big_data=big_data):
543
+ remaining = deadline - time.monotonic()
544
+ if remaining <= 0:
545
+ raise JobError(
546
+ f"Export file {file_name!r} not ready within {timeout:g} seconds"
547
+ )
548
+ if report:
549
+ report(progress_event("file", elapsed=time.monotonic() - started))
550
+ time.sleep(min(poll_interval, remaining))
551
+
552
+ def export_table_to_file(
553
+ self,
554
+ dataset_id: int,
555
+ *,
556
+ table_name: str,
557
+ table_schema_id: str,
558
+ dest: str | os.PathLike[str],
559
+ mime_type: str = "csv",
560
+ level_error: list[str] | None = None,
561
+ id_rules: str | None = None,
562
+ field_value: str | None = None,
563
+ big_data: bool = True,
564
+ poll_interval: float = 2.0,
565
+ timeout: float = 300.0,
566
+ progress: Progress = None,
567
+ ) -> None:
568
+ _check_extension(mime_type)
569
+ file_name = f"{table_name}.{mime_type}"
570
+ self._warn_if_stale(dataset_id, file_name, big_data)
571
+ self.start_table_export(
572
+ dataset_id,
573
+ mime_type=mime_type,
574
+ table_schema_id=table_schema_id,
575
+ level_error=level_error,
576
+ id_rules=id_rules,
577
+ field_value=field_value,
578
+ big_data=big_data,
579
+ )
580
+ self._wait_for_export_file(
581
+ dataset_id, file_name, dest, big_data, poll_interval, timeout, progress
582
+ )
583
+
584
+ def export_dataset_to_file(
585
+ self,
586
+ dataset_id: int,
587
+ *,
588
+ dataset_name: str,
589
+ dest: str | os.PathLike[str],
590
+ mime_type: str = "csv",
591
+ big_data: bool = True,
592
+ poll_interval: float = 2.0,
593
+ timeout: float = 300.0,
594
+ progress: Progress = None,
595
+ ) -> None:
596
+ file_name = _dataset_file_name(dataset_name, mime_type)
597
+ self._warn_if_stale(dataset_id, file_name, big_data)
598
+ self.start_dataset_export(dataset_id, mime_type=mime_type, big_data=big_data)
599
+ self._wait_for_export_file(
600
+ dataset_id,
601
+ file_name,
602
+ dest,
603
+ big_data,
604
+ poll_interval,
605
+ timeout,
606
+ progress,
607
+ )
608
+
609
+
610
+ class AsyncDatasetsResource(_GeneratedAsyncDatasetsResource):
611
+ async def import_big_file_data(
612
+ self,
613
+ dataset_id: int,
614
+ file: str | os.PathLike[str],
615
+ dataflow_id: int | None = None,
616
+ delimiter: str | None = None,
617
+ fme_job_id: str | None = None,
618
+ integration_id: int | None = None,
619
+ job_id: int | None = None,
620
+ provider_id: int | None = None,
621
+ replace: bool | None = None,
622
+ table_schema_id: str | None = None,
623
+ preparation_code: str | None = None,
624
+ *,
625
+ timeout: float | httpx.Timeout | None = None,
626
+ ) -> dict[str, Any]:
627
+ """Import file to dataset data (Large files)
628
+
629
+ Allowed roles:
630
+
631
+ Reporting dataset: LEAD REPORTER, REPORTER WRITE, NATIONAL COORDINATOR
632
+
633
+ Data collection: CUSTODIAN, STEWARD
634
+
635
+ Test dataset: CUSTODIAN, STEWARD, STEWARD SUPPORT
636
+
637
+ Reference dataset: CUSTODIAN, STEWARD
638
+
639
+ Design dataset: CUSTODIAN, STEWARD, EDITOR WRITE
640
+
641
+ EU dataset: CUSTODIAN, STEWARD
642
+
643
+ Overrides the generated method. An empty response body returns `{}` (data lake
644
+ datasets answer `void`: the import is done inside the request, MasterDataLakes `DatasetControllerImpl:391-422`;
645
+ other datasets answer a job map), and `timeout` overrides the client's timeout for
646
+ this call only (a lake import can run for minutes). `preparation_code` is sent as
647
+ `code`."""
648
+ params = {
649
+ "dataflowId": dataflow_id,
650
+ "delimiter": delimiter,
651
+ "fmeJobId": fme_job_id,
652
+ "integrationId": integration_id,
653
+ "jobId": job_id,
654
+ "providerId": provider_id,
655
+ "replace": replace,
656
+ "tableSchemaId": table_schema_id,
657
+ "code": preparation_code,
658
+ }
659
+ params = {k: v for k, v in params.items() if v is not None}
660
+ options: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
661
+ url = f"/dataset/v2/importFileData/{dataset_id}"
662
+ with stream_upload_file("file", file) as files:
663
+ response = await self._client.request(
664
+ "POST", url, params=params, files=files, **options
665
+ )
666
+ return cast(dict[str, Any], response.json()) if response.content.strip() else {}
667
+
668
+ async def delete_import_table(
669
+ self,
670
+ dataset_id: int,
671
+ table_schema_id: str,
672
+ dataflow_id: int | None = None,
673
+ provider_id: int | None = None,
674
+ ) -> None:
675
+ params = {"dataflowId": dataflow_id, "providerId": provider_id}
676
+ params = {k: v for k, v in params.items() if v is not None}
677
+ await self._client.request(
678
+ "DELETE", f"/dataset/{dataset_id}/deleteImportTable/{table_schema_id}", params=params
679
+ )
680
+
681
+ delete_import_table.__doc__ = _DELETE_IMPORT_TABLE_DOC
682
+
683
+ async def update_dataset_name(self, dataset_id: int, name: str) -> None:
684
+ """Async counterpart of DatasetsResource.update_dataset_name."""
685
+ await self._client.request(
686
+ "PUT",
687
+ "/datasetmetabase/updateDatasetName",
688
+ params={"datasetId": dataset_id, "datasetName": name},
689
+ )
690
+
691
+ async def wait_for_job(
692
+ self, job: dict[str, Any] | str, *, poll_interval: float = 2.0, timeout: float = 300.0
693
+ ) -> dict[str, Any]:
694
+ return await async_wait_for_job(
695
+ self._client, job, poll_interval=poll_interval, timeout=timeout
696
+ )
697
+
698
+ wait_for_job.__doc__ = _WAIT_FOR_JOB_DOC
699
+
700
+ async def import_file_and_wait(
701
+ self,
702
+ dataset_id: int,
703
+ file: str | os.PathLike[str],
704
+ *,
705
+ dataflow_id: int | None = None,
706
+ delimiter: str | None = None,
707
+ integration_id: int | None = None,
708
+ provider_id: int | None = None,
709
+ replace: bool | None = None,
710
+ table_schema_id: str | None = None,
711
+ big_file: bool = True,
712
+ confirm_statistics: bool = True,
713
+ poll_interval: float = 2.0,
714
+ timeout: float = 300.0,
715
+ ) -> dict[str, Any]:
716
+ before = await self.get_import_related_statistics(dataset_id)
717
+ options: dict[str, Any] = {
718
+ "dataflow_id": dataflow_id,
719
+ "delimiter": delimiter,
720
+ "integration_id": integration_id,
721
+ "provider_id": provider_id,
722
+ "replace": replace,
723
+ "table_schema_id": table_schema_id,
724
+ }
725
+ job_finished = False
726
+ job: Any = None # only a big-file upload answers with a job to wait on
727
+ if big_file:
728
+ job = await self.import_big_file_data(dataset_id, file, **options)
729
+ else:
730
+ await self.import_file_data(dataset_id, file, **options)
731
+ # `timeout` bounds the waiting, not the upload.
732
+ deadline = _monotonic() + timeout
733
+ if isinstance(job, dict) and job.get("pollingUrl"):
734
+ remaining = max(0.0, deadline - _monotonic())
735
+ await async_wait_for_job(
736
+ self._client, job, poll_interval=poll_interval, timeout=remaining
737
+ )
738
+ job_finished = True
739
+ deadline = min(deadline, _monotonic() + _STATS_GRACE)
740
+ elif big_file and not confirm_statistics:
741
+ raise JobError(_NO_JOB_MESSAGE.format(job=job))
742
+ while True:
743
+ after = await self.get_import_related_statistics(dataset_id)
744
+ if (big_file and not confirm_statistics) or import_statistics_changed(
745
+ before, after, table_schema_id
746
+ ):
747
+ return after
748
+ if _monotonic() >= deadline:
749
+ if job_finished:
750
+ return after
751
+ raise JobError(f"Import not visible in statistics within {timeout:g} seconds")
752
+ await _async_sleep(poll_interval)
753
+
754
+ import_file_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
755
+
756
+ async def _finish_export(
757
+ self,
758
+ job: dict[str, Any],
759
+ dest: str | os.PathLike[str],
760
+ poll_interval: float,
761
+ timeout: float,
762
+ ) -> None:
763
+ answer = await async_wait_for_job(
764
+ self._client, job, poll_interval=poll_interval, timeout=timeout
765
+ )
766
+ await self._client.download("GET", download_url_of(answer), dest=dest)
767
+
768
+ async def export_zip_csv_to_file(
769
+ self,
770
+ dataset_id: int,
771
+ *,
772
+ dataflow_id: int,
773
+ dest: str | os.PathLike[str],
774
+ table_schema_id: str | None = None,
775
+ poll_interval: float = 2.0,
776
+ timeout: float = 300.0,
777
+ ) -> None:
778
+ job = await self.etl_export_zip_csv(
779
+ dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
780
+ )
781
+ await self._finish_export(job, dest, poll_interval, timeout)
782
+
783
+ export_zip_csv_to_file.__doc__ = _EXPORT_DOCS["zip_csv"]
784
+
785
+ async def export_zip_parquet_to_file(
786
+ self,
787
+ dataset_id: int,
788
+ *,
789
+ dataflow_id: int,
790
+ dest: str | os.PathLike[str],
791
+ table_schema_id: str | None = None,
792
+ poll_interval: float = 2.0,
793
+ timeout: float = 300.0,
794
+ ) -> None:
795
+ job = await self.etl_export_zip_parquet(
796
+ dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
797
+ )
798
+ await self._finish_export(job, dest, poll_interval, timeout)
799
+
800
+ export_zip_parquet_to_file.__doc__ = _EXPORT_DOCS["zip_parquet"]
801
+
802
+ async def export_dataset_with_job_to_file(
803
+ self,
804
+ dataset_id: int,
805
+ *,
806
+ dataflow_id: int,
807
+ dest: str | os.PathLike[str],
808
+ table_schema_id: str | None = None,
809
+ provider_id: int | None = None,
810
+ poll_interval: float = 2.0,
811
+ timeout: float = 300.0,
812
+ ) -> None:
813
+ job = await self.etl_export_dataset_with_job(
814
+ dataset_id,
815
+ table_schema_id=table_schema_id,
816
+ provider_id=provider_id,
817
+ dataflow_id=dataflow_id,
818
+ )
819
+ await self._finish_export(job, dest, poll_interval, timeout)
820
+
821
+ export_dataset_with_job_to_file.__doc__ = _EXPORT_DOCS["with_job"]
822
+
823
+ async def start_table_export(
824
+ self,
825
+ dataset_id: int,
826
+ *,
827
+ mime_type: str = "csv",
828
+ table_schema_id: str | None = None,
829
+ level_error: list[str] | None = None,
830
+ id_rules: str | None = None,
831
+ field_value: str | None = None,
832
+ big_data: bool = True,
833
+ ) -> None:
834
+ path, params, body = _table_export_request(
835
+ dataset_id, mime_type, table_schema_id, level_error, id_rules, field_value, big_data
836
+ )
837
+ await self._client.request("POST", path, params=params, json=body)
838
+
839
+ async def start_dataset_export(
840
+ self, dataset_id: int, *, mime_type: str = "csv", big_data: bool = True
841
+ ) -> None:
842
+ await self._client.request(
843
+ "GET",
844
+ f"/dataset/{dataset_id}/" + _file_route("exportDatasetFile", big_data),
845
+ params={"mimeType": mime_type},
846
+ )
847
+
848
+ async def fetch_export_file(
849
+ self,
850
+ dataset_id: int,
851
+ file_name: str,
852
+ *,
853
+ dest: str | os.PathLike[str],
854
+ big_data: bool = True,
855
+ ) -> bool:
856
+ part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
857
+ try:
858
+ await self._client.download(
859
+ "GET",
860
+ f"/dataset/{dataset_id}/" + _file_route("downloadFile", big_data),
861
+ params={"fileName": file_name},
862
+ dest=part,
863
+ )
864
+ if not part.exists() or part.stat().st_size == 0:
865
+ return False
866
+ part.replace(dest)
867
+ return True
868
+ except NotFoundError:
869
+ return False # not generated yet
870
+ finally:
871
+ part.unlink(missing_ok=True)
872
+
873
+ async def _warn_if_stale(self, dataset_id: int, file_name: str, big_data: bool) -> None:
874
+ with tempfile.TemporaryDirectory() as tmp:
875
+ probe = Path(tmp) / "probe"
876
+ if await self.fetch_export_file(dataset_id, file_name, dest=probe, big_data=big_data):
877
+ warnings.warn(_STALE_WARNING.format(name=file_name), RuntimeWarning, stacklevel=4)
878
+
879
+ async def _wait_for_export_file(
880
+ self,
881
+ dataset_id: int,
882
+ file_name: str,
883
+ dest: str | os.PathLike[str],
884
+ big_data: bool,
885
+ poll_interval: float,
886
+ timeout: float,
887
+ ) -> None:
888
+ deadline = time.monotonic() + timeout
889
+ while not await self.fetch_export_file(dataset_id, file_name, dest=dest, big_data=big_data):
890
+ remaining = deadline - time.monotonic()
891
+ if remaining <= 0:
892
+ raise JobError(f"Export file {file_name!r} not ready within {timeout:g} seconds")
893
+ await asyncio.sleep(min(poll_interval, remaining))
894
+
895
+ async def export_table_to_file(
896
+ self,
897
+ dataset_id: int,
898
+ *,
899
+ table_name: str,
900
+ table_schema_id: str,
901
+ dest: str | os.PathLike[str],
902
+ mime_type: str = "csv",
903
+ level_error: list[str] | None = None,
904
+ id_rules: str | None = None,
905
+ field_value: str | None = None,
906
+ big_data: bool = True,
907
+ poll_interval: float = 2.0,
908
+ timeout: float = 300.0,
909
+ ) -> None:
910
+ _check_extension(mime_type)
911
+ file_name = f"{table_name}.{mime_type}"
912
+ await self._warn_if_stale(dataset_id, file_name, big_data)
913
+ await self.start_table_export(
914
+ dataset_id,
915
+ mime_type=mime_type,
916
+ table_schema_id=table_schema_id,
917
+ level_error=level_error,
918
+ id_rules=id_rules,
919
+ field_value=field_value,
920
+ big_data=big_data,
921
+ )
922
+ await self._wait_for_export_file(
923
+ dataset_id, file_name, dest, big_data, poll_interval, timeout
924
+ )
925
+
926
+ async def export_dataset_to_file(
927
+ self,
928
+ dataset_id: int,
929
+ *,
930
+ dataset_name: str,
931
+ dest: str | os.PathLike[str],
932
+ mime_type: str = "csv",
933
+ big_data: bool = True,
934
+ poll_interval: float = 2.0,
935
+ timeout: float = 300.0,
936
+ ) -> None:
937
+ file_name = _dataset_file_name(dataset_name, mime_type)
938
+ await self._warn_if_stale(dataset_id, file_name, big_data)
939
+ await self.start_dataset_export(dataset_id, mime_type=mime_type, big_data=big_data)
940
+ await self._wait_for_export_file(
941
+ dataset_id,
942
+ file_name,
943
+ dest,
944
+ big_data,
945
+ poll_interval,
946
+ timeout,
947
+ )
948
+
949
+
950
+ def _set_docs(docs: dict[str, str]) -> None:
951
+ for name, doc in docs.items():
952
+ getattr(DatasetsResource, name).__doc__ = doc
953
+ getattr(AsyncDatasetsResource, name).__doc__ = doc
954
+
955
+
956
+ _set_docs(
957
+ {
958
+ "start_table_export": (
959
+ """POST /dataset/exportFile (or `exportFileDL` with `big_data=True`): start exporting one table.
960
+
961
+ `mime_type` is the file extension (`csv`, `xlsx`, ...); without
962
+ `table_schema_id` the whole dataset is exported under the dataset's name.
963
+ Optional filters (`ExportFilterVO`): `level_error` (names of the backend's
964
+ `ErrorTypeEnum`), `id_rules`, `field_value`; `big_data=True` also sends an
965
+ empty `qcCodes` string. Returns nothing; the file is
966
+ named `<table name>.<mime_type>`."""
967
+ + _FILE_EXPORT_NOTE
968
+ ),
969
+ "start_dataset_export": (
970
+ """GET /dataset/{datasetId}/exportDatasetFile (or `exportDatasetFileDL`): start exporting the whole dataset.
971
+
972
+ `mime_type` is `csv`, `xlsx`, ... or `zip csv` / `zip xlsx` (a space; the
973
+ backend splits it). The file is `<dataset name>.zip` for the zip types,
974
+ else `<dataset name>.<ext>`. Returns nothing."""
975
+ + _FILE_EXPORT_NOTE
976
+ ),
977
+ "fetch_export_file": (
978
+ """GET /dataset/{datasetId}/downloadFile (or `downloadFileDL`): download a generated export file.
979
+
980
+ Saves to `dest` and returns True, or returns False without leaving a file
981
+ when the backend answered with an empty body (file not generated yet or not
982
+ found: the backend swallows that error)."""
983
+ + _FILE_EXPORT_NOTE
984
+ ),
985
+ "export_table_to_file": (
986
+ """Export one table to `dest`, waiting for the file: start the export, then poll the download.
987
+
988
+ `table_name` is the table's name (the backend names the file after it);
989
+ `table_schema_id` selects the table; `mime_type` must be a bare extension (`csv`, `xlsx`).
990
+ Raises `JobError` after `timeout` seconds without a file."""
991
+ + _FILE_EXPORT_NOTE
992
+ ),
993
+ "export_dataset_to_file": (
994
+ """Export the whole dataset to `dest`, waiting for the file.
995
+
996
+ `dataset_name` is the dataset's name (the backend names the file after it).
997
+ Raises `JobError` after `timeout` seconds without a file."""
998
+ + _FILE_EXPORT_NOTE
999
+ ),
1000
+ }
1001
+ )