reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1001 @@
|
|
|
1
|
+
"""Hand-written extension of the generated DatasetsResource classes.
|
|
2
|
+
|
|
3
|
+
Adds endpoints hidden from Swagger via `@ApiOperation(hidden = true)` in the
|
|
4
|
+
live backend, so they never appear in spec/api-docs.json ("Known
|
|
5
|
+
spec-snapshot gap"), and helpers that wait for the async Orchestrator jobs
|
|
6
|
+
behind the `etl_export_*` v3/v4/v5 and `import_big_file_data` methods (polling
|
|
7
|
+
lives in `reportnet_api/jobs.py`). This file is never touched by
|
|
8
|
+
scripts/generate_resources.py since it lives outside datasets.py.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import os
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
import uuid
|
|
18
|
+
import warnings
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, cast
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
from .._core import stream_upload_file
|
|
25
|
+
from .._exceptions import NotFoundError
|
|
26
|
+
from ..jobs import (
|
|
27
|
+
JobError,
|
|
28
|
+
Progress,
|
|
29
|
+
async_wait_for_job,
|
|
30
|
+
download_url_of,
|
|
31
|
+
import_statistics_changed,
|
|
32
|
+
progress_event,
|
|
33
|
+
progress_scope,
|
|
34
|
+
wait_for_job,
|
|
35
|
+
)
|
|
36
|
+
from .datasets import AsyncDatasetsResource as _GeneratedAsyncDatasetsResource
|
|
37
|
+
from .datasets import DatasetsResource as _GeneratedDatasetsResource
|
|
38
|
+
|
|
39
|
+
ExportJobError = JobError # name from before the shared poller; same class
|
|
40
|
+
|
|
41
|
+
_EXPORT_TO_FILE_DOC = """{summary}
|
|
42
|
+
|
|
43
|
+
`{starter}()` (`{route}`) only starts an async job. This polls its
|
|
44
|
+
`pollingUrl` (`/orchestrator/jobs/pollForJobStatus/{{jobId}}`, answers
|
|
45
|
+
`{{"status": ...}}`; `QUEUED` / `IN_PROGRESS` while it runs, see
|
|
46
|
+
`reportnet_api.jobs`) every `poll_interval` seconds until `FINISHED`, then
|
|
47
|
+
downloads the `downloadUrl` it returns
|
|
48
|
+
(`/orchestrator/jobs/downloadEtlExportedFile/{{jobId}}`) to `dest`.
|
|
49
|
+
Raises `JobError` (`ExportJobError`) on a failed/refused/cancelled job, on
|
|
50
|
+
an `error` answer, or after `timeout` seconds. Same roles as `{starter}()`.
|
|
51
|
+
`progress=True` shows one updating status line, a callable gets event dicts
|
|
52
|
+
(see `reportnet_api.jobs.progress_event`).
|
|
53
|
+
|
|
54
|
+
Written from the `MasterDataLakes` branch of the Reportnet 3 backend repository (the job
|
|
55
|
+
and the orchestrator's poll/download routes); untested live.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
_WAIT_FOR_JOB_DOC = """Wait for an async job to finish and return its last status answer.
|
|
59
|
+
|
|
60
|
+
Pass what `import_big_file_data()` (or any job-starting method) returned,
|
|
61
|
+
or its `pollingUrl`. Polls every `poll_interval` seconds; raises `JobError`
|
|
62
|
+
on a failed/refused/cancelled job or after `timeout` seconds. For an import,
|
|
63
|
+
`FINISHED` is the job's own completion; `check_import_process()` is not a
|
|
64
|
+
reliable signal (live-tested). `progress=True` shows one updating status
|
|
65
|
+
line, a callable gets event dicts (`reportnet_api.jobs.progress_event`).
|
|
66
|
+
Untested live.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
# After a big-file job reports FINISHED it is authoritative; statistics only get
|
|
70
|
+
# this long to show the change (they can lag, or stay equal for an empty/identical file).
|
|
71
|
+
_STATS_GRACE = 10.0
|
|
72
|
+
|
|
73
|
+
# Indirection so tests can drive import_file_and_wait's clock without patching `time`/`asyncio`.
|
|
74
|
+
_monotonic = time.monotonic
|
|
75
|
+
_sleep = time.sleep
|
|
76
|
+
_async_sleep = asyncio.sleep
|
|
77
|
+
|
|
78
|
+
_NO_JOB_MESSAGE = (
|
|
79
|
+
"The file was uploaded but the answer carried no pollingUrl to wait on ({job!r}); "
|
|
80
|
+
"the import may still complete, so check get_import_related_statistics() before retrying"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
_IMPORT_AND_WAIT_DOC = """Upload a file into a dataset and wait until the import is visible.
|
|
84
|
+
|
|
85
|
+
Reads `get_import_related_statistics()` as a baseline, uploads `file` with
|
|
86
|
+
`import_big_file_data()` (`big_file=True`, the default; async job) or
|
|
87
|
+
`import_file_data()` (`big_file=False`; returns nothing and merges in the
|
|
88
|
+
background), then waits:
|
|
89
|
+
|
|
90
|
+
- big file: polls the answer's `pollingUrl` until the job is `FINISHED`
|
|
91
|
+
(`wait_for_job()`); if the answer has no `pollingUrl` this step is skipped.
|
|
92
|
+
Once the job is `FINISHED` it is authoritative: statistics then get only a
|
|
93
|
+
short grace period to show the change, after which the latest statistics
|
|
94
|
+
are returned anyway (an empty or identical file may change nothing).
|
|
95
|
+
- otherwise (small file, or no `pollingUrl`): polls
|
|
96
|
+
`get_import_related_statistics()` until a table's
|
|
97
|
+
`(lastImportDate, numberOfRecordsImported)` differs from the baseline
|
|
98
|
+
(only `table_schema_id`'s table when given), else raises on timeout.
|
|
99
|
+
Pass `confirm_statistics=False` to trust the job alone (big file only; a
|
|
100
|
+
missing `pollingUrl` then raises, though the upload was already sent).
|
|
101
|
+
|
|
102
|
+
`check_import_process()` is never used: its lock stayed set long after
|
|
103
|
+
the data was in (live-tested). `list_imported_files()` is not used
|
|
104
|
+
either: a file is listed before its data is merged. Returns the final
|
|
105
|
+
statistics answer. Raises `JobError` on a failed/refused/cancelled job,
|
|
106
|
+
or when `timeout` seconds pass while waiting (the upload may still complete
|
|
107
|
+
afterwards; nothing is rolled back, so check before retrying). `timeout`
|
|
108
|
+
bounds the waiting, not the baseline read or the upload itself, and
|
|
109
|
+
`poll_interval` / `timeout` apply to both polling phases combined. Roles as
|
|
110
|
+
for the underlying import method. `progress=True` shows one updating status
|
|
111
|
+
line (a callable gets event dicts); its elapsed time counts from the start
|
|
112
|
+
of the waiting.
|
|
113
|
+
|
|
114
|
+
Limits of the statistics check: without `table_schema_id` any table's
|
|
115
|
+
change counts, so a concurrent import into the same dataset can be
|
|
116
|
+
mistaken for yours; and a `replace` that makes a table drop out of the
|
|
117
|
+
statistics is not seen as a change.
|
|
118
|
+
|
|
119
|
+
Verified live (2026-10-23, Bearer, scratch dataset): `import_big_file_data`
|
|
120
|
+
returns a `pollingUrl` carrying `datasetId`/`dataflowId`, the job reached
|
|
121
|
+
`FINISHED` in ~20s, and `lastImportDate` changed (the count stayed at 1:
|
|
122
|
+
`numberOfRecordsImported` looks like the last import's rows, not a table
|
|
123
|
+
total, so don't compare the count alone). The file name must match a table
|
|
124
|
+
name or the job is `CANCELED`. The imported row was confirmed in the table
|
|
125
|
+
via the v4 CSV export. An ApiKey import also worked (job
|
|
126
|
+
FINISHED, statistics changed). NOT verified live: the `big_file=False`
|
|
127
|
+
path with this helper.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
_EXPORT_DOCS = {
|
|
131
|
+
"zip_csv": _EXPORT_TO_FILE_DOC.format(
|
|
132
|
+
summary="Export a dataset as a ZIP of CSV files into `dest`, waiting for the job.",
|
|
133
|
+
starter="etl_export_zip_csv",
|
|
134
|
+
route="/dataset/v4/etlExport/{id}",
|
|
135
|
+
),
|
|
136
|
+
"zip_parquet": _EXPORT_TO_FILE_DOC.format(
|
|
137
|
+
summary="Export a dataset as a ZIP of Parquet files into `dest`, waiting for the job.",
|
|
138
|
+
starter="etl_export_zip_parquet",
|
|
139
|
+
route="/dataset/v5/etlExport/{id}",
|
|
140
|
+
),
|
|
141
|
+
"with_job": _EXPORT_TO_FILE_DOC.format(
|
|
142
|
+
summary="Export a dataset (v3 job variant) into `dest`, waiting for the job.",
|
|
143
|
+
starter="etl_export_dataset_with_job",
|
|
144
|
+
route="/dataset/v3/etlExport/{id}",
|
|
145
|
+
),
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
_DELETE_IMPORT_TABLE_DOC = """DELETE /dataset/{dataset_id}/deleteImportTable/{table_schema_id}
|
|
149
|
+
|
|
150
|
+
Deletes all data in one table. Hidden legacy (unversioned) alias of
|
|
151
|
+
`delete_table_data()` (`/dataset/v1/{id}/deleteTableData/{tableId}`):
|
|
152
|
+
`DatasetControllerImpl#deleteImportTableLegacy` just calls
|
|
153
|
+
`deleteTableData`. The Reportnet FME integration uses this route.
|
|
154
|
+
Destructive and not undoable. Returns `None` (the controller is `void`).
|
|
155
|
+
|
|
156
|
+
Role rules are the same as `delete_table_data()`; the hidden route
|
|
157
|
+
also accepts an ApiKey (`dataflow_id`/`provider_id` feed that check).
|
|
158
|
+
Untested live: the backend source audited is older than production.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
_FILE_EXPORT_NOTE = """
|
|
163
|
+
|
|
164
|
+
Not job-based: the backend starts generating the file in the background
|
|
165
|
+
and tells only the Reportnet UI (a user notification); there is no job
|
|
166
|
+
id or polling URL. `fetch_export_file()` / the `*_to_file()` helpers
|
|
167
|
+
therefore poll the download until a non-empty body arrives. A file left
|
|
168
|
+
from an earlier export with the same name can be served instead of the
|
|
169
|
+
new one (the normal download deletes the file, the `big_data` one does not).
|
|
170
|
+
Read from `DatasetControllerImpl` / `FileTreatmentHelper` (`MasterDataLakes`
|
|
171
|
+
branch; `develop` has no `big_data` variants). Live (2026-10-23, Bearer only: the
|
|
172
|
+
routes reject an ApiKey with 403) production behaves like the `big_data`
|
|
173
|
+
(`...DL`) routes, which is what the Reportnet UI calls; the plain routes
|
|
174
|
+
accepted a request but never produced a file. Hence `big_data=True` is the
|
|
175
|
+
default; for the dataset export use `zip csv` (plain `csv` produced no file).
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _file_route(name: str, big_data: bool) -> str:
|
|
180
|
+
return f"{name}DL" if big_data else name
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _table_export_request(
|
|
184
|
+
dataset_id: int,
|
|
185
|
+
mime_type: str,
|
|
186
|
+
table_schema_id: str | None,
|
|
187
|
+
level_error: list[str] | None,
|
|
188
|
+
id_rules: str | None,
|
|
189
|
+
field_value: str | None,
|
|
190
|
+
big_data: bool,
|
|
191
|
+
) -> tuple[str, dict[str, Any], dict[str, Any]]:
|
|
192
|
+
params: dict[str, Any] = {"datasetId": dataset_id, "mimeType": mime_type}
|
|
193
|
+
if table_schema_id is not None:
|
|
194
|
+
params["tableSchemaId"] = table_schema_id
|
|
195
|
+
# The big-data route reads these without a null check, so always send them.
|
|
196
|
+
body: dict[str, Any] = {
|
|
197
|
+
"levelError": level_error or [],
|
|
198
|
+
"idRules": id_rules or "",
|
|
199
|
+
"fieldValue": field_value or "",
|
|
200
|
+
}
|
|
201
|
+
if big_data:
|
|
202
|
+
body["qcCodes"] = "" # a String in the backend's ExportFilterVO (MasterDataLakes)
|
|
203
|
+
return "/dataset/" + _file_route("exportFile", big_data), params, body
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _check_extension(mime_type: str) -> None:
|
|
207
|
+
"""A table export is saved as `<table>.<mime_type>`, so it must be a bare extension."""
|
|
208
|
+
if (
|
|
209
|
+
not mime_type
|
|
210
|
+
or mime_type != mime_type.strip()
|
|
211
|
+
or " " in mime_type
|
|
212
|
+
or mime_type.startswith(".")
|
|
213
|
+
):
|
|
214
|
+
raise ValueError(
|
|
215
|
+
f"mime_type must be a bare file extension such as 'csv' or 'xlsx', got {mime_type!r}"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
_STALE_WARNING = (
|
|
220
|
+
"An export file named {name!r} already exists on the server and may be returned instead "
|
|
221
|
+
"of the new export; compare the result with the dataset if the data changed."
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _dataset_file_name(dataset_name: str, mime_type: str) -> str:
|
|
226
|
+
"""File the dataset export writes: `<name>.zip` for `zip csv`/`zip xlsx`, else `<name>.<ext>`."""
|
|
227
|
+
parts = mime_type.split()
|
|
228
|
+
return (
|
|
229
|
+
f"{dataset_name}.zip"
|
|
230
|
+
if len(parts) > 1 and parts[0] == "zip"
|
|
231
|
+
else f"{dataset_name}.{parts[-1]}"
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class DatasetsResource(_GeneratedDatasetsResource):
|
|
236
|
+
def import_big_file_data(
|
|
237
|
+
self,
|
|
238
|
+
dataset_id: int,
|
|
239
|
+
file: str | os.PathLike[str],
|
|
240
|
+
dataflow_id: int | None = None,
|
|
241
|
+
delimiter: str | None = None,
|
|
242
|
+
fme_job_id: str | None = None,
|
|
243
|
+
integration_id: int | None = None,
|
|
244
|
+
job_id: int | None = None,
|
|
245
|
+
provider_id: int | None = None,
|
|
246
|
+
replace: bool | None = None,
|
|
247
|
+
table_schema_id: str | None = None,
|
|
248
|
+
preparation_code: str | None = None,
|
|
249
|
+
*,
|
|
250
|
+
timeout: float | httpx.Timeout | None = None,
|
|
251
|
+
) -> dict[str, Any]:
|
|
252
|
+
"""Import file to dataset data (Large files)
|
|
253
|
+
|
|
254
|
+
Allowed roles:
|
|
255
|
+
|
|
256
|
+
Reporting dataset: LEAD REPORTER, REPORTER WRITE, NATIONAL COORDINATOR
|
|
257
|
+
|
|
258
|
+
Data collection: CUSTODIAN, STEWARD
|
|
259
|
+
|
|
260
|
+
Test dataset: CUSTODIAN, STEWARD, STEWARD SUPPORT
|
|
261
|
+
|
|
262
|
+
Reference dataset: CUSTODIAN, STEWARD
|
|
263
|
+
|
|
264
|
+
Design dataset: CUSTODIAN, STEWARD, EDITOR WRITE
|
|
265
|
+
|
|
266
|
+
EU dataset: CUSTODIAN, STEWARD
|
|
267
|
+
|
|
268
|
+
Overrides the generated method. An empty response body returns `{}` (data lake
|
|
269
|
+
datasets answer `void`: the import is done inside the request, MasterDataLakes `DatasetControllerImpl:391-422`;
|
|
270
|
+
other datasets answer a job map), and `timeout` overrides the client's timeout for
|
|
271
|
+
this call only (a lake import can run for minutes). `preparation_code` is sent as
|
|
272
|
+
`code`."""
|
|
273
|
+
params = {
|
|
274
|
+
"dataflowId": dataflow_id,
|
|
275
|
+
"delimiter": delimiter,
|
|
276
|
+
"fmeJobId": fme_job_id,
|
|
277
|
+
"integrationId": integration_id,
|
|
278
|
+
"jobId": job_id,
|
|
279
|
+
"providerId": provider_id,
|
|
280
|
+
"replace": replace,
|
|
281
|
+
"tableSchemaId": table_schema_id,
|
|
282
|
+
"code": preparation_code,
|
|
283
|
+
}
|
|
284
|
+
params = {k: v for k, v in params.items() if v is not None}
|
|
285
|
+
options: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
|
|
286
|
+
url = f"/dataset/v2/importFileData/{dataset_id}"
|
|
287
|
+
with stream_upload_file("file", file) as files:
|
|
288
|
+
response = self._client.request("POST", url, params=params, files=files, **options)
|
|
289
|
+
return cast(dict[str, Any], response.json()) if response.content.strip() else {}
|
|
290
|
+
|
|
291
|
+
def delete_import_table(
|
|
292
|
+
self,
|
|
293
|
+
dataset_id: int,
|
|
294
|
+
table_schema_id: str,
|
|
295
|
+
dataflow_id: int | None = None,
|
|
296
|
+
provider_id: int | None = None,
|
|
297
|
+
) -> None:
|
|
298
|
+
params = {"dataflowId": dataflow_id, "providerId": provider_id}
|
|
299
|
+
params = {k: v for k, v in params.items() if v is not None}
|
|
300
|
+
self._client.request(
|
|
301
|
+
"DELETE", f"/dataset/{dataset_id}/deleteImportTable/{table_schema_id}", params=params
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
delete_import_table.__doc__ = _DELETE_IMPORT_TABLE_DOC
|
|
305
|
+
|
|
306
|
+
def update_dataset_name(self, dataset_id: int, name: str) -> None:
|
|
307
|
+
"""PUT /datasetmetabase/updateDatasetName: rename a dataset (as the web
|
|
308
|
+
app renames a design dataset, i.e. its schema).
|
|
309
|
+
|
|
310
|
+
`DatasetMetabaseControllerImpl#updateDatasetName` (hidden). Requires
|
|
311
|
+
`DATASCHEMA_STEWARD`, `DATASCHEMA_CUSTODIAN` or `DATASCHEMA_EDITOR_WRITE`
|
|
312
|
+
on the dataset. The backend trims `name` and answers `400` unless it
|
|
313
|
+
matches `[a-zA-Z0-9\\s()_-]+`, or for an unknown dataset. It does not
|
|
314
|
+
check that the name is unique in the dataflow.
|
|
315
|
+
"""
|
|
316
|
+
self._client.request(
|
|
317
|
+
"PUT",
|
|
318
|
+
"/datasetmetabase/updateDatasetName",
|
|
319
|
+
params={"datasetId": dataset_id, "datasetName": name},
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
def wait_for_job(
|
|
323
|
+
self,
|
|
324
|
+
job: dict[str, Any] | str,
|
|
325
|
+
*,
|
|
326
|
+
poll_interval: float = 2.0,
|
|
327
|
+
timeout: float = 300.0,
|
|
328
|
+
progress: Progress = None,
|
|
329
|
+
) -> dict[str, Any]:
|
|
330
|
+
return wait_for_job(
|
|
331
|
+
self._client, job, poll_interval=poll_interval, timeout=timeout, progress=progress
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
wait_for_job.__doc__ = _WAIT_FOR_JOB_DOC
|
|
335
|
+
|
|
336
|
+
def import_file_and_wait(
|
|
337
|
+
self,
|
|
338
|
+
dataset_id: int,
|
|
339
|
+
file: str | os.PathLike[str],
|
|
340
|
+
*,
|
|
341
|
+
dataflow_id: int | None = None,
|
|
342
|
+
delimiter: str | None = None,
|
|
343
|
+
integration_id: int | None = None,
|
|
344
|
+
provider_id: int | None = None,
|
|
345
|
+
replace: bool | None = None,
|
|
346
|
+
table_schema_id: str | None = None,
|
|
347
|
+
big_file: bool = True,
|
|
348
|
+
confirm_statistics: bool = True,
|
|
349
|
+
poll_interval: float = 2.0,
|
|
350
|
+
timeout: float = 300.0,
|
|
351
|
+
progress: Progress = None,
|
|
352
|
+
) -> dict[str, Any]:
|
|
353
|
+
before = self.get_import_related_statistics(dataset_id)
|
|
354
|
+
options: dict[str, Any] = {
|
|
355
|
+
"dataflow_id": dataflow_id,
|
|
356
|
+
"delimiter": delimiter,
|
|
357
|
+
"integration_id": integration_id,
|
|
358
|
+
"provider_id": provider_id,
|
|
359
|
+
"replace": replace,
|
|
360
|
+
"table_schema_id": table_schema_id,
|
|
361
|
+
}
|
|
362
|
+
job_finished = False
|
|
363
|
+
with progress_scope(progress) as report:
|
|
364
|
+
job: Any = None # only a big-file upload answers with a job to wait on
|
|
365
|
+
if big_file:
|
|
366
|
+
job = self.import_big_file_data(dataset_id, file, **options)
|
|
367
|
+
else:
|
|
368
|
+
self.import_file_data(dataset_id, file, **options)
|
|
369
|
+
# `timeout` bounds the waiting, not the upload.
|
|
370
|
+
started = _monotonic()
|
|
371
|
+
deadline = started + timeout
|
|
372
|
+
if isinstance(job, dict) and job.get("pollingUrl"):
|
|
373
|
+
remaining = max(0.0, deadline - _monotonic())
|
|
374
|
+
wait_for_job(
|
|
375
|
+
self._client,
|
|
376
|
+
job,
|
|
377
|
+
poll_interval=poll_interval,
|
|
378
|
+
timeout=remaining,
|
|
379
|
+
progress=report,
|
|
380
|
+
)
|
|
381
|
+
job_finished = True
|
|
382
|
+
deadline = min(deadline, _monotonic() + _STATS_GRACE)
|
|
383
|
+
elif big_file and not confirm_statistics:
|
|
384
|
+
raise JobError(_NO_JOB_MESSAGE.format(job=job))
|
|
385
|
+
while True:
|
|
386
|
+
after = self.get_import_related_statistics(dataset_id)
|
|
387
|
+
if (big_file and not confirm_statistics) or import_statistics_changed(
|
|
388
|
+
before, after, table_schema_id
|
|
389
|
+
):
|
|
390
|
+
return after
|
|
391
|
+
if _monotonic() >= deadline:
|
|
392
|
+
if job_finished:
|
|
393
|
+
return after
|
|
394
|
+
raise JobError(f"Import not visible in statistics within {timeout:g} seconds")
|
|
395
|
+
if report:
|
|
396
|
+
elapsed = _monotonic() - started
|
|
397
|
+
report(progress_event("confirm", elapsed=elapsed))
|
|
398
|
+
_sleep(poll_interval)
|
|
399
|
+
|
|
400
|
+
import_file_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
|
|
401
|
+
|
|
402
|
+
def _finish_export(
|
|
403
|
+
self,
|
|
404
|
+
job: dict[str, Any],
|
|
405
|
+
dest: str | os.PathLike[str],
|
|
406
|
+
poll_interval: float,
|
|
407
|
+
timeout: float,
|
|
408
|
+
progress: Progress = None,
|
|
409
|
+
) -> None:
|
|
410
|
+
answer = wait_for_job(
|
|
411
|
+
self._client, job, poll_interval=poll_interval, timeout=timeout, progress=progress
|
|
412
|
+
)
|
|
413
|
+
self._client.download("GET", download_url_of(answer), dest=dest)
|
|
414
|
+
|
|
415
|
+
def export_zip_csv_to_file(
|
|
416
|
+
self,
|
|
417
|
+
dataset_id: int,
|
|
418
|
+
*,
|
|
419
|
+
dataflow_id: int,
|
|
420
|
+
dest: str | os.PathLike[str],
|
|
421
|
+
table_schema_id: str | None = None,
|
|
422
|
+
poll_interval: float = 2.0,
|
|
423
|
+
timeout: float = 300.0,
|
|
424
|
+
progress: Progress = None,
|
|
425
|
+
) -> None:
|
|
426
|
+
job = self.etl_export_zip_csv(
|
|
427
|
+
dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
|
|
428
|
+
)
|
|
429
|
+
self._finish_export(job, dest, poll_interval, timeout, progress)
|
|
430
|
+
|
|
431
|
+
export_zip_csv_to_file.__doc__ = _EXPORT_DOCS["zip_csv"]
|
|
432
|
+
|
|
433
|
+
def export_zip_parquet_to_file(
|
|
434
|
+
self,
|
|
435
|
+
dataset_id: int,
|
|
436
|
+
*,
|
|
437
|
+
dataflow_id: int,
|
|
438
|
+
dest: str | os.PathLike[str],
|
|
439
|
+
table_schema_id: str | None = None,
|
|
440
|
+
poll_interval: float = 2.0,
|
|
441
|
+
timeout: float = 300.0,
|
|
442
|
+
progress: Progress = None,
|
|
443
|
+
) -> None:
|
|
444
|
+
job = self.etl_export_zip_parquet(
|
|
445
|
+
dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
|
|
446
|
+
)
|
|
447
|
+
self._finish_export(job, dest, poll_interval, timeout, progress)
|
|
448
|
+
|
|
449
|
+
export_zip_parquet_to_file.__doc__ = _EXPORT_DOCS["zip_parquet"]
|
|
450
|
+
|
|
451
|
+
def export_dataset_with_job_to_file(
|
|
452
|
+
self,
|
|
453
|
+
dataset_id: int,
|
|
454
|
+
*,
|
|
455
|
+
dataflow_id: int,
|
|
456
|
+
dest: str | os.PathLike[str],
|
|
457
|
+
table_schema_id: str | None = None,
|
|
458
|
+
provider_id: int | None = None,
|
|
459
|
+
poll_interval: float = 2.0,
|
|
460
|
+
timeout: float = 300.0,
|
|
461
|
+
progress: Progress = None,
|
|
462
|
+
) -> None:
|
|
463
|
+
job = self.etl_export_dataset_with_job(
|
|
464
|
+
dataset_id,
|
|
465
|
+
table_schema_id=table_schema_id,
|
|
466
|
+
provider_id=provider_id,
|
|
467
|
+
dataflow_id=dataflow_id,
|
|
468
|
+
)
|
|
469
|
+
self._finish_export(job, dest, poll_interval, timeout, progress)
|
|
470
|
+
|
|
471
|
+
export_dataset_with_job_to_file.__doc__ = _EXPORT_DOCS["with_job"]
|
|
472
|
+
|
|
473
|
+
def start_table_export(
|
|
474
|
+
self,
|
|
475
|
+
dataset_id: int,
|
|
476
|
+
*,
|
|
477
|
+
mime_type: str = "csv",
|
|
478
|
+
table_schema_id: str | None = None,
|
|
479
|
+
level_error: list[str] | None = None,
|
|
480
|
+
id_rules: str | None = None,
|
|
481
|
+
field_value: str | None = None,
|
|
482
|
+
big_data: bool = True,
|
|
483
|
+
) -> None:
|
|
484
|
+
path, params, body = _table_export_request(
|
|
485
|
+
dataset_id, mime_type, table_schema_id, level_error, id_rules, field_value, big_data
|
|
486
|
+
)
|
|
487
|
+
self._client.request("POST", path, params=params, json=body)
|
|
488
|
+
|
|
489
|
+
def start_dataset_export(
|
|
490
|
+
self, dataset_id: int, *, mime_type: str = "csv", big_data: bool = True
|
|
491
|
+
) -> None:
|
|
492
|
+
self._client.request(
|
|
493
|
+
"GET",
|
|
494
|
+
f"/dataset/{dataset_id}/" + _file_route("exportDatasetFile", big_data),
|
|
495
|
+
params={"mimeType": mime_type},
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
def fetch_export_file(
|
|
499
|
+
self,
|
|
500
|
+
dataset_id: int,
|
|
501
|
+
file_name: str,
|
|
502
|
+
*,
|
|
503
|
+
dest: str | os.PathLike[str],
|
|
504
|
+
big_data: bool = True,
|
|
505
|
+
) -> bool:
|
|
506
|
+
part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
|
|
507
|
+
try:
|
|
508
|
+
self._client.download(
|
|
509
|
+
"GET",
|
|
510
|
+
f"/dataset/{dataset_id}/" + _file_route("downloadFile", big_data),
|
|
511
|
+
params={"fileName": file_name},
|
|
512
|
+
dest=part,
|
|
513
|
+
)
|
|
514
|
+
if not part.exists() or part.stat().st_size == 0:
|
|
515
|
+
return False
|
|
516
|
+
part.replace(dest)
|
|
517
|
+
return True
|
|
518
|
+
except NotFoundError:
|
|
519
|
+
return False # not generated yet
|
|
520
|
+
finally:
|
|
521
|
+
part.unlink(missing_ok=True)
|
|
522
|
+
|
|
523
|
+
def _warn_if_stale(self, dataset_id: int, file_name: str, big_data: bool) -> None:
|
|
524
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
525
|
+
probe = Path(tmp) / "probe"
|
|
526
|
+
if self.fetch_export_file(dataset_id, file_name, dest=probe, big_data=big_data):
|
|
527
|
+
warnings.warn(_STALE_WARNING.format(name=file_name), RuntimeWarning, stacklevel=4)
|
|
528
|
+
|
|
529
|
+
def _wait_for_export_file(
|
|
530
|
+
self,
|
|
531
|
+
dataset_id: int,
|
|
532
|
+
file_name: str,
|
|
533
|
+
dest: str | os.PathLike[str],
|
|
534
|
+
big_data: bool,
|
|
535
|
+
poll_interval: float,
|
|
536
|
+
timeout: float,
|
|
537
|
+
progress: Progress = None,
|
|
538
|
+
) -> None:
|
|
539
|
+
started = time.monotonic()
|
|
540
|
+
deadline = started + timeout
|
|
541
|
+
with progress_scope(progress) as report:
|
|
542
|
+
while not self.fetch_export_file(dataset_id, file_name, dest=dest, big_data=big_data):
|
|
543
|
+
remaining = deadline - time.monotonic()
|
|
544
|
+
if remaining <= 0:
|
|
545
|
+
raise JobError(
|
|
546
|
+
f"Export file {file_name!r} not ready within {timeout:g} seconds"
|
|
547
|
+
)
|
|
548
|
+
if report:
|
|
549
|
+
report(progress_event("file", elapsed=time.monotonic() - started))
|
|
550
|
+
time.sleep(min(poll_interval, remaining))
|
|
551
|
+
|
|
552
|
+
def export_table_to_file(
|
|
553
|
+
self,
|
|
554
|
+
dataset_id: int,
|
|
555
|
+
*,
|
|
556
|
+
table_name: str,
|
|
557
|
+
table_schema_id: str,
|
|
558
|
+
dest: str | os.PathLike[str],
|
|
559
|
+
mime_type: str = "csv",
|
|
560
|
+
level_error: list[str] | None = None,
|
|
561
|
+
id_rules: str | None = None,
|
|
562
|
+
field_value: str | None = None,
|
|
563
|
+
big_data: bool = True,
|
|
564
|
+
poll_interval: float = 2.0,
|
|
565
|
+
timeout: float = 300.0,
|
|
566
|
+
progress: Progress = None,
|
|
567
|
+
) -> None:
|
|
568
|
+
_check_extension(mime_type)
|
|
569
|
+
file_name = f"{table_name}.{mime_type}"
|
|
570
|
+
self._warn_if_stale(dataset_id, file_name, big_data)
|
|
571
|
+
self.start_table_export(
|
|
572
|
+
dataset_id,
|
|
573
|
+
mime_type=mime_type,
|
|
574
|
+
table_schema_id=table_schema_id,
|
|
575
|
+
level_error=level_error,
|
|
576
|
+
id_rules=id_rules,
|
|
577
|
+
field_value=field_value,
|
|
578
|
+
big_data=big_data,
|
|
579
|
+
)
|
|
580
|
+
self._wait_for_export_file(
|
|
581
|
+
dataset_id, file_name, dest, big_data, poll_interval, timeout, progress
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
def export_dataset_to_file(
|
|
585
|
+
self,
|
|
586
|
+
dataset_id: int,
|
|
587
|
+
*,
|
|
588
|
+
dataset_name: str,
|
|
589
|
+
dest: str | os.PathLike[str],
|
|
590
|
+
mime_type: str = "csv",
|
|
591
|
+
big_data: bool = True,
|
|
592
|
+
poll_interval: float = 2.0,
|
|
593
|
+
timeout: float = 300.0,
|
|
594
|
+
progress: Progress = None,
|
|
595
|
+
) -> None:
|
|
596
|
+
file_name = _dataset_file_name(dataset_name, mime_type)
|
|
597
|
+
self._warn_if_stale(dataset_id, file_name, big_data)
|
|
598
|
+
self.start_dataset_export(dataset_id, mime_type=mime_type, big_data=big_data)
|
|
599
|
+
self._wait_for_export_file(
|
|
600
|
+
dataset_id,
|
|
601
|
+
file_name,
|
|
602
|
+
dest,
|
|
603
|
+
big_data,
|
|
604
|
+
poll_interval,
|
|
605
|
+
timeout,
|
|
606
|
+
progress,
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
class AsyncDatasetsResource(_GeneratedAsyncDatasetsResource):
|
|
611
|
+
async def import_big_file_data(
|
|
612
|
+
self,
|
|
613
|
+
dataset_id: int,
|
|
614
|
+
file: str | os.PathLike[str],
|
|
615
|
+
dataflow_id: int | None = None,
|
|
616
|
+
delimiter: str | None = None,
|
|
617
|
+
fme_job_id: str | None = None,
|
|
618
|
+
integration_id: int | None = None,
|
|
619
|
+
job_id: int | None = None,
|
|
620
|
+
provider_id: int | None = None,
|
|
621
|
+
replace: bool | None = None,
|
|
622
|
+
table_schema_id: str | None = None,
|
|
623
|
+
preparation_code: str | None = None,
|
|
624
|
+
*,
|
|
625
|
+
timeout: float | httpx.Timeout | None = None,
|
|
626
|
+
) -> dict[str, Any]:
|
|
627
|
+
"""Import file to dataset data (Large files)
|
|
628
|
+
|
|
629
|
+
Allowed roles:
|
|
630
|
+
|
|
631
|
+
Reporting dataset: LEAD REPORTER, REPORTER WRITE, NATIONAL COORDINATOR
|
|
632
|
+
|
|
633
|
+
Data collection: CUSTODIAN, STEWARD
|
|
634
|
+
|
|
635
|
+
Test dataset: CUSTODIAN, STEWARD, STEWARD SUPPORT
|
|
636
|
+
|
|
637
|
+
Reference dataset: CUSTODIAN, STEWARD
|
|
638
|
+
|
|
639
|
+
Design dataset: CUSTODIAN, STEWARD, EDITOR WRITE
|
|
640
|
+
|
|
641
|
+
EU dataset: CUSTODIAN, STEWARD
|
|
642
|
+
|
|
643
|
+
Overrides the generated method. An empty response body returns `{}` (data lake
|
|
644
|
+
datasets answer `void`: the import is done inside the request, MasterDataLakes `DatasetControllerImpl:391-422`;
|
|
645
|
+
other datasets answer a job map), and `timeout` overrides the client's timeout for
|
|
646
|
+
this call only (a lake import can run for minutes). `preparation_code` is sent as
|
|
647
|
+
`code`."""
|
|
648
|
+
params = {
|
|
649
|
+
"dataflowId": dataflow_id,
|
|
650
|
+
"delimiter": delimiter,
|
|
651
|
+
"fmeJobId": fme_job_id,
|
|
652
|
+
"integrationId": integration_id,
|
|
653
|
+
"jobId": job_id,
|
|
654
|
+
"providerId": provider_id,
|
|
655
|
+
"replace": replace,
|
|
656
|
+
"tableSchemaId": table_schema_id,
|
|
657
|
+
"code": preparation_code,
|
|
658
|
+
}
|
|
659
|
+
params = {k: v for k, v in params.items() if v is not None}
|
|
660
|
+
options: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
|
|
661
|
+
url = f"/dataset/v2/importFileData/{dataset_id}"
|
|
662
|
+
with stream_upload_file("file", file) as files:
|
|
663
|
+
response = await self._client.request(
|
|
664
|
+
"POST", url, params=params, files=files, **options
|
|
665
|
+
)
|
|
666
|
+
return cast(dict[str, Any], response.json()) if response.content.strip() else {}
|
|
667
|
+
|
|
668
|
+
async def delete_import_table(
|
|
669
|
+
self,
|
|
670
|
+
dataset_id: int,
|
|
671
|
+
table_schema_id: str,
|
|
672
|
+
dataflow_id: int | None = None,
|
|
673
|
+
provider_id: int | None = None,
|
|
674
|
+
) -> None:
|
|
675
|
+
params = {"dataflowId": dataflow_id, "providerId": provider_id}
|
|
676
|
+
params = {k: v for k, v in params.items() if v is not None}
|
|
677
|
+
await self._client.request(
|
|
678
|
+
"DELETE", f"/dataset/{dataset_id}/deleteImportTable/{table_schema_id}", params=params
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
delete_import_table.__doc__ = _DELETE_IMPORT_TABLE_DOC
|
|
682
|
+
|
|
683
|
+
async def update_dataset_name(self, dataset_id: int, name: str) -> None:
|
|
684
|
+
"""Async counterpart of DatasetsResource.update_dataset_name."""
|
|
685
|
+
await self._client.request(
|
|
686
|
+
"PUT",
|
|
687
|
+
"/datasetmetabase/updateDatasetName",
|
|
688
|
+
params={"datasetId": dataset_id, "datasetName": name},
|
|
689
|
+
)
|
|
690
|
+
|
|
691
|
+
async def wait_for_job(
|
|
692
|
+
self, job: dict[str, Any] | str, *, poll_interval: float = 2.0, timeout: float = 300.0
|
|
693
|
+
) -> dict[str, Any]:
|
|
694
|
+
return await async_wait_for_job(
|
|
695
|
+
self._client, job, poll_interval=poll_interval, timeout=timeout
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
wait_for_job.__doc__ = _WAIT_FOR_JOB_DOC
|
|
699
|
+
|
|
700
|
+
async def import_file_and_wait(
|
|
701
|
+
self,
|
|
702
|
+
dataset_id: int,
|
|
703
|
+
file: str | os.PathLike[str],
|
|
704
|
+
*,
|
|
705
|
+
dataflow_id: int | None = None,
|
|
706
|
+
delimiter: str | None = None,
|
|
707
|
+
integration_id: int | None = None,
|
|
708
|
+
provider_id: int | None = None,
|
|
709
|
+
replace: bool | None = None,
|
|
710
|
+
table_schema_id: str | None = None,
|
|
711
|
+
big_file: bool = True,
|
|
712
|
+
confirm_statistics: bool = True,
|
|
713
|
+
poll_interval: float = 2.0,
|
|
714
|
+
timeout: float = 300.0,
|
|
715
|
+
) -> dict[str, Any]:
|
|
716
|
+
before = await self.get_import_related_statistics(dataset_id)
|
|
717
|
+
options: dict[str, Any] = {
|
|
718
|
+
"dataflow_id": dataflow_id,
|
|
719
|
+
"delimiter": delimiter,
|
|
720
|
+
"integration_id": integration_id,
|
|
721
|
+
"provider_id": provider_id,
|
|
722
|
+
"replace": replace,
|
|
723
|
+
"table_schema_id": table_schema_id,
|
|
724
|
+
}
|
|
725
|
+
job_finished = False
|
|
726
|
+
job: Any = None # only a big-file upload answers with a job to wait on
|
|
727
|
+
if big_file:
|
|
728
|
+
job = await self.import_big_file_data(dataset_id, file, **options)
|
|
729
|
+
else:
|
|
730
|
+
await self.import_file_data(dataset_id, file, **options)
|
|
731
|
+
# `timeout` bounds the waiting, not the upload.
|
|
732
|
+
deadline = _monotonic() + timeout
|
|
733
|
+
if isinstance(job, dict) and job.get("pollingUrl"):
|
|
734
|
+
remaining = max(0.0, deadline - _monotonic())
|
|
735
|
+
await async_wait_for_job(
|
|
736
|
+
self._client, job, poll_interval=poll_interval, timeout=remaining
|
|
737
|
+
)
|
|
738
|
+
job_finished = True
|
|
739
|
+
deadline = min(deadline, _monotonic() + _STATS_GRACE)
|
|
740
|
+
elif big_file and not confirm_statistics:
|
|
741
|
+
raise JobError(_NO_JOB_MESSAGE.format(job=job))
|
|
742
|
+
while True:
|
|
743
|
+
after = await self.get_import_related_statistics(dataset_id)
|
|
744
|
+
if (big_file and not confirm_statistics) or import_statistics_changed(
|
|
745
|
+
before, after, table_schema_id
|
|
746
|
+
):
|
|
747
|
+
return after
|
|
748
|
+
if _monotonic() >= deadline:
|
|
749
|
+
if job_finished:
|
|
750
|
+
return after
|
|
751
|
+
raise JobError(f"Import not visible in statistics within {timeout:g} seconds")
|
|
752
|
+
await _async_sleep(poll_interval)
|
|
753
|
+
|
|
754
|
+
import_file_and_wait.__doc__ = _IMPORT_AND_WAIT_DOC
|
|
755
|
+
|
|
756
|
+
async def _finish_export(
|
|
757
|
+
self,
|
|
758
|
+
job: dict[str, Any],
|
|
759
|
+
dest: str | os.PathLike[str],
|
|
760
|
+
poll_interval: float,
|
|
761
|
+
timeout: float,
|
|
762
|
+
) -> None:
|
|
763
|
+
answer = await async_wait_for_job(
|
|
764
|
+
self._client, job, poll_interval=poll_interval, timeout=timeout
|
|
765
|
+
)
|
|
766
|
+
await self._client.download("GET", download_url_of(answer), dest=dest)
|
|
767
|
+
|
|
768
|
+
async def export_zip_csv_to_file(
|
|
769
|
+
self,
|
|
770
|
+
dataset_id: int,
|
|
771
|
+
*,
|
|
772
|
+
dataflow_id: int,
|
|
773
|
+
dest: str | os.PathLike[str],
|
|
774
|
+
table_schema_id: str | None = None,
|
|
775
|
+
poll_interval: float = 2.0,
|
|
776
|
+
timeout: float = 300.0,
|
|
777
|
+
) -> None:
|
|
778
|
+
job = await self.etl_export_zip_csv(
|
|
779
|
+
dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
|
|
780
|
+
)
|
|
781
|
+
await self._finish_export(job, dest, poll_interval, timeout)
|
|
782
|
+
|
|
783
|
+
export_zip_csv_to_file.__doc__ = _EXPORT_DOCS["zip_csv"]
|
|
784
|
+
|
|
785
|
+
async def export_zip_parquet_to_file(
|
|
786
|
+
self,
|
|
787
|
+
dataset_id: int,
|
|
788
|
+
*,
|
|
789
|
+
dataflow_id: int,
|
|
790
|
+
dest: str | os.PathLike[str],
|
|
791
|
+
table_schema_id: str | None = None,
|
|
792
|
+
poll_interval: float = 2.0,
|
|
793
|
+
timeout: float = 300.0,
|
|
794
|
+
) -> None:
|
|
795
|
+
job = await self.etl_export_zip_parquet(
|
|
796
|
+
dataset_id, table_schema_id=table_schema_id, dataflow_id=dataflow_id
|
|
797
|
+
)
|
|
798
|
+
await self._finish_export(job, dest, poll_interval, timeout)
|
|
799
|
+
|
|
800
|
+
export_zip_parquet_to_file.__doc__ = _EXPORT_DOCS["zip_parquet"]
|
|
801
|
+
|
|
802
|
+
async def export_dataset_with_job_to_file(
|
|
803
|
+
self,
|
|
804
|
+
dataset_id: int,
|
|
805
|
+
*,
|
|
806
|
+
dataflow_id: int,
|
|
807
|
+
dest: str | os.PathLike[str],
|
|
808
|
+
table_schema_id: str | None = None,
|
|
809
|
+
provider_id: int | None = None,
|
|
810
|
+
poll_interval: float = 2.0,
|
|
811
|
+
timeout: float = 300.0,
|
|
812
|
+
) -> None:
|
|
813
|
+
job = await self.etl_export_dataset_with_job(
|
|
814
|
+
dataset_id,
|
|
815
|
+
table_schema_id=table_schema_id,
|
|
816
|
+
provider_id=provider_id,
|
|
817
|
+
dataflow_id=dataflow_id,
|
|
818
|
+
)
|
|
819
|
+
await self._finish_export(job, dest, poll_interval, timeout)
|
|
820
|
+
|
|
821
|
+
export_dataset_with_job_to_file.__doc__ = _EXPORT_DOCS["with_job"]
|
|
822
|
+
|
|
823
|
+
async def start_table_export(
|
|
824
|
+
self,
|
|
825
|
+
dataset_id: int,
|
|
826
|
+
*,
|
|
827
|
+
mime_type: str = "csv",
|
|
828
|
+
table_schema_id: str | None = None,
|
|
829
|
+
level_error: list[str] | None = None,
|
|
830
|
+
id_rules: str | None = None,
|
|
831
|
+
field_value: str | None = None,
|
|
832
|
+
big_data: bool = True,
|
|
833
|
+
) -> None:
|
|
834
|
+
path, params, body = _table_export_request(
|
|
835
|
+
dataset_id, mime_type, table_schema_id, level_error, id_rules, field_value, big_data
|
|
836
|
+
)
|
|
837
|
+
await self._client.request("POST", path, params=params, json=body)
|
|
838
|
+
|
|
839
|
+
async def start_dataset_export(
|
|
840
|
+
self, dataset_id: int, *, mime_type: str = "csv", big_data: bool = True
|
|
841
|
+
) -> None:
|
|
842
|
+
await self._client.request(
|
|
843
|
+
"GET",
|
|
844
|
+
f"/dataset/{dataset_id}/" + _file_route("exportDatasetFile", big_data),
|
|
845
|
+
params={"mimeType": mime_type},
|
|
846
|
+
)
|
|
847
|
+
|
|
848
|
+
async def fetch_export_file(
|
|
849
|
+
self,
|
|
850
|
+
dataset_id: int,
|
|
851
|
+
file_name: str,
|
|
852
|
+
*,
|
|
853
|
+
dest: str | os.PathLike[str],
|
|
854
|
+
big_data: bool = True,
|
|
855
|
+
) -> bool:
|
|
856
|
+
part = Path(f"{dest}.{uuid.uuid4().hex[:8]}.part")
|
|
857
|
+
try:
|
|
858
|
+
await self._client.download(
|
|
859
|
+
"GET",
|
|
860
|
+
f"/dataset/{dataset_id}/" + _file_route("downloadFile", big_data),
|
|
861
|
+
params={"fileName": file_name},
|
|
862
|
+
dest=part,
|
|
863
|
+
)
|
|
864
|
+
if not part.exists() or part.stat().st_size == 0:
|
|
865
|
+
return False
|
|
866
|
+
part.replace(dest)
|
|
867
|
+
return True
|
|
868
|
+
except NotFoundError:
|
|
869
|
+
return False # not generated yet
|
|
870
|
+
finally:
|
|
871
|
+
part.unlink(missing_ok=True)
|
|
872
|
+
|
|
873
|
+
async def _warn_if_stale(self, dataset_id: int, file_name: str, big_data: bool) -> None:
|
|
874
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
875
|
+
probe = Path(tmp) / "probe"
|
|
876
|
+
if await self.fetch_export_file(dataset_id, file_name, dest=probe, big_data=big_data):
|
|
877
|
+
warnings.warn(_STALE_WARNING.format(name=file_name), RuntimeWarning, stacklevel=4)
|
|
878
|
+
|
|
879
|
+
async def _wait_for_export_file(
|
|
880
|
+
self,
|
|
881
|
+
dataset_id: int,
|
|
882
|
+
file_name: str,
|
|
883
|
+
dest: str | os.PathLike[str],
|
|
884
|
+
big_data: bool,
|
|
885
|
+
poll_interval: float,
|
|
886
|
+
timeout: float,
|
|
887
|
+
) -> None:
|
|
888
|
+
deadline = time.monotonic() + timeout
|
|
889
|
+
while not await self.fetch_export_file(dataset_id, file_name, dest=dest, big_data=big_data):
|
|
890
|
+
remaining = deadline - time.monotonic()
|
|
891
|
+
if remaining <= 0:
|
|
892
|
+
raise JobError(f"Export file {file_name!r} not ready within {timeout:g} seconds")
|
|
893
|
+
await asyncio.sleep(min(poll_interval, remaining))
|
|
894
|
+
|
|
895
|
+
async def export_table_to_file(
|
|
896
|
+
self,
|
|
897
|
+
dataset_id: int,
|
|
898
|
+
*,
|
|
899
|
+
table_name: str,
|
|
900
|
+
table_schema_id: str,
|
|
901
|
+
dest: str | os.PathLike[str],
|
|
902
|
+
mime_type: str = "csv",
|
|
903
|
+
level_error: list[str] | None = None,
|
|
904
|
+
id_rules: str | None = None,
|
|
905
|
+
field_value: str | None = None,
|
|
906
|
+
big_data: bool = True,
|
|
907
|
+
poll_interval: float = 2.0,
|
|
908
|
+
timeout: float = 300.0,
|
|
909
|
+
) -> None:
|
|
910
|
+
_check_extension(mime_type)
|
|
911
|
+
file_name = f"{table_name}.{mime_type}"
|
|
912
|
+
await self._warn_if_stale(dataset_id, file_name, big_data)
|
|
913
|
+
await self.start_table_export(
|
|
914
|
+
dataset_id,
|
|
915
|
+
mime_type=mime_type,
|
|
916
|
+
table_schema_id=table_schema_id,
|
|
917
|
+
level_error=level_error,
|
|
918
|
+
id_rules=id_rules,
|
|
919
|
+
field_value=field_value,
|
|
920
|
+
big_data=big_data,
|
|
921
|
+
)
|
|
922
|
+
await self._wait_for_export_file(
|
|
923
|
+
dataset_id, file_name, dest, big_data, poll_interval, timeout
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
async def export_dataset_to_file(
|
|
927
|
+
self,
|
|
928
|
+
dataset_id: int,
|
|
929
|
+
*,
|
|
930
|
+
dataset_name: str,
|
|
931
|
+
dest: str | os.PathLike[str],
|
|
932
|
+
mime_type: str = "csv",
|
|
933
|
+
big_data: bool = True,
|
|
934
|
+
poll_interval: float = 2.0,
|
|
935
|
+
timeout: float = 300.0,
|
|
936
|
+
) -> None:
|
|
937
|
+
file_name = _dataset_file_name(dataset_name, mime_type)
|
|
938
|
+
await self._warn_if_stale(dataset_id, file_name, big_data)
|
|
939
|
+
await self.start_dataset_export(dataset_id, mime_type=mime_type, big_data=big_data)
|
|
940
|
+
await self._wait_for_export_file(
|
|
941
|
+
dataset_id,
|
|
942
|
+
file_name,
|
|
943
|
+
dest,
|
|
944
|
+
big_data,
|
|
945
|
+
poll_interval,
|
|
946
|
+
timeout,
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def _set_docs(docs: dict[str, str]) -> None:
|
|
951
|
+
for name, doc in docs.items():
|
|
952
|
+
getattr(DatasetsResource, name).__doc__ = doc
|
|
953
|
+
getattr(AsyncDatasetsResource, name).__doc__ = doc
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
_set_docs(
|
|
957
|
+
{
|
|
958
|
+
"start_table_export": (
|
|
959
|
+
"""POST /dataset/exportFile (or `exportFileDL` with `big_data=True`): start exporting one table.
|
|
960
|
+
|
|
961
|
+
`mime_type` is the file extension (`csv`, `xlsx`, ...); without
|
|
962
|
+
`table_schema_id` the whole dataset is exported under the dataset's name.
|
|
963
|
+
Optional filters (`ExportFilterVO`): `level_error` (names of the backend's
|
|
964
|
+
`ErrorTypeEnum`), `id_rules`, `field_value`; `big_data=True` also sends an
|
|
965
|
+
empty `qcCodes` string. Returns nothing; the file is
|
|
966
|
+
named `<table name>.<mime_type>`."""
|
|
967
|
+
+ _FILE_EXPORT_NOTE
|
|
968
|
+
),
|
|
969
|
+
"start_dataset_export": (
|
|
970
|
+
"""GET /dataset/{datasetId}/exportDatasetFile (or `exportDatasetFileDL`): start exporting the whole dataset.
|
|
971
|
+
|
|
972
|
+
`mime_type` is `csv`, `xlsx`, ... or `zip csv` / `zip xlsx` (a space; the
|
|
973
|
+
backend splits it). The file is `<dataset name>.zip` for the zip types,
|
|
974
|
+
else `<dataset name>.<ext>`. Returns nothing."""
|
|
975
|
+
+ _FILE_EXPORT_NOTE
|
|
976
|
+
),
|
|
977
|
+
"fetch_export_file": (
|
|
978
|
+
"""GET /dataset/{datasetId}/downloadFile (or `downloadFileDL`): download a generated export file.
|
|
979
|
+
|
|
980
|
+
Saves to `dest` and returns True, or returns False without leaving a file
|
|
981
|
+
when the backend answered with an empty body (file not generated yet or not
|
|
982
|
+
found: the backend swallows that error)."""
|
|
983
|
+
+ _FILE_EXPORT_NOTE
|
|
984
|
+
),
|
|
985
|
+
"export_table_to_file": (
|
|
986
|
+
"""Export one table to `dest`, waiting for the file: start the export, then poll the download.
|
|
987
|
+
|
|
988
|
+
`table_name` is the table's name (the backend names the file after it);
|
|
989
|
+
`table_schema_id` selects the table; `mime_type` must be a bare extension (`csv`, `xlsx`).
|
|
990
|
+
Raises `JobError` after `timeout` seconds without a file."""
|
|
991
|
+
+ _FILE_EXPORT_NOTE
|
|
992
|
+
),
|
|
993
|
+
"export_dataset_to_file": (
|
|
994
|
+
"""Export the whole dataset to `dest`, waiting for the file.
|
|
995
|
+
|
|
996
|
+
`dataset_name` is the dataset's name (the backend names the file after it).
|
|
997
|
+
Raises `JobError` after `timeout` seconds without a file."""
|
|
998
|
+
+ _FILE_EXPORT_NOTE
|
|
999
|
+
),
|
|
1000
|
+
}
|
|
1001
|
+
)
|