reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
reportnet_api/jobs.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
"""Hand-written polling for async Orchestrator jobs (`pollForJobStatus`).
|
|
2
|
+
|
|
3
|
+
Several endpoints (`etl_export_dataset_with_job`, `etl_export_zip_csv`,
|
|
4
|
+
`etl_export_zip_parquet`, `import_big_file_data`) only start a job and answer
|
|
5
|
+
`{"pollingUrl": "/orchestrator/jobs/pollForJobStatus/{jobId}?datasetId=..."}`.
|
|
6
|
+
`wait_for_job()` / `async_wait_for_job()` poll that URL until the job ends.
|
|
7
|
+
|
|
8
|
+
Grounded in the Reportnet 3 backend source (read via the GitHub API, 2026-10):
|
|
9
|
+
- `JobControllerImpl#pollForJobStatus` (branch `MasterDataLakes`) answers HTTP
|
|
10
|
+
200 with a map: `{"status": <JobStatusEnum value>}`, plus `"downloadUrl"`
|
|
11
|
+
(`/orchestrator/jobs/downloadEtlExportedFile/{jobId}?datasetId=&dataflowId=
|
|
12
|
+
[&providerId=]`) only when the job type is `FILE_EXPORT` and status is
|
|
13
|
+
`FINISHED`. An unknown job or any exception is also a 200, with
|
|
14
|
+
`{"error": "..."}` and no status. `datasetId` and `dataflowId` query params
|
|
15
|
+
are required, which is why the `pollingUrl` is used verbatim.
|
|
16
|
+
- `JobStatusEnum` (re-read 2026-10-23 on both branches): `MasterDataLakes` has
|
|
17
|
+
QUEUED, IN_PROGRESS, REFUSED, CANCELED, FAILED, FINISHED, CANCELED_BY_ADMIN;
|
|
18
|
+
`develop` has QUEUED, IN_PROGRESS, ABORTED, CANCELLED, FAILED, FINISHED and
|
|
19
|
+
no `pollForJobStatus` at all. Every value of either enum is handled on
|
|
20
|
+
purpose: QUEUED/IN_PROGRESS keep waiting, FINISHED succeeds, the rest fail.
|
|
21
|
+
A status outside both enums keeps waiting (forward-compatible) but warns.
|
|
22
|
+
- The poll route is guarded by `checkApiKey` over a long list of dataset/dataflow
|
|
23
|
+
roles (stewards, custodians, reporters, observers) and takes an optional
|
|
24
|
+
`providerId`; it is not custodian-only. A provider-scoped (reporter) ApiKey
|
|
25
|
+
only passes the guard if `providerId` is on the poll URL, so pass
|
|
26
|
+
`provider_id=` when starting the job (the API then embeds it in `pollingUrl`
|
|
27
|
+
and `downloadUrl`). Backend-source reading only, not tested live.
|
|
28
|
+
Seen live (2026-10-23, `import_big_file_data`): a refused import answered
|
|
29
|
+
`{"status": "CANCELED", "info": "File name does not match any table name..."}`;
|
|
30
|
+
`info` is appended to the `JobError` message.
|
|
31
|
+
Not tested live otherwise.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import asyncio
|
|
37
|
+
import re
|
|
38
|
+
import sys
|
|
39
|
+
import time
|
|
40
|
+
import warnings
|
|
41
|
+
from collections.abc import Callable, Iterator
|
|
42
|
+
from contextlib import contextmanager
|
|
43
|
+
from typing import TYPE_CHECKING, Any, TextIO
|
|
44
|
+
|
|
45
|
+
if TYPE_CHECKING:
|
|
46
|
+
from .async_client import AsyncReportnetClient
|
|
47
|
+
from .client import ReportnetClient
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class JobError(Exception):
|
|
51
|
+
"""An async job failed, was refused or cancelled, or did not finish in time."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Every JobStatusEnum value of both backend branches (see module docstring).
|
|
55
|
+
JOB_RUNNING_STATUSES = frozenset({"QUEUED", "IN_PROGRESS"})
|
|
56
|
+
JOB_FINISHED_STATUS = "FINISHED"
|
|
57
|
+
JOB_FAILED_STATUSES = frozenset(
|
|
58
|
+
{"FAILED", "REFUSED", "CANCELED", "CANCELED_BY_ADMIN", "ABORTED", "CANCELLED"}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def polling_url_of(job: dict[str, Any] | str) -> str:
|
|
63
|
+
"""The `pollingUrl` of a job-start answer (or the string itself)."""
|
|
64
|
+
if isinstance(job, str):
|
|
65
|
+
return job
|
|
66
|
+
url = job.get("pollingUrl")
|
|
67
|
+
if not url:
|
|
68
|
+
raise JobError(f"Request did not start a job: {job}")
|
|
69
|
+
return str(url)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _check(answer: dict[str, Any]) -> bool:
|
|
73
|
+
"""True once FINISHED, False while it runs; raises JobError when it can't finish."""
|
|
74
|
+
if answer.get("error"):
|
|
75
|
+
raise JobError(str(answer["error"]))
|
|
76
|
+
status = answer.get("status")
|
|
77
|
+
if status == JOB_FINISHED_STATUS:
|
|
78
|
+
return True
|
|
79
|
+
if status in JOB_FAILED_STATUSES:
|
|
80
|
+
info = answer.get("info")
|
|
81
|
+
raise JobError(f"Job {status}: {info}" if info else f"Job {status}")
|
|
82
|
+
if status not in JOB_RUNNING_STATUSES:
|
|
83
|
+
warnings.warn(
|
|
84
|
+
f"Unrecognised job status {status!r}; treating it as still running",
|
|
85
|
+
RuntimeWarning,
|
|
86
|
+
stacklevel=2,
|
|
87
|
+
)
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _timeout_error(timeout: float) -> JobError:
|
|
92
|
+
return JobError(f"Job did not finish within {timeout:g} seconds")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
ProgressEvent = dict[str, Any]
|
|
96
|
+
# What the waiting helpers accept: off (None/False), the built-in line (True), or
|
|
97
|
+
# your own function that receives one event dict per step.
|
|
98
|
+
Progress = bool | Callable[[ProgressEvent], None] | None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def progress_event(
|
|
102
|
+
kind: str,
|
|
103
|
+
*,
|
|
104
|
+
status: str | None = None,
|
|
105
|
+
elapsed: float,
|
|
106
|
+
job_id: str | None = None,
|
|
107
|
+
page: int | None = None,
|
|
108
|
+
rows: int | None = None,
|
|
109
|
+
) -> ProgressEvent:
|
|
110
|
+
"""The dict a progress callback receives.
|
|
111
|
+
|
|
112
|
+
`kind` is ``"job"`` (one poll of an Orchestrator job; `status` is its
|
|
113
|
+
status), ``"file"`` (waiting for an export file to appear), ``"confirm"``
|
|
114
|
+
(import finished, checking the statistics) or ``"page"`` (`read_tables`
|
|
115
|
+
fetched page `page`, `rows` rows so far). Keys not relevant to a kind are None.
|
|
116
|
+
"""
|
|
117
|
+
return {
|
|
118
|
+
"kind": kind,
|
|
119
|
+
"status": status,
|
|
120
|
+
"elapsed": elapsed,
|
|
121
|
+
"job_id": job_id,
|
|
122
|
+
"page": page,
|
|
123
|
+
"rows": rows,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _describe(event: ProgressEvent) -> str:
|
|
128
|
+
secs = f"{event['elapsed']:.0f}s"
|
|
129
|
+
if event["kind"] == "page":
|
|
130
|
+
rows = f", {event['rows']} rows so far" if event["rows"] is not None else ""
|
|
131
|
+
return f"Read page {event['page']}{rows} ({secs})"
|
|
132
|
+
if event["kind"] == "file":
|
|
133
|
+
return f"Waiting for the export file ({secs})"
|
|
134
|
+
if event["kind"] == "confirm":
|
|
135
|
+
return f"Import finished, confirming statistics ({secs})"
|
|
136
|
+
name = f"Job {event['job_id']}" if event["job_id"] else "Job"
|
|
137
|
+
return f"{name}: {event['status']} ({secs})"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class ProgressPrinter:
|
|
141
|
+
"""The built-in `progress=True` display: one line, updated in place.
|
|
142
|
+
|
|
143
|
+
On a terminal (and in Jupyter, where `\\r` redraws the output line) each
|
|
144
|
+
event rewrites the same line. Anywhere else (a log file, CI, a pipe) a new
|
|
145
|
+
line is printed only when the kind, status or page changes, so a 5-minute
|
|
146
|
+
wait does not print 150 lines. Call `close()` to end a live line.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(self, stream: TextIO | None = None, *, live: bool | None = None) -> None:
|
|
150
|
+
self._stream = stream
|
|
151
|
+
self._live = live
|
|
152
|
+
self._last_key: tuple[Any, ...] | None = None
|
|
153
|
+
self._width = 0
|
|
154
|
+
self._open = False
|
|
155
|
+
|
|
156
|
+
def _is_live(self, stream: TextIO) -> bool:
|
|
157
|
+
if self._live is not None:
|
|
158
|
+
return self._live
|
|
159
|
+
isatty = getattr(stream, "isatty", None)
|
|
160
|
+
# ipykernel's stream says isatty() False but does honour `\\r`.
|
|
161
|
+
return bool(isatty and isatty()) or "ipykernel" in sys.modules
|
|
162
|
+
|
|
163
|
+
def __call__(self, event: ProgressEvent) -> None:
|
|
164
|
+
stream = self._stream or sys.stdout
|
|
165
|
+
text = _describe(event)
|
|
166
|
+
if self._is_live(stream):
|
|
167
|
+
self._width = max(self._width, len(text))
|
|
168
|
+
stream.write(f"\r{text:<{self._width}}")
|
|
169
|
+
self._open = True
|
|
170
|
+
else:
|
|
171
|
+
key = (event["kind"], event["status"], event["page"])
|
|
172
|
+
if key == self._last_key:
|
|
173
|
+
return
|
|
174
|
+
self._last_key = key
|
|
175
|
+
stream.write(text + "\n")
|
|
176
|
+
stream.flush()
|
|
177
|
+
|
|
178
|
+
def close(self) -> None:
|
|
179
|
+
if self._open:
|
|
180
|
+
(self._stream or sys.stdout).write("\n")
|
|
181
|
+
self._open = False
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@contextmanager
|
|
185
|
+
def progress_scope(progress: Progress) -> Iterator[Callable[[ProgressEvent], None] | None]:
|
|
186
|
+
"""Resolve a `progress` argument to a callback (or None) for the duration of a wait.
|
|
187
|
+
|
|
188
|
+
`True` makes a `ProgressPrinter` and ends its line on exit; a callable is
|
|
189
|
+
passed through untouched, so a helper can resolve once and hand the result to
|
|
190
|
+
the helpers it calls.
|
|
191
|
+
"""
|
|
192
|
+
if progress is True:
|
|
193
|
+
printer = ProgressPrinter()
|
|
194
|
+
try:
|
|
195
|
+
yield printer
|
|
196
|
+
finally:
|
|
197
|
+
printer.close()
|
|
198
|
+
elif callable(progress):
|
|
199
|
+
yield progress
|
|
200
|
+
else:
|
|
201
|
+
yield None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def job_id_of(job: dict[str, Any] | str) -> str | None:
|
|
205
|
+
"""The job id of a job-start answer: its `jobId`, else parsed from the `pollingUrl`."""
|
|
206
|
+
if isinstance(job, dict) and job.get("jobId") is not None:
|
|
207
|
+
return str(job["jobId"])
|
|
208
|
+
match = re.search(r"pollForJobStatus/([^/?#]+)", polling_url_of(job))
|
|
209
|
+
return match.group(1) if match else None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _interrupted(job_id: str | None) -> None:
|
|
213
|
+
label = f"job {job_id}" if job_id else "the job"
|
|
214
|
+
print(
|
|
215
|
+
f"Interrupted: stopped waiting, but {label} keeps running on the server "
|
|
216
|
+
"(this client cannot cancel it). Check on it later with Job.wait()/wait_for_job().",
|
|
217
|
+
file=sys.stderr,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _wait(
|
|
222
|
+
client: ReportnetClient,
|
|
223
|
+
job: dict[str, Any] | str,
|
|
224
|
+
poll_interval: float,
|
|
225
|
+
timeout: float,
|
|
226
|
+
progress: Progress,
|
|
227
|
+
announce_interrupt: bool,
|
|
228
|
+
) -> dict[str, Any]:
|
|
229
|
+
url = polling_url_of(job)
|
|
230
|
+
job_id = job_id_of(job)
|
|
231
|
+
started = time.monotonic()
|
|
232
|
+
deadline = started + timeout
|
|
233
|
+
with progress_scope(progress) as report:
|
|
234
|
+
try:
|
|
235
|
+
while True:
|
|
236
|
+
answer: dict[str, Any] = client.request("GET", url).json()
|
|
237
|
+
if report:
|
|
238
|
+
status = answer.get("status") or ("ERROR" if answer.get("error") else None)
|
|
239
|
+
elapsed = time.monotonic() - started
|
|
240
|
+
report(progress_event("job", status=status, elapsed=elapsed, job_id=job_id))
|
|
241
|
+
if _check(answer):
|
|
242
|
+
return answer
|
|
243
|
+
if time.monotonic() >= deadline:
|
|
244
|
+
raise _timeout_error(timeout)
|
|
245
|
+
time.sleep(poll_interval)
|
|
246
|
+
except KeyboardInterrupt:
|
|
247
|
+
if announce_interrupt or report:
|
|
248
|
+
_interrupted(job_id)
|
|
249
|
+
raise
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def wait_for_job(
|
|
253
|
+
client: ReportnetClient,
|
|
254
|
+
job: dict[str, Any] | str,
|
|
255
|
+
*,
|
|
256
|
+
poll_interval: float = 2.0,
|
|
257
|
+
timeout: float = 300.0,
|
|
258
|
+
progress: Progress = None,
|
|
259
|
+
) -> dict[str, Any]:
|
|
260
|
+
"""Poll a job's `pollingUrl` until FINISHED and return the last answer.
|
|
261
|
+
|
|
262
|
+
`job` is the dict a job-starting method returned (or its `pollingUrl`).
|
|
263
|
+
Raises `JobError` on a failure status, an `error` answer, or after
|
|
264
|
+
`timeout` seconds. `progress=True` shows one updating status line; a
|
|
265
|
+
callable receives an event dict per poll (see `progress_event()`). With
|
|
266
|
+
progress on, Ctrl-C prints the job id and that the job keeps running.
|
|
267
|
+
"""
|
|
268
|
+
return _wait(client, job, poll_interval, timeout, progress, False)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class Job:
|
|
272
|
+
"""A started Orchestrator job you can check on later.
|
|
273
|
+
|
|
274
|
+
Build one from what a job-starting method returned, do other work, then
|
|
275
|
+
ask `.status` or call `.wait()`::
|
|
276
|
+
|
|
277
|
+
job = Job.from_polling_url(client, client.datasets.etl_export_zip_csv(...))
|
|
278
|
+
... # other work
|
|
279
|
+
answer = job.wait(progress=True)
|
|
280
|
+
|
|
281
|
+
Nothing is polled until you ask. The job lives on the server; dropping the
|
|
282
|
+
object does not stop it (and this client cannot cancel one).
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
def __init__(
|
|
286
|
+
self,
|
|
287
|
+
client: ReportnetClient,
|
|
288
|
+
polling_url: str,
|
|
289
|
+
*,
|
|
290
|
+
job_id: str | None = None,
|
|
291
|
+
kind: str | None = None,
|
|
292
|
+
) -> None:
|
|
293
|
+
self._client = client
|
|
294
|
+
self.polling_url = polling_url
|
|
295
|
+
self.job_id = job_id if job_id is not None else job_id_of(polling_url)
|
|
296
|
+
self.kind = kind
|
|
297
|
+
self.last_status: str | None = None
|
|
298
|
+
|
|
299
|
+
@classmethod
|
|
300
|
+
def from_polling_url(
|
|
301
|
+
cls, client: ReportnetClient, job: dict[str, Any] | str, *, kind: str | None = None
|
|
302
|
+
) -> Job:
|
|
303
|
+
"""From a job-start answer (its `pollingUrl`, `jobId` if present) or a bare URL."""
|
|
304
|
+
return cls(client, polling_url_of(job), job_id=job_id_of(job), kind=kind)
|
|
305
|
+
|
|
306
|
+
def poll(self) -> dict[str, Any]:
|
|
307
|
+
"""One poll: the full status answer. Read-only; remembers the status for the repr."""
|
|
308
|
+
answer: dict[str, Any] = self._client.request("GET", self.polling_url).json()
|
|
309
|
+
self.last_status = answer.get("status") or self.last_status
|
|
310
|
+
return answer
|
|
311
|
+
|
|
312
|
+
@property
|
|
313
|
+
def status(self) -> str | None:
|
|
314
|
+
"""The job's status from one poll (e.g. ``"IN_PROGRESS"``), None if it names none."""
|
|
315
|
+
status = self.poll().get("status")
|
|
316
|
+
return str(status) if status is not None else None
|
|
317
|
+
|
|
318
|
+
def wait(
|
|
319
|
+
self, *, poll_interval: float = 2.0, timeout: float = 300.0, progress: Progress = None
|
|
320
|
+
) -> dict[str, Any]:
|
|
321
|
+
"""Like `wait_for_job()` for this job; Ctrl-C always reports the job id."""
|
|
322
|
+
answer = _wait(self._client, self.polling_url, poll_interval, timeout, progress, True)
|
|
323
|
+
self.last_status = answer.get("status") or self.last_status
|
|
324
|
+
return answer
|
|
325
|
+
|
|
326
|
+
def __repr__(self) -> str:
|
|
327
|
+
parts = [f"id={self.job_id}" if self.job_id else f"url={self.polling_url!r}"]
|
|
328
|
+
if self.kind:
|
|
329
|
+
parts.append(f"kind={self.kind}")
|
|
330
|
+
parts.append(f"last_status={self.last_status}")
|
|
331
|
+
return f"Job({', '.join(parts)})"
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def async_wait_for_job(
|
|
335
|
+
client: AsyncReportnetClient,
|
|
336
|
+
job: dict[str, Any] | str,
|
|
337
|
+
*,
|
|
338
|
+
poll_interval: float = 2.0,
|
|
339
|
+
timeout: float = 300.0,
|
|
340
|
+
) -> dict[str, Any]:
|
|
341
|
+
"""Async twin of `wait_for_job()`."""
|
|
342
|
+
url = polling_url_of(job)
|
|
343
|
+
deadline = time.monotonic() + timeout
|
|
344
|
+
while True:
|
|
345
|
+
answer: dict[str, Any] = (await client.request("GET", url)).json()
|
|
346
|
+
if _check(answer):
|
|
347
|
+
return answer
|
|
348
|
+
if time.monotonic() >= deadline:
|
|
349
|
+
raise _timeout_error(timeout)
|
|
350
|
+
await asyncio.sleep(poll_interval)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def download_url_of(answer: dict[str, Any]) -> str:
|
|
354
|
+
"""The `downloadUrl` of a finished export job."""
|
|
355
|
+
url = answer.get("downloadUrl")
|
|
356
|
+
if not url:
|
|
357
|
+
raise JobError("Job finished without a download URL")
|
|
358
|
+
return str(url)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def import_statistics_changed(
|
|
362
|
+
before: dict[str, Any], after: dict[str, Any], table_schema_id: str | None = None
|
|
363
|
+
) -> bool:
|
|
364
|
+
"""True if `get_import_related_statistics()` shows a new import since `before`.
|
|
365
|
+
|
|
366
|
+
The answer maps table schema id -> `{"lastImportDate", "numberOfRecordsImported",
|
|
367
|
+
"fileExtension"}`. An import counts as visible once a table's
|
|
368
|
+
`(lastImportDate, numberOfRecordsImported)` pair differs from the baseline (or
|
|
369
|
+
the table is new); comparing the pair, not the count alone, also catches a
|
|
370
|
+
re-import that leaves the count unchanged. With `table_schema_id` only that
|
|
371
|
+
table is checked, otherwise any table.
|
|
372
|
+
"""
|
|
373
|
+
keys = [table_schema_id] if table_schema_id else list(after)
|
|
374
|
+
for key in keys:
|
|
375
|
+
now = after.get(key)
|
|
376
|
+
if not isinstance(now, dict):
|
|
377
|
+
continue
|
|
378
|
+
was = before.get(key)
|
|
379
|
+
was = was if isinstance(was, dict) else {}
|
|
380
|
+
pair = ("lastImportDate", "numberOfRecordsImported")
|
|
381
|
+
if tuple(now.get(k) for k in pair) != tuple(was.get(k) for k in pair):
|
|
382
|
+
return True
|
|
383
|
+
return False
|
|
File without changes
|