reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reportnet_api/jobs.py ADDED
@@ -0,0 +1,383 @@
1
+ """Hand-written polling for async Orchestrator jobs (`pollForJobStatus`).
2
+
3
+ Several endpoints (`etl_export_dataset_with_job`, `etl_export_zip_csv`,
4
+ `etl_export_zip_parquet`, `import_big_file_data`) only start a job and answer
5
+ `{"pollingUrl": "/orchestrator/jobs/pollForJobStatus/{jobId}?datasetId=..."}`.
6
+ `wait_for_job()` / `async_wait_for_job()` poll that URL until the job ends.
7
+
8
+ Grounded in the Reportnet 3 backend source (read via the GitHub API, 2026-10):
9
+ - `JobControllerImpl#pollForJobStatus` (branch `MasterDataLakes`) answers HTTP
10
+ 200 with a map: `{"status": <JobStatusEnum value>}`, plus `"downloadUrl"`
11
+ (`/orchestrator/jobs/downloadEtlExportedFile/{jobId}?datasetId=&dataflowId=
12
+ [&providerId=]`) only when the job type is `FILE_EXPORT` and status is
13
+ `FINISHED`. An unknown job or any exception is also a 200, with
14
+ `{"error": "..."}` and no status. `datasetId` and `dataflowId` query params
15
+ are required, which is why the `pollingUrl` is used verbatim.
16
+ - `JobStatusEnum` (re-read 2026-10-23 on both branches): `MasterDataLakes` has
17
+ QUEUED, IN_PROGRESS, REFUSED, CANCELED, FAILED, FINISHED, CANCELED_BY_ADMIN;
18
+ `develop` has QUEUED, IN_PROGRESS, ABORTED, CANCELLED, FAILED, FINISHED and
19
+ no `pollForJobStatus` at all. Every value of either enum is handled on
20
+ purpose: QUEUED/IN_PROGRESS keep waiting, FINISHED succeeds, the rest fail.
21
+ A status outside both enums keeps waiting (forward-compatible) but warns.
22
+ - The poll route is guarded by `checkApiKey` over a long list of dataset/dataflow
23
+ roles (stewards, custodians, reporters, observers) and takes an optional
24
+ `providerId`; it is not custodian-only. A provider-scoped (reporter) ApiKey
25
+ only passes the guard if `providerId` is on the poll URL, so pass
26
+ `provider_id=` when starting the job (the API then embeds it in `pollingUrl`
27
+ and `downloadUrl`). Backend-source reading only, not tested live.
28
+ Seen live (2026-10-23, `import_big_file_data`): a refused import answered
29
+ `{"status": "CANCELED", "info": "File name does not match any table name..."}`;
30
+ `info` is appended to the `JobError` message.
31
+ Not tested live otherwise.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import asyncio
37
+ import re
38
+ import sys
39
+ import time
40
+ import warnings
41
+ from collections.abc import Callable, Iterator
42
+ from contextlib import contextmanager
43
+ from typing import TYPE_CHECKING, Any, TextIO
44
+
45
+ if TYPE_CHECKING:
46
+ from .async_client import AsyncReportnetClient
47
+ from .client import ReportnetClient
48
+
49
+
50
+ class JobError(Exception):
51
+ """An async job failed, was refused or cancelled, or did not finish in time."""
52
+
53
+
54
+ # Every JobStatusEnum value of both backend branches (see module docstring).
55
+ JOB_RUNNING_STATUSES = frozenset({"QUEUED", "IN_PROGRESS"})
56
+ JOB_FINISHED_STATUS = "FINISHED"
57
+ JOB_FAILED_STATUSES = frozenset(
58
+ {"FAILED", "REFUSED", "CANCELED", "CANCELED_BY_ADMIN", "ABORTED", "CANCELLED"}
59
+ )
60
+
61
+
62
+ def polling_url_of(job: dict[str, Any] | str) -> str:
63
+ """The `pollingUrl` of a job-start answer (or the string itself)."""
64
+ if isinstance(job, str):
65
+ return job
66
+ url = job.get("pollingUrl")
67
+ if not url:
68
+ raise JobError(f"Request did not start a job: {job}")
69
+ return str(url)
70
+
71
+
72
+ def _check(answer: dict[str, Any]) -> bool:
73
+ """True once FINISHED, False while it runs; raises JobError when it can't finish."""
74
+ if answer.get("error"):
75
+ raise JobError(str(answer["error"]))
76
+ status = answer.get("status")
77
+ if status == JOB_FINISHED_STATUS:
78
+ return True
79
+ if status in JOB_FAILED_STATUSES:
80
+ info = answer.get("info")
81
+ raise JobError(f"Job {status}: {info}" if info else f"Job {status}")
82
+ if status not in JOB_RUNNING_STATUSES:
83
+ warnings.warn(
84
+ f"Unrecognised job status {status!r}; treating it as still running",
85
+ RuntimeWarning,
86
+ stacklevel=2,
87
+ )
88
+ return False
89
+
90
+
91
+ def _timeout_error(timeout: float) -> JobError:
92
+ return JobError(f"Job did not finish within {timeout:g} seconds")
93
+
94
+
95
+ ProgressEvent = dict[str, Any]
96
+ # What the waiting helpers accept: off (None/False), the built-in line (True), or
97
+ # your own function that receives one event dict per step.
98
+ Progress = bool | Callable[[ProgressEvent], None] | None
99
+
100
+
101
+ def progress_event(
102
+ kind: str,
103
+ *,
104
+ status: str | None = None,
105
+ elapsed: float,
106
+ job_id: str | None = None,
107
+ page: int | None = None,
108
+ rows: int | None = None,
109
+ ) -> ProgressEvent:
110
+ """The dict a progress callback receives.
111
+
112
+ `kind` is ``"job"`` (one poll of an Orchestrator job; `status` is its
113
+ status), ``"file"`` (waiting for an export file to appear), ``"confirm"``
114
+ (import finished, checking the statistics) or ``"page"`` (`read_tables`
115
+ fetched page `page`, `rows` rows so far). Keys not relevant to a kind are None.
116
+ """
117
+ return {
118
+ "kind": kind,
119
+ "status": status,
120
+ "elapsed": elapsed,
121
+ "job_id": job_id,
122
+ "page": page,
123
+ "rows": rows,
124
+ }
125
+
126
+
127
+ def _describe(event: ProgressEvent) -> str:
128
+ secs = f"{event['elapsed']:.0f}s"
129
+ if event["kind"] == "page":
130
+ rows = f", {event['rows']} rows so far" if event["rows"] is not None else ""
131
+ return f"Read page {event['page']}{rows} ({secs})"
132
+ if event["kind"] == "file":
133
+ return f"Waiting for the export file ({secs})"
134
+ if event["kind"] == "confirm":
135
+ return f"Import finished, confirming statistics ({secs})"
136
+ name = f"Job {event['job_id']}" if event["job_id"] else "Job"
137
+ return f"{name}: {event['status']} ({secs})"
138
+
139
+
140
+ class ProgressPrinter:
141
+ """The built-in `progress=True` display: one line, updated in place.
142
+
143
+ On a terminal (and in Jupyter, where `\\r` redraws the output line) each
144
+ event rewrites the same line. Anywhere else (a log file, CI, a pipe) a new
145
+ line is printed only when the kind, status or page changes, so a 5-minute
146
+ wait does not print 150 lines. Call `close()` to end a live line.
147
+ """
148
+
149
+ def __init__(self, stream: TextIO | None = None, *, live: bool | None = None) -> None:
150
+ self._stream = stream
151
+ self._live = live
152
+ self._last_key: tuple[Any, ...] | None = None
153
+ self._width = 0
154
+ self._open = False
155
+
156
+ def _is_live(self, stream: TextIO) -> bool:
157
+ if self._live is not None:
158
+ return self._live
159
+ isatty = getattr(stream, "isatty", None)
160
+ # ipykernel's stream says isatty() False but does honour `\\r`.
161
+ return bool(isatty and isatty()) or "ipykernel" in sys.modules
162
+
163
+ def __call__(self, event: ProgressEvent) -> None:
164
+ stream = self._stream or sys.stdout
165
+ text = _describe(event)
166
+ if self._is_live(stream):
167
+ self._width = max(self._width, len(text))
168
+ stream.write(f"\r{text:<{self._width}}")
169
+ self._open = True
170
+ else:
171
+ key = (event["kind"], event["status"], event["page"])
172
+ if key == self._last_key:
173
+ return
174
+ self._last_key = key
175
+ stream.write(text + "\n")
176
+ stream.flush()
177
+
178
+ def close(self) -> None:
179
+ if self._open:
180
+ (self._stream or sys.stdout).write("\n")
181
+ self._open = False
182
+
183
+
184
+ @contextmanager
185
+ def progress_scope(progress: Progress) -> Iterator[Callable[[ProgressEvent], None] | None]:
186
+ """Resolve a `progress` argument to a callback (or None) for the duration of a wait.
187
+
188
+ `True` makes a `ProgressPrinter` and ends its line on exit; a callable is
189
+ passed through untouched, so a helper can resolve once and hand the result to
190
+ the helpers it calls.
191
+ """
192
+ if progress is True:
193
+ printer = ProgressPrinter()
194
+ try:
195
+ yield printer
196
+ finally:
197
+ printer.close()
198
+ elif callable(progress):
199
+ yield progress
200
+ else:
201
+ yield None
202
+
203
+
204
+ def job_id_of(job: dict[str, Any] | str) -> str | None:
205
+ """The job id of a job-start answer: its `jobId`, else parsed from the `pollingUrl`."""
206
+ if isinstance(job, dict) and job.get("jobId") is not None:
207
+ return str(job["jobId"])
208
+ match = re.search(r"pollForJobStatus/([^/?#]+)", polling_url_of(job))
209
+ return match.group(1) if match else None
210
+
211
+
212
+ def _interrupted(job_id: str | None) -> None:
213
+ label = f"job {job_id}" if job_id else "the job"
214
+ print(
215
+ f"Interrupted: stopped waiting, but {label} keeps running on the server "
216
+ "(this client cannot cancel it). Check on it later with Job.wait()/wait_for_job().",
217
+ file=sys.stderr,
218
+ )
219
+
220
+
221
+ def _wait(
222
+ client: ReportnetClient,
223
+ job: dict[str, Any] | str,
224
+ poll_interval: float,
225
+ timeout: float,
226
+ progress: Progress,
227
+ announce_interrupt: bool,
228
+ ) -> dict[str, Any]:
229
+ url = polling_url_of(job)
230
+ job_id = job_id_of(job)
231
+ started = time.monotonic()
232
+ deadline = started + timeout
233
+ with progress_scope(progress) as report:
234
+ try:
235
+ while True:
236
+ answer: dict[str, Any] = client.request("GET", url).json()
237
+ if report:
238
+ status = answer.get("status") or ("ERROR" if answer.get("error") else None)
239
+ elapsed = time.monotonic() - started
240
+ report(progress_event("job", status=status, elapsed=elapsed, job_id=job_id))
241
+ if _check(answer):
242
+ return answer
243
+ if time.monotonic() >= deadline:
244
+ raise _timeout_error(timeout)
245
+ time.sleep(poll_interval)
246
+ except KeyboardInterrupt:
247
+ if announce_interrupt or report:
248
+ _interrupted(job_id)
249
+ raise
250
+
251
+
252
+ def wait_for_job(
253
+ client: ReportnetClient,
254
+ job: dict[str, Any] | str,
255
+ *,
256
+ poll_interval: float = 2.0,
257
+ timeout: float = 300.0,
258
+ progress: Progress = None,
259
+ ) -> dict[str, Any]:
260
+ """Poll a job's `pollingUrl` until FINISHED and return the last answer.
261
+
262
+ `job` is the dict a job-starting method returned (or its `pollingUrl`).
263
+ Raises `JobError` on a failure status, an `error` answer, or after
264
+ `timeout` seconds. `progress=True` shows one updating status line; a
265
+ callable receives an event dict per poll (see `progress_event()`). With
266
+ progress on, Ctrl-C prints the job id and that the job keeps running.
267
+ """
268
+ return _wait(client, job, poll_interval, timeout, progress, False)
269
+
270
+
271
+ class Job:
272
+ """A started Orchestrator job you can check on later.
273
+
274
+ Build one from what a job-starting method returned, do other work, then
275
+ ask `.status` or call `.wait()`::
276
+
277
+ job = Job.from_polling_url(client, client.datasets.etl_export_zip_csv(...))
278
+ ... # other work
279
+ answer = job.wait(progress=True)
280
+
281
+ Nothing is polled until you ask. The job lives on the server; dropping the
282
+ object does not stop it (and this client cannot cancel one).
283
+ """
284
+
285
+ def __init__(
286
+ self,
287
+ client: ReportnetClient,
288
+ polling_url: str,
289
+ *,
290
+ job_id: str | None = None,
291
+ kind: str | None = None,
292
+ ) -> None:
293
+ self._client = client
294
+ self.polling_url = polling_url
295
+ self.job_id = job_id if job_id is not None else job_id_of(polling_url)
296
+ self.kind = kind
297
+ self.last_status: str | None = None
298
+
299
+ @classmethod
300
+ def from_polling_url(
301
+ cls, client: ReportnetClient, job: dict[str, Any] | str, *, kind: str | None = None
302
+ ) -> Job:
303
+ """From a job-start answer (its `pollingUrl`, `jobId` if present) or a bare URL."""
304
+ return cls(client, polling_url_of(job), job_id=job_id_of(job), kind=kind)
305
+
306
+ def poll(self) -> dict[str, Any]:
307
+ """One poll: the full status answer. Read-only; remembers the status for the repr."""
308
+ answer: dict[str, Any] = self._client.request("GET", self.polling_url).json()
309
+ self.last_status = answer.get("status") or self.last_status
310
+ return answer
311
+
312
+ @property
313
+ def status(self) -> str | None:
314
+ """The job's status from one poll (e.g. ``"IN_PROGRESS"``), None if it names none."""
315
+ status = self.poll().get("status")
316
+ return str(status) if status is not None else None
317
+
318
+ def wait(
319
+ self, *, poll_interval: float = 2.0, timeout: float = 300.0, progress: Progress = None
320
+ ) -> dict[str, Any]:
321
+ """Like `wait_for_job()` for this job; Ctrl-C always reports the job id."""
322
+ answer = _wait(self._client, self.polling_url, poll_interval, timeout, progress, True)
323
+ self.last_status = answer.get("status") or self.last_status
324
+ return answer
325
+
326
+ def __repr__(self) -> str:
327
+ parts = [f"id={self.job_id}" if self.job_id else f"url={self.polling_url!r}"]
328
+ if self.kind:
329
+ parts.append(f"kind={self.kind}")
330
+ parts.append(f"last_status={self.last_status}")
331
+ return f"Job({', '.join(parts)})"
332
+
333
+
334
+ async def async_wait_for_job(
335
+ client: AsyncReportnetClient,
336
+ job: dict[str, Any] | str,
337
+ *,
338
+ poll_interval: float = 2.0,
339
+ timeout: float = 300.0,
340
+ ) -> dict[str, Any]:
341
+ """Async twin of `wait_for_job()`."""
342
+ url = polling_url_of(job)
343
+ deadline = time.monotonic() + timeout
344
+ while True:
345
+ answer: dict[str, Any] = (await client.request("GET", url)).json()
346
+ if _check(answer):
347
+ return answer
348
+ if time.monotonic() >= deadline:
349
+ raise _timeout_error(timeout)
350
+ await asyncio.sleep(poll_interval)
351
+
352
+
353
+ def download_url_of(answer: dict[str, Any]) -> str:
354
+ """The `downloadUrl` of a finished export job."""
355
+ url = answer.get("downloadUrl")
356
+ if not url:
357
+ raise JobError("Job finished without a download URL")
358
+ return str(url)
359
+
360
+
361
+ def import_statistics_changed(
362
+ before: dict[str, Any], after: dict[str, Any], table_schema_id: str | None = None
363
+ ) -> bool:
364
+ """True if `get_import_related_statistics()` shows a new import since `before`.
365
+
366
+ The answer maps table schema id -> `{"lastImportDate", "numberOfRecordsImported",
367
+ "fileExtension"}`. An import counts as visible once a table's
368
+ `(lastImportDate, numberOfRecordsImported)` pair differs from the baseline (or
369
+ the table is new); comparing the pair, not the count alone, also catches a
370
+ re-import that leaves the count unchanged. With `table_schema_id` only that
371
+ table is checked, otherwise any table.
372
+ """
373
+ keys = [table_schema_id] if table_schema_id else list(after)
374
+ for key in keys:
375
+ now = after.get(key)
376
+ if not isinstance(now, dict):
377
+ continue
378
+ was = before.get(key)
379
+ was = was if isinstance(was, dict) else {}
380
+ pair = ("lastImportDate", "numberOfRecordsImported")
381
+ if tuple(now.get(k) for k in pair) != tuple(was.get(k) for k in pair):
382
+ return True
383
+ return False
File without changes