reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
reportnet_api/easy.py
ADDED
|
@@ -0,0 +1,1359 @@
|
|
|
1
|
+
"""Hand-written convenience layer for notebooks and non-developers.
|
|
2
|
+
|
|
3
|
+
Thin wrappers over `ReportnetClient` that hide API plumbing: credentials from
|
|
4
|
+
the environment, flat listings, and tables as pandas DataFrames. The full
|
|
5
|
+
client stays available for anything not covered here. pandas is optional
|
|
6
|
+
(`pip install "reportnet-api[pandas]"`); it is only imported when a function
|
|
7
|
+
returning a DataFrame is called.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import getpass
|
|
13
|
+
import json
|
|
14
|
+
import numbers
|
|
15
|
+
import operator
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import tempfile
|
|
19
|
+
import time
|
|
20
|
+
import warnings
|
|
21
|
+
from collections.abc import Callable, Iterable
|
|
22
|
+
from decimal import Decimal
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
25
|
+
|
|
26
|
+
import httpx
|
|
27
|
+
|
|
28
|
+
from ._core import RetryPolicy, caller_stacklevel
|
|
29
|
+
from ._exceptions import ForbiddenError, ReportnetAPIError, _is_rejected_credential_500
|
|
30
|
+
from .client import ReportnetClient
|
|
31
|
+
from .demo import demo_client as _demo_client
|
|
32
|
+
from .jobs import JobError, Progress, progress_event, progress_scope
|
|
33
|
+
from .resources.dataflow_ext import DataflowKind
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
import pandas as pd
|
|
37
|
+
|
|
38
|
+
from .handles import Dataflow, Dataset
|
|
39
|
+
|
|
40
|
+
DEFAULT_BASE_URL = "https://api.reportnet.europa.eu"
|
|
41
|
+
|
|
42
|
+
_DATASET_GROUPS = {
|
|
43
|
+
"designDatasets": "design",
|
|
44
|
+
"reportingDatasets": "reporting",
|
|
45
|
+
"dataCollections": "data collection",
|
|
46
|
+
"euDatasets": "eu",
|
|
47
|
+
"referenceDatasets": "reference",
|
|
48
|
+
"testDatasets": "test",
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Three non-empty base64url parts: header.payload.signature.
|
|
53
|
+
_JWT_SHAPE = re.compile(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")
|
|
54
|
+
_SCHEMES = {"apikey": "ApiKey", "bearer": "Bearer"}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def connect(
|
|
58
|
+
token: str | None = None,
|
|
59
|
+
*,
|
|
60
|
+
base_url: str | None = None,
|
|
61
|
+
scheme: str | None = None,
|
|
62
|
+
token_provider: Callable[[], str] | None = None,
|
|
63
|
+
retry: RetryPolicy | bool | None = None,
|
|
64
|
+
) -> ReportnetClient:
|
|
65
|
+
"""Build a client from arguments or environment variables.
|
|
66
|
+
|
|
67
|
+
`token` falls back to `REPORTNET_API_KEY`, `base_url` to `REPORTNET_BASE_URL`
|
|
68
|
+
and then production. The scheme is chosen from the value: a ``"Bearer "``
|
|
69
|
+
prefix or a JWT-shaped value (three dot-separated base64url parts) is sent
|
|
70
|
+
as Bearer; anything else is a dataflow API key (``ApiKey`` scheme). Live,
|
|
71
|
+
both an ApiKey and a Bearer token are bare UUIDs and look alike, so for a
|
|
72
|
+
Bearer token pass `scheme="Bearer"` (or the ``"Bearer "`` prefix); sent as
|
|
73
|
+
an ApiKey it fails with a 500 "Access is denied". Which one you have
|
|
74
|
+
matters: API keys carry Reporter-level access only. Set `REPORTNET_TOKEN_SCHEME`
|
|
75
|
+
(``ApiKey`` or ``Bearer``) to fix the scheme once, e.g. next to a Bearer token
|
|
76
|
+
in `REPORTNET_API_KEY`; precedence is `scheme=`, a ``"Bearer "`` prefix, that
|
|
77
|
+
variable, then the guess from the value. The chosen scheme shows in the
|
|
78
|
+
client's repr; the secret never does.
|
|
79
|
+
|
|
80
|
+
`token_provider` is a zero-arg callable returning the current token, called
|
|
81
|
+
before every request (instead of `token`; the scheme defaults to Bearer).
|
|
82
|
+
`retry=True` retries transient failures with a default `RetryPolicy`; pass
|
|
83
|
+
your own policy for other settings.
|
|
84
|
+
|
|
85
|
+
A Bearer token expires after a few minutes. If you have your own way to get fresh
|
|
86
|
+
tokens, pass it as `token_provider`; `client.user.refresh_token()` renews a login
|
|
87
|
+
when you hold the refresh token issued with it. This library does not log in.
|
|
88
|
+
"""
|
|
89
|
+
if scheme is not None:
|
|
90
|
+
if scheme.lower() not in _SCHEMES:
|
|
91
|
+
raise ValueError(f"scheme must be 'ApiKey' or 'Bearer', got {scheme!r}.")
|
|
92
|
+
scheme = _SCHEMES[scheme.lower()]
|
|
93
|
+
env_scheme = os.environ.get("REPORTNET_TOKEN_SCHEME", "").strip()
|
|
94
|
+
if env_scheme and env_scheme.lower() not in _SCHEMES:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"REPORTNET_TOKEN_SCHEME must be 'ApiKey' or 'Bearer', got {env_scheme!r}."
|
|
97
|
+
)
|
|
98
|
+
if token_provider is not None:
|
|
99
|
+
if token:
|
|
100
|
+
raise ValueError("Pass either token or token_provider, not both.")
|
|
101
|
+
credential: str | Callable[[], str] = token_provider
|
|
102
|
+
scheme = scheme or _SCHEMES.get(env_scheme.lower()) or "Bearer"
|
|
103
|
+
else:
|
|
104
|
+
token = (token or os.environ.get("REPORTNET_API_KEY") or "").strip()
|
|
105
|
+
if not token:
|
|
106
|
+
raise ValueError(
|
|
107
|
+
"No credential: pass token=... or set the REPORTNET_API_KEY environment variable."
|
|
108
|
+
)
|
|
109
|
+
guessed = _SCHEMES.get(env_scheme.lower()) or "ApiKey"
|
|
110
|
+
if token.lower().startswith("bearer "):
|
|
111
|
+
guessed = "Bearer"
|
|
112
|
+
token = token[len("bearer ") :].strip()
|
|
113
|
+
elif _JWT_SHAPE.fullmatch(token) and not env_scheme:
|
|
114
|
+
guessed = "Bearer"
|
|
115
|
+
credential = token
|
|
116
|
+
scheme = scheme or guessed
|
|
117
|
+
policy = RetryPolicy() if retry is True else (retry or None)
|
|
118
|
+
url = base_url or os.environ.get("REPORTNET_BASE_URL") or DEFAULT_BASE_URL
|
|
119
|
+
return ReportnetClient(base_url=url, token=credential, token_scheme=scheme, retry=policy)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def demo_client(*, quiet: bool = False) -> ReportnetClient:
|
|
123
|
+
"""A client that answers from built-in fictional data: no network, no credential.
|
|
124
|
+
|
|
125
|
+
For training and first tries (`list_dataflows(c)`, `list_datasets(c, 1)`,
|
|
126
|
+
`read_table(c, 102, 1)`, `find_dataflow(c, "demo")`); see `reportnet_api.demo`.
|
|
127
|
+
Prints a one-line notice unless `quiet=True`.
|
|
128
|
+
"""
|
|
129
|
+
return _demo_client(quiet=quiet)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def connect_interactive(
|
|
133
|
+
*,
|
|
134
|
+
base_url: str | None = None,
|
|
135
|
+
scheme: str | None = None,
|
|
136
|
+
retry: RetryPolicy | bool | None = None,
|
|
137
|
+
prompt: str = "Reportnet API key or token (input hidden): ",
|
|
138
|
+
) -> ReportnetClient:
|
|
139
|
+
"""Like `connect()`, but asks for the credential with a hidden prompt.
|
|
140
|
+
|
|
141
|
+
Use it in a notebook so the key is typed, not pasted into a cell that gets
|
|
142
|
+
saved with the notebook. Same `scheme`/`retry` options as `connect()`.
|
|
143
|
+
"""
|
|
144
|
+
return connect(getpass.getpass(prompt), base_url=base_url, scheme=scheme, retry=retry)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _pandas() -> Any:
|
|
148
|
+
try:
|
|
149
|
+
import pandas
|
|
150
|
+
except ImportError:
|
|
151
|
+
raise ImportError(
|
|
152
|
+
"pandas is required for DataFrame output: install it with `pip install pandas` "
|
|
153
|
+
"(or use read_records() for plain dicts)"
|
|
154
|
+
) from None
|
|
155
|
+
return pandas
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _require_id(value: object, name: str, hint: str) -> int:
|
|
159
|
+
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
|
160
|
+
raise ValueError(f"{name} must be a positive integer id, got {value!r}. {hint}")
|
|
161
|
+
return value
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _hinted(err: ReportnetAPIError, hint: str) -> ReportnetAPIError:
|
|
165
|
+
"""Same exception type and attributes, with a plain-language hint appended."""
|
|
166
|
+
return type(err)(
|
|
167
|
+
f"{err} - {hint}",
|
|
168
|
+
status_code=err.status_code,
|
|
169
|
+
response_body=err.response_body,
|
|
170
|
+
request=err.request,
|
|
171
|
+
response=err.response,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
_DATAFLOW_HINT = "Pick an id from the list_dataflows() table."
|
|
176
|
+
_DATASET_HINT = "Use a dataset_id from list_datasets(), not the dataflow id."
|
|
177
|
+
_FORBIDDEN_HINT = "Also check that you did not mix up dataflow and dataset ids."
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def list_dataflows(
|
|
181
|
+
client: ReportnetClient, kind: DataflowKind = "reporting", *, page_size: int = 200
|
|
182
|
+
) -> pd.DataFrame:
|
|
183
|
+
"""All dataflows visible to the credential, one row each (id, name, status, ...)."""
|
|
184
|
+
pd = _pandas()
|
|
185
|
+
rows: list[dict[str, Any]] = []
|
|
186
|
+
page = 0
|
|
187
|
+
while True:
|
|
188
|
+
result = client.dataflow.dataflows(kind, page_size=page_size, page_num=page)
|
|
189
|
+
batch = result.get("dataflows") or []
|
|
190
|
+
rows.extend(batch)
|
|
191
|
+
total = result.get("filteredRecords") or result.get("totalRecords") or 0
|
|
192
|
+
page += 1
|
|
193
|
+
if not batch or len(rows) >= total:
|
|
194
|
+
break
|
|
195
|
+
return pd.DataFrame(rows)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def find_dataflow(client: ReportnetClient, query: int | str) -> Dataflow:
|
|
199
|
+
"""One dataflow handle by id or name text (see `handles.find_dataflow`).
|
|
200
|
+
|
|
201
|
+
The handle resolves datasets by id, country code or name:
|
|
202
|
+
`find_dataflow(c, "air quality").dataset("AT")`. Raises `LookupError` listing
|
|
203
|
+
the candidates when nothing or more than one matches.
|
|
204
|
+
"""
|
|
205
|
+
from .handles import find_dataflow as _find
|
|
206
|
+
|
|
207
|
+
return _find(client, query)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def read_many(datasets: Iterable[Dataset], table: str, **kwargs: Any) -> pd.DataFrame:
|
|
211
|
+
"""One table from several dataset handles, stacked (see `handles.read_many`).
|
|
212
|
+
|
|
213
|
+
Adds `dataset` and `country` columns; sequential; datasets without rows for the
|
|
214
|
+
table are skipped with a `RuntimeWarning`. Needs ETL access, like `read_tables()`.
|
|
215
|
+
"""
|
|
216
|
+
from .handles import read_many as _read_many
|
|
217
|
+
|
|
218
|
+
return _read_many(datasets, table, **kwargs)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def list_datasets(
|
|
222
|
+
client: ReportnetClient, dataflow_id: int, *, provider_id: int | None = None
|
|
223
|
+
) -> pd.DataFrame:
|
|
224
|
+
"""Datasets of a dataflow as a flat table: dataset_id, name, kind, provider_id.
|
|
225
|
+
|
|
226
|
+
An unknown dataflow id answers HTTP 500 (`ServerError`, see `explain()`) and
|
|
227
|
+
no access answers 403 (live-tested). `LookupError` is a guard for an empty
|
|
228
|
+
200 body, which the notes say the backend can send when it swallows an
|
|
229
|
+
error; not reproduced on this route.
|
|
230
|
+
"""
|
|
231
|
+
pd = _pandas()
|
|
232
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
233
|
+
try:
|
|
234
|
+
flow = client.dataflow.dataflow_datasets(dataflow_id, provider_id=provider_id)
|
|
235
|
+
except ForbiddenError as err:
|
|
236
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
237
|
+
if not flow:
|
|
238
|
+
# The backend answers 200 with an empty body when it swallows an error.
|
|
239
|
+
raise LookupError(
|
|
240
|
+
f"Empty answer for dataflow {dataflow_id}: the id is unknown or this credential "
|
|
241
|
+
"has no access to it (the server does not say which). "
|
|
242
|
+
f"{_DATAFLOW_HINT} Reporters also need provider_id."
|
|
243
|
+
)
|
|
244
|
+
rows = []
|
|
245
|
+
for group, kind in _DATASET_GROUPS.items():
|
|
246
|
+
for ds in flow.get(group) or []:
|
|
247
|
+
rows.append(
|
|
248
|
+
{
|
|
249
|
+
"dataset_id": ds.get("id"),
|
|
250
|
+
"name": ds.get("dataSetName"),
|
|
251
|
+
"kind": kind,
|
|
252
|
+
"provider_id": ds.get("dataProviderId"),
|
|
253
|
+
}
|
|
254
|
+
)
|
|
255
|
+
return pd.DataFrame(rows, columns=["dataset_id", "name", "kind", "provider_id"])
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def is_data_lake(client: ReportnetClient, dataflow_id: int) -> bool:
|
|
259
|
+
"""True when the dataflow is a data-lake one (`bigData: true` in its dataset listing).
|
|
260
|
+
|
|
261
|
+
Data-lake dataflows keep validation results and table data in the lake, so the
|
|
262
|
+
old per-dataset routes answer an empty `200` there.
|
|
263
|
+
"""
|
|
264
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
265
|
+
try:
|
|
266
|
+
flow = client.dataflow.dataflow_datasets(dataflow_id)
|
|
267
|
+
except ForbiddenError as err:
|
|
268
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
269
|
+
return bool((flow or {}).get("bigData"))
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
_ERROR_COLUMNS = [
|
|
273
|
+
"short_code",
|
|
274
|
+
"level",
|
|
275
|
+
"entity",
|
|
276
|
+
"table",
|
|
277
|
+
"field",
|
|
278
|
+
"message",
|
|
279
|
+
"records",
|
|
280
|
+
"rule_id",
|
|
281
|
+
]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def validation_errors(
|
|
285
|
+
client: ReportnetClient,
|
|
286
|
+
dataset_id: int,
|
|
287
|
+
dataflow_id: int,
|
|
288
|
+
*,
|
|
289
|
+
level_errors: list[str] | None = None,
|
|
290
|
+
type_entities: list[str] | None = None,
|
|
291
|
+
page_size: int = 100,
|
|
292
|
+
data_lake: bool | None = None,
|
|
293
|
+
) -> pd.DataFrame:
|
|
294
|
+
"""Failed validations of a dataset, one row per failing rule (what the UI shows).
|
|
295
|
+
|
|
296
|
+
Columns: short_code, level, entity, table, field, message, records (number of
|
|
297
|
+
failing records), rule_id. Picks the data-lake route (`list_group_validations_dl`)
|
|
298
|
+
when the dataflow has `bigData: true`, else `list_group_validations`; the old
|
|
299
|
+
route answers an empty list on a lake dataset even when the UI shows failures.
|
|
300
|
+
Pass `data_lake=True`/`False` when the caller already knows (it skips the
|
|
301
|
+
`bigData` lookup, whose empty answer would read as not a lake); `None` detects it.
|
|
302
|
+
An empty frame means no failures, or a validation that has not run yet: this
|
|
303
|
+
does not start one. Needs a credential that may read validations (an ApiKey
|
|
304
|
+
got 403 on the sandbox).
|
|
305
|
+
"""
|
|
306
|
+
pd = _pandas()
|
|
307
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
308
|
+
lake = is_data_lake(client, dataflow_id) if data_lake is None else data_lake
|
|
309
|
+
fetch = (
|
|
310
|
+
client.validation.list_group_validations_dl
|
|
311
|
+
if lake
|
|
312
|
+
else client.validation.list_group_validations
|
|
313
|
+
)
|
|
314
|
+
rows: list[dict[str, Any]] = []
|
|
315
|
+
page = 0
|
|
316
|
+
while True:
|
|
317
|
+
try:
|
|
318
|
+
result = fetch(
|
|
319
|
+
dataset_id,
|
|
320
|
+
page_num=page,
|
|
321
|
+
page_size=page_size,
|
|
322
|
+
level_errors=level_errors,
|
|
323
|
+
type_entities=type_entities,
|
|
324
|
+
)
|
|
325
|
+
except ForbiddenError as err:
|
|
326
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
327
|
+
batch = (result or {}).get("errors") or []
|
|
328
|
+
for e in batch:
|
|
329
|
+
rows.append(
|
|
330
|
+
{
|
|
331
|
+
"short_code": e.get("shortCode"),
|
|
332
|
+
"level": e.get("levelError"),
|
|
333
|
+
"entity": e.get("typeEntity"),
|
|
334
|
+
"table": e.get("nameTableSchema"),
|
|
335
|
+
"field": e.get("nameFieldSchema"),
|
|
336
|
+
"message": e.get("message"),
|
|
337
|
+
"records": e.get("numberOfRecords"),
|
|
338
|
+
"rule_id": e.get("idRule"),
|
|
339
|
+
}
|
|
340
|
+
)
|
|
341
|
+
total = (
|
|
342
|
+
(result or {}).get("totalFilteredRecords") or (result or {}).get("totalRecords") or 0
|
|
343
|
+
)
|
|
344
|
+
page += 1
|
|
345
|
+
if not batch or len(rows) >= total or page >= _MAX_PAGES:
|
|
346
|
+
break
|
|
347
|
+
return pd.DataFrame(rows, columns=_ERROR_COLUMNS)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def validation_summary(client: ReportnetClient, dataset_id: int, dataflow_id: int) -> pd.DataFrame:
|
|
351
|
+
"""Failures counted per table and level: table, level, rules, records.
|
|
352
|
+
|
|
353
|
+
`rules` is the number of failing rules, `records` the sum of their failing
|
|
354
|
+
record counts (one record can fail several rules, so it is not a distinct
|
|
355
|
+
count). Built on `validation_errors()`, so it has the same lake handling and
|
|
356
|
+
does not start a validation. Empty frame: no failures, or none run yet.
|
|
357
|
+
"""
|
|
358
|
+
pd = _pandas()
|
|
359
|
+
errors = validation_errors(client, dataset_id, dataflow_id)
|
|
360
|
+
cols = ["table", "level", "rules", "records"]
|
|
361
|
+
if errors.empty:
|
|
362
|
+
return pd.DataFrame(columns=cols)
|
|
363
|
+
grouped = (
|
|
364
|
+
errors.fillna({"table": "", "level": ""})
|
|
365
|
+
.groupby(["table", "level"], as_index=False)
|
|
366
|
+
.agg(rules=("short_code", "size"), records=("records", "sum"))
|
|
367
|
+
)
|
|
368
|
+
return grouped[cols]
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _latest_notification_ms(client: ReportnetClient, dataset_id: int, event_type: str) -> int:
|
|
372
|
+
"""`insertDate` (ms) of the newest `event_type` user notification for the dataset, or 0.
|
|
373
|
+
|
|
374
|
+
Reads `GET /notification/findUserNotifications` (hidden route; the list the
|
|
375
|
+
Reportnet UI shows, newest first, for the calling user only).
|
|
376
|
+
"""
|
|
377
|
+
answer = client.request(
|
|
378
|
+
"GET", "/notification/findUserNotifications", params={"pageNum": 0, "pageSize": 50}
|
|
379
|
+
).json()
|
|
380
|
+
times = [
|
|
381
|
+
int(n.get("insertDate") or 0)
|
|
382
|
+
for n in (answer or {}).get("userNotifications") or []
|
|
383
|
+
if n.get("eventType") == event_type
|
|
384
|
+
and (n.get("content") or {}).get("datasetId") == dataset_id
|
|
385
|
+
]
|
|
386
|
+
return max(times, default=0)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def validate_and_wait(
|
|
390
|
+
client: ReportnetClient,
|
|
391
|
+
dataset_id: int,
|
|
392
|
+
dataflow_id: int,
|
|
393
|
+
*,
|
|
394
|
+
released: bool = False,
|
|
395
|
+
timeout: float = 600.0,
|
|
396
|
+
poll_interval: float = 3.0,
|
|
397
|
+
data_lake: bool | None = None,
|
|
398
|
+
) -> pd.DataFrame:
|
|
399
|
+
"""Start a validation, wait until it finishes, return `validation_errors()`.
|
|
400
|
+
|
|
401
|
+
`data_lake` is passed to `validation_errors()` (None: detect from `bigData`).
|
|
402
|
+
|
|
403
|
+
WRITES: replaces the dataset's validation results and takes a dataset lock.
|
|
404
|
+
The completion signal is the user notification the backend creates when a run
|
|
405
|
+
ends (`VALIDATION_FINISHED_EVENT`, or `VALIDATION_RELEASE_FINISHED_EVENT` with
|
|
406
|
+
`released=True`), read from `/notification/findUserNotifications`: the dataset
|
|
407
|
+
has no status field to poll (live, the sandbox). A run took
|
|
408
|
+
about 35-55 s there. Only the calling user's notifications are visible, and a
|
|
409
|
+
failed run has no event of its own here, so it ends as a `JobError` timeout.
|
|
410
|
+
Notification times are server-side, so a newer one than any seen before the
|
|
411
|
+
start counts; a run started by someone else at the same moment could be
|
|
412
|
+
mistaken for this one. Raises `JobError` after `timeout` seconds.
|
|
413
|
+
"""
|
|
414
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
415
|
+
event = "VALIDATION_RELEASE_FINISHED_EVENT" if released else "VALIDATION_FINISHED_EVENT"
|
|
416
|
+
baseline = _latest_notification_ms(client, dataset_id, event)
|
|
417
|
+
try:
|
|
418
|
+
client.validation.start_validation(dataset_id, released=released)
|
|
419
|
+
except ForbiddenError as err:
|
|
420
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
421
|
+
deadline = time.monotonic() + timeout
|
|
422
|
+
while _latest_notification_ms(client, dataset_id, event) <= baseline:
|
|
423
|
+
if time.monotonic() >= deadline:
|
|
424
|
+
raise JobError(
|
|
425
|
+
f"No {event} notification for dataset {dataset_id} within {timeout:g} "
|
|
426
|
+
"seconds (the validation may still be running, or it failed: a failed "
|
|
427
|
+
"run has no event of its own)"
|
|
428
|
+
)
|
|
429
|
+
time.sleep(poll_interval)
|
|
430
|
+
return validation_errors(client, dataset_id, dataflow_id, data_lake=data_lake)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def export_validations(
|
|
434
|
+
client: ReportnetClient,
|
|
435
|
+
dataset_id: int,
|
|
436
|
+
dest: str | os.PathLike[str],
|
|
437
|
+
*,
|
|
438
|
+
timeout: float = 300.0,
|
|
439
|
+
poll_interval: float = 2.0,
|
|
440
|
+
) -> Path:
|
|
441
|
+
"""Save a dataset's validation failures as a CSV at `dest` and return the path.
|
|
442
|
+
|
|
443
|
+
Generates the export and waits for it (`validation.export_validations_to_file`).
|
|
444
|
+
Columns: Entity, Table, Field, Code, QC Name, QC Description, Severity Level,
|
|
445
|
+
Message, Number of records. Works on data-lake datasets (live, the sandbox,
|
|
446
|
+
). Close any open Reportnet page for the same dataset first: the
|
|
447
|
+
backend deletes the file when it is served, so the web app, which downloads it
|
|
448
|
+
on the notification, can take it before this does and the wait ends in a
|
|
449
|
+
`JobError` timeout. Leaves a notification in the user's Reportnet UI.
|
|
450
|
+
"""
|
|
451
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
452
|
+
path = Path(dest)
|
|
453
|
+
try:
|
|
454
|
+
client.validation.export_validations_to_file(
|
|
455
|
+
dataset_id, path, timeout=timeout, poll_interval=poll_interval
|
|
456
|
+
)
|
|
457
|
+
except ForbiddenError as err:
|
|
458
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
459
|
+
return path
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def list_providers(client: ReportnetClient, group_id: int = 1) -> pd.DataFrame:
|
|
463
|
+
"""Data providers of a group as a table: provider_id, code, label, group_id.
|
|
464
|
+
|
|
465
|
+
A provider is a reporting country; `provider_id` is the `providerId` other
|
|
466
|
+
calls take and `code` its country code. Group `1` is the member
|
|
467
|
+
countries; `client.representative.data_provider_groups()` lists the groups.
|
|
468
|
+
"""
|
|
469
|
+
pd = _pandas()
|
|
470
|
+
_require_id(group_id, "group_id", "Group 1 is the member countries.")
|
|
471
|
+
try:
|
|
472
|
+
providers = client.representative.data_providers(group_id) or []
|
|
473
|
+
except ForbiddenError as err:
|
|
474
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
475
|
+
rows = [
|
|
476
|
+
{
|
|
477
|
+
"provider_id": p.get("id"),
|
|
478
|
+
"code": p.get("code"),
|
|
479
|
+
"label": p.get("label"),
|
|
480
|
+
"group_id": p.get("groupId"),
|
|
481
|
+
}
|
|
482
|
+
for p in providers
|
|
483
|
+
]
|
|
484
|
+
return pd.DataFrame(rows, columns=["provider_id", "code", "label", "group_id"])
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def list_tables(client: ReportnetClient, dataset_id: int) -> pd.DataFrame:
|
|
488
|
+
"""Tables defined in a dataset's schema: table_id and name."""
|
|
489
|
+
pd = _pandas()
|
|
490
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
491
|
+
try:
|
|
492
|
+
schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
|
|
493
|
+
except ForbiddenError as err:
|
|
494
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
495
|
+
rows = [
|
|
496
|
+
{"table_id": t.idTableSchema, "name": t.nameTableSchema} for t in schema.tableSchemas or []
|
|
497
|
+
]
|
|
498
|
+
return pd.DataFrame(rows, columns=["table_id", "name"])
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _records_to_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
502
|
+
rows = []
|
|
503
|
+
for rec in records:
|
|
504
|
+
row: dict[str, Any] = {}
|
|
505
|
+
if rec.get("countryCode") is not None:
|
|
506
|
+
row["country_code"] = rec["countryCode"]
|
|
507
|
+
fields = rec.get("fields")
|
|
508
|
+
if isinstance(fields, list): # [{"fieldName": ..., "value": ...}, ...]
|
|
509
|
+
for f in fields:
|
|
510
|
+
row[f.get("fieldName")] = f.get("value")
|
|
511
|
+
elif isinstance(fields, dict):
|
|
512
|
+
row.update(fields)
|
|
513
|
+
else: # unknown shape: keep everything rather than silently dropping it
|
|
514
|
+
row.update({k: v for k, v in rec.items() if k != "countryCode"})
|
|
515
|
+
rows.append(row)
|
|
516
|
+
return rows
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
_MAX_PAGES = 10_000 # backstop against a backend that keeps returning full pages
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _schema_table_names(client: ReportnetClient, dataset_id: int) -> set[str] | None:
|
|
523
|
+
try:
|
|
524
|
+
return {
|
|
525
|
+
t.nameTableSchema
|
|
526
|
+
for t in client.dataset_schema.find_data_schema_by_dataset_id(
|
|
527
|
+
dataset_id=dataset_id
|
|
528
|
+
).tableSchemas
|
|
529
|
+
or []
|
|
530
|
+
if t.nameTableSchema
|
|
531
|
+
}
|
|
532
|
+
except ReportnetAPIError:
|
|
533
|
+
return None # best-effort cross-check only
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _schema_field_types(client: ReportnetClient, dataset_id: int) -> dict[str, dict[str, str]]:
|
|
537
|
+
"""``{table name: {field name: backend DataType}}`` from the dataset schema."""
|
|
538
|
+
schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
|
|
539
|
+
out: dict[str, dict[str, str]] = {}
|
|
540
|
+
for t in schema.tableSchemas or []:
|
|
541
|
+
fields = t.recordSchema.fieldSchema if t.recordSchema else None
|
|
542
|
+
out[t.nameTableSchema or ""] = {
|
|
543
|
+
f.name: str(getattr(f.type, "value", f.type)) for f in fields or [] if f.name
|
|
544
|
+
}
|
|
545
|
+
return out
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
_BOOL_TEXT = {"true": True, "false": False}
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _convert_column(series: pd.Series, field_type: str) -> pd.Series:
|
|
552
|
+
"""Convert one text column by its schema type; raises ValueError if any value
|
|
553
|
+
does not fit. Empty text becomes NA."""
|
|
554
|
+
pd = _pandas()
|
|
555
|
+
text = series.astype("string").str.strip()
|
|
556
|
+
text = text.mask(text == "")
|
|
557
|
+
if field_type == "NUMBER_INTEGER":
|
|
558
|
+
num = pd.to_numeric(text, errors="raise")
|
|
559
|
+
if (num.dropna() % 1 != 0).any():
|
|
560
|
+
raise ValueError("non-integer value in an integer field")
|
|
561
|
+
return num.astype("Int64")
|
|
562
|
+
if field_type == "NUMBER_DECIMAL":
|
|
563
|
+
return pd.to_numeric(text, errors="raise").astype("Float64")
|
|
564
|
+
if field_type == "DATE":
|
|
565
|
+
return pd.to_datetime(text, format="%Y-%m-%d", errors="raise")
|
|
566
|
+
if field_type == "DATETIME":
|
|
567
|
+
return pd.to_datetime(text, format="ISO8601", errors="raise", utc=True)
|
|
568
|
+
if field_type == "BOOLEAN":
|
|
569
|
+
low = text.str.lower()
|
|
570
|
+
if not low.dropna().isin(_BOOL_TEXT).all():
|
|
571
|
+
raise ValueError("value is not true/false")
|
|
572
|
+
return low.map(_BOOL_TEXT).astype("boolean")
|
|
573
|
+
return series # text-like, geometry and codelist types stay as exported
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def apply_schema_dtypes(
|
|
577
|
+
tables: dict[str, pd.DataFrame], field_types: dict[str, dict[str, str]]
|
|
578
|
+
) -> dict[str, pd.DataFrame]:
|
|
579
|
+
"""Convert columns to the types the dataset schema declares.
|
|
580
|
+
|
|
581
|
+
Handles NUMBER_INTEGER (`Int64`), NUMBER_DECIMAL (`Float64`), DATE, DATETIME (UTC)
|
|
582
|
+
and BOOLEAN (`true`/`false`, any case); every other type stays text. A column
|
|
583
|
+
whose values do not all fit keeps its original text and a `RuntimeWarning` names
|
|
584
|
+
it, so a wrong guess never silently changes data. Returns new frames.
|
|
585
|
+
"""
|
|
586
|
+
out = {}
|
|
587
|
+
for name, df in tables.items():
|
|
588
|
+
df = df.copy()
|
|
589
|
+
types = field_types.get(name, {})
|
|
590
|
+
for col in df.columns:
|
|
591
|
+
ftype = types.get(col)
|
|
592
|
+
if ftype is None:
|
|
593
|
+
continue
|
|
594
|
+
try:
|
|
595
|
+
df[col] = _convert_column(df[col], ftype)
|
|
596
|
+
except (ValueError, TypeError) as err:
|
|
597
|
+
warnings.warn(
|
|
598
|
+
f"Table {name!r}, column {col!r} ({ftype}): left as text, {err}",
|
|
599
|
+
RuntimeWarning,
|
|
600
|
+
stacklevel=3,
|
|
601
|
+
)
|
|
602
|
+
out[name] = df
|
|
603
|
+
return out
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _collect_tables(
|
|
607
|
+
client: ReportnetClient,
|
|
608
|
+
dataset_id: int,
|
|
609
|
+
dataflow_id: int,
|
|
610
|
+
*,
|
|
611
|
+
table_schema_id: str | None,
|
|
612
|
+
provider_id: int | None,
|
|
613
|
+
page_size: int,
|
|
614
|
+
progress: Progress,
|
|
615
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
616
|
+
"""Page through the ETL export; ``{table name: rows}`` as plain dicts (no pandas)."""
|
|
617
|
+
try:
|
|
618
|
+
size = operator.index(page_size) # accepts numpy integers, rejects 2.5 and "10"
|
|
619
|
+
except TypeError:
|
|
620
|
+
size = 0
|
|
621
|
+
if isinstance(page_size, bool) or size < 1:
|
|
622
|
+
raise ValueError(f"page_size must be a positive integer, got {page_size!r}")
|
|
623
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
624
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
625
|
+
collected: dict[str, list[dict[str, Any]]] = {}
|
|
626
|
+
seen: dict[str, set[str]] = {}
|
|
627
|
+
totals: dict[str, int] = {}
|
|
628
|
+
duplicates: dict[str, int] = {}
|
|
629
|
+
page = 1 # counts requests, for progress and the page cap
|
|
630
|
+
offset = 0 # 0-based row offset: the backend's `offset` is not a page number
|
|
631
|
+
started = time.monotonic()
|
|
632
|
+
with progress_scope(progress) as report, tempfile.TemporaryDirectory() as tmp:
|
|
633
|
+
dest = Path(tmp) / "export.json"
|
|
634
|
+
while True:
|
|
635
|
+
try:
|
|
636
|
+
client.datasets.etl_export_dataset(
|
|
637
|
+
dataset_id,
|
|
638
|
+
dest,
|
|
639
|
+
limit=page_size,
|
|
640
|
+
offset=offset,
|
|
641
|
+
provider_id=provider_id,
|
|
642
|
+
table_schema_id=table_schema_id,
|
|
643
|
+
dataflow_id=dataflow_id,
|
|
644
|
+
)
|
|
645
|
+
except ForbiddenError as err:
|
|
646
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
647
|
+
payload = json.loads(dest.read_text(encoding="utf-8") or "{}")
|
|
648
|
+
more = False
|
|
649
|
+
for t in payload.get("tables") or []:
|
|
650
|
+
name = t.get("tableName") or "table"
|
|
651
|
+
rows = collected.setdefault(name, [])
|
|
652
|
+
ids = seen.setdefault(name, set())
|
|
653
|
+
recs = t.get("records") or []
|
|
654
|
+
fresh = []
|
|
655
|
+
for rec in recs:
|
|
656
|
+
rid = rec.get("id_record")
|
|
657
|
+
if rid is not None:
|
|
658
|
+
if rid in ids:
|
|
659
|
+
duplicates[name] = duplicates.get(name, 0) + 1
|
|
660
|
+
continue
|
|
661
|
+
ids.add(rid)
|
|
662
|
+
fresh.append(rec)
|
|
663
|
+
rows.extend(_records_to_rows(fresh))
|
|
664
|
+
if not fresh:
|
|
665
|
+
continue # nothing new on this page: do not ask for more
|
|
666
|
+
# The backend only sends `totalRecords` when a filter such as
|
|
667
|
+
# `tableSchemaId` is set (per its develop source; live it was
|
|
668
|
+
# present without one); without it, a full page is the only sign
|
|
669
|
+
# that more may follow.
|
|
670
|
+
if "totalRecords" in t:
|
|
671
|
+
totals[name] = int(t["totalRecords"] or 0)
|
|
672
|
+
more = more or len(rows) < totals[name]
|
|
673
|
+
else:
|
|
674
|
+
more = more or len(recs) >= page_size
|
|
675
|
+
if report:
|
|
676
|
+
rows_so_far = sum(len(r) for r in collected.values())
|
|
677
|
+
elapsed = time.monotonic() - started
|
|
678
|
+
report(progress_event("page", elapsed=elapsed, page=page, rows=rows_so_far))
|
|
679
|
+
if not more:
|
|
680
|
+
break
|
|
681
|
+
if page >= _MAX_PAGES:
|
|
682
|
+
warnings.warn(
|
|
683
|
+
f"read_tables stopped after {_MAX_PAGES} pages; the data is incomplete",
|
|
684
|
+
RuntimeWarning,
|
|
685
|
+
stacklevel=caller_stacklevel(),
|
|
686
|
+
)
|
|
687
|
+
break
|
|
688
|
+
page += 1
|
|
689
|
+
offset += page_size
|
|
690
|
+
for name, n in duplicates.items():
|
|
691
|
+
warnings.warn(
|
|
692
|
+
f"Table {name!r}: the backend returned {n} duplicate record(s) across pages; "
|
|
693
|
+
"they were dropped",
|
|
694
|
+
RuntimeWarning,
|
|
695
|
+
stacklevel=caller_stacklevel(),
|
|
696
|
+
)
|
|
697
|
+
for name, total in totals.items():
|
|
698
|
+
if len(collected[name]) != total:
|
|
699
|
+
warnings.warn(
|
|
700
|
+
f"Table {name!r}: got {len(collected[name])} rows but the backend reports "
|
|
701
|
+
f"totalRecords={total}; the result may be incomplete",
|
|
702
|
+
RuntimeWarning,
|
|
703
|
+
stacklevel=caller_stacklevel(),
|
|
704
|
+
)
|
|
705
|
+
if not any(collected.values()):
|
|
706
|
+
warnings.warn(
|
|
707
|
+
"The ETL export answered with no rows. If the dataset is not empty, it may be "
|
|
708
|
+
"in a data-lake dataflow (`bigData: true`), where this route answers 200 with "
|
|
709
|
+
"0 rows (live-tested); read it with `easy.export_to_file()` instead",
|
|
710
|
+
RuntimeWarning,
|
|
711
|
+
stacklevel=caller_stacklevel(),
|
|
712
|
+
)
|
|
713
|
+
if table_schema_id is None:
|
|
714
|
+
expected = _schema_table_names(client, dataset_id)
|
|
715
|
+
missing = sorted(expected - collected.keys()) if expected else []
|
|
716
|
+
if missing:
|
|
717
|
+
warnings.warn(
|
|
718
|
+
f"Tables {missing} are in the dataset schema but missing from the export",
|
|
719
|
+
RuntimeWarning,
|
|
720
|
+
stacklevel=caller_stacklevel(),
|
|
721
|
+
)
|
|
722
|
+
return collected
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def read_tables(
|
|
726
|
+
client: ReportnetClient,
|
|
727
|
+
dataset_id: int,
|
|
728
|
+
dataflow_id: int,
|
|
729
|
+
*,
|
|
730
|
+
table_schema_id: str | None = None,
|
|
731
|
+
provider_id: int | None = None,
|
|
732
|
+
page_size: int = 10000,
|
|
733
|
+
dtypes: Literal["schema"] | None = None,
|
|
734
|
+
progress: Progress = None,
|
|
735
|
+
) -> dict[str, pd.DataFrame]:
|
|
736
|
+
"""Download a dataset's data as ``{table_name: DataFrame}``.
|
|
737
|
+
|
|
738
|
+
Pages through the synchronous ETL export until every row has been fetched.
|
|
739
|
+
Note: ETL access is credential-dependent (an API key works; some Bearer
|
|
740
|
+
users get 403) - see the README.
|
|
741
|
+
|
|
742
|
+
The backend's `offset` is a 0-based row offset (live-tested on production
|
|
743
|
+
and the sandbox), so pages are fetched at offsets 0,
|
|
744
|
+
`page_size`, 2 * `page_size`, ... Multi-table datasets are not verified. As a
|
|
745
|
+
guard, rows are de-duplicated by the backend's `id_record`, paging stops when
|
|
746
|
+
a page brings nothing new, and a `RuntimeWarning` is raised for duplicates, a
|
|
747
|
+
row count that differs from `totalRecords`, or tables of the schema missing
|
|
748
|
+
from the answer.
|
|
749
|
+
|
|
750
|
+
Data-lake caveat: for a dataflow with `bigData: true` this route answers `200`
|
|
751
|
+
with 0 rows even when data exists (live-tested), so a result
|
|
752
|
+
with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export)
|
|
753
|
+
for such a dataset.
|
|
754
|
+
|
|
755
|
+
Values arrive from the backend as text, empty cells as `''`. With
|
|
756
|
+
`dtypes="schema"` columns are converted using the field types in the dataset
|
|
757
|
+
schema (see `apply_schema_dtypes`: integers, decimals, dates, datetimes and
|
|
758
|
+
booleans; everything else stays text; a column that does not fit stays text with
|
|
759
|
+
a warning). Live-checked on the sandbox for integer, decimal, date and datetime
|
|
760
|
+
text; boolean text is unverified. The default (`None`) keeps all columns as text.
|
|
761
|
+
|
|
762
|
+
`progress=True` shows one updating line with the page number and rows so far;
|
|
763
|
+
a callable receives an event dict per page (`reportnet_api.jobs.progress_event`).
|
|
764
|
+
"""
|
|
765
|
+
pd = _pandas()
|
|
766
|
+
if dtypes not in (None, "schema"):
|
|
767
|
+
raise ValueError('dtypes must be None or "schema"')
|
|
768
|
+
collected = _collect_tables(
|
|
769
|
+
client,
|
|
770
|
+
dataset_id,
|
|
771
|
+
dataflow_id,
|
|
772
|
+
table_schema_id=table_schema_id,
|
|
773
|
+
provider_id=provider_id,
|
|
774
|
+
page_size=page_size,
|
|
775
|
+
progress=progress,
|
|
776
|
+
)
|
|
777
|
+
frames = {name: pd.DataFrame(rows) for name, rows in collected.items()}
|
|
778
|
+
if dtypes == "schema":
|
|
779
|
+
try:
|
|
780
|
+
field_types = _schema_field_types(client, dataset_id)
|
|
781
|
+
except ReportnetAPIError as err:
|
|
782
|
+
warnings.warn(
|
|
783
|
+
f"dtypes='schema': could not read the dataset schema ({err}); columns left as text",
|
|
784
|
+
RuntimeWarning,
|
|
785
|
+
stacklevel=caller_stacklevel(),
|
|
786
|
+
)
|
|
787
|
+
return frames
|
|
788
|
+
frames = apply_schema_dtypes(frames, field_types)
|
|
789
|
+
return frames
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def read_records(
|
|
793
|
+
client: ReportnetClient,
|
|
794
|
+
dataset_id: int,
|
|
795
|
+
dataflow_id: int,
|
|
796
|
+
table_name: str | None = None,
|
|
797
|
+
**kwargs: Any,
|
|
798
|
+
) -> list[dict[str, Any]]:
|
|
799
|
+
"""One table as a list of dicts, without pandas (for FME and plain scripts).
|
|
800
|
+
|
|
801
|
+
Same paging, de-duplication and warnings as `read_tables`; each row is a dict
|
|
802
|
+
``{field name: text}`` (plus `country_code` when the backend sends one) and empty
|
|
803
|
+
cells are `''`. With no `table_name`, the dataset must contain exactly one table with
|
|
804
|
+
data. Keyword arguments (`table_schema_id`, `provider_id`, `page_size`, `progress`)
|
|
805
|
+
are those of `read_tables`.
|
|
806
|
+
"""
|
|
807
|
+
tables = read_all_records(client, dataset_id, dataflow_id, **kwargs)
|
|
808
|
+
if table_name is not None:
|
|
809
|
+
try:
|
|
810
|
+
return tables[table_name]
|
|
811
|
+
except KeyError:
|
|
812
|
+
raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}") from None
|
|
813
|
+
if len(tables) != 1:
|
|
814
|
+
filled = {name: rows for name, rows in tables.items() if len(rows)}
|
|
815
|
+
if len(filled) == 1:
|
|
816
|
+
return next(iter(filled.values()))
|
|
817
|
+
if not filled:
|
|
818
|
+
raise ValueError(
|
|
819
|
+
f"All {len(tables)} tables are empty: {sorted(tables)}; "
|
|
820
|
+
"pass table_name=... to get one anyway"
|
|
821
|
+
)
|
|
822
|
+
raise ValueError(
|
|
823
|
+
f"Dataset has {len(filled)} tables with data {sorted(filled)}; pass table_name=..."
|
|
824
|
+
)
|
|
825
|
+
return next(iter(tables.values()))
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def read_all_records(
|
|
829
|
+
client: ReportnetClient,
|
|
830
|
+
dataset_id: int,
|
|
831
|
+
dataflow_id: int,
|
|
832
|
+
*,
|
|
833
|
+
table_schema_id: str | None = None,
|
|
834
|
+
provider_id: int | None = None,
|
|
835
|
+
page_size: int = 10000,
|
|
836
|
+
progress: Progress = None,
|
|
837
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
838
|
+
"""Every table as ``{table name: list of row dicts}``, without pandas (see `read_records`)."""
|
|
839
|
+
return _collect_tables(
|
|
840
|
+
client,
|
|
841
|
+
dataset_id,
|
|
842
|
+
dataflow_id,
|
|
843
|
+
table_schema_id=table_schema_id,
|
|
844
|
+
provider_id=provider_id,
|
|
845
|
+
page_size=page_size,
|
|
846
|
+
progress=progress,
|
|
847
|
+
)
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def read_table(
|
|
851
|
+
client: ReportnetClient,
|
|
852
|
+
dataset_id: int,
|
|
853
|
+
dataflow_id: int,
|
|
854
|
+
table_name: str | None = None,
|
|
855
|
+
**kwargs: Any,
|
|
856
|
+
) -> pd.DataFrame:
|
|
857
|
+
"""One table of a dataset as a DataFrame.
|
|
858
|
+
|
|
859
|
+
With no `table_name`, the dataset must contain exactly one table with data.
|
|
860
|
+
Extra keyword arguments go to `read_tables`, e.g. `dtypes="schema"` for typed columns.
|
|
861
|
+
"""
|
|
862
|
+
tables = read_tables(client, dataset_id, dataflow_id, **kwargs)
|
|
863
|
+
if table_name is not None:
|
|
864
|
+
try:
|
|
865
|
+
return tables[table_name]
|
|
866
|
+
except KeyError:
|
|
867
|
+
raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}") from None
|
|
868
|
+
if len(tables) != 1:
|
|
869
|
+
filled = {name: rows for name, rows in tables.items() if len(rows)}
|
|
870
|
+
if len(filled) == 1:
|
|
871
|
+
return next(iter(filled.values()))
|
|
872
|
+
if not filled:
|
|
873
|
+
raise ValueError(
|
|
874
|
+
f"All {len(tables)} tables are empty: {sorted(tables)}; "
|
|
875
|
+
"pass table_name=... to get one anyway"
|
|
876
|
+
)
|
|
877
|
+
raise ValueError(
|
|
878
|
+
f"Dataset has {len(filled)} tables with data {sorted(filled)}; pass table_name=..."
|
|
879
|
+
)
|
|
880
|
+
return next(iter(tables.values()))
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def export_to_file(
|
|
884
|
+
client: ReportnetClient,
|
|
885
|
+
dataset_id: int,
|
|
886
|
+
dataflow_id: int,
|
|
887
|
+
dest: str | os.PathLike[str],
|
|
888
|
+
*,
|
|
889
|
+
file_format: Literal["csv", "parquet"] = "csv",
|
|
890
|
+
table_schema_id: str | None = None,
|
|
891
|
+
timeout: float = 300.0,
|
|
892
|
+
progress: Progress = None,
|
|
893
|
+
) -> Path:
|
|
894
|
+
"""Export a dataset as a ZIP of CSV (or Parquet) files at `dest`.
|
|
895
|
+
|
|
896
|
+
Starts the async export job, waits for it to finish, then downloads the
|
|
897
|
+
ZIP. Raises `reportnet_api.jobs.JobError` if the job fails or `timeout`
|
|
898
|
+
seconds pass. Same ETL-access caveat as `read_tables()`. `progress=True` shows
|
|
899
|
+
one updating status line; a callable gets event dicts (see `wait_for_job()`).
|
|
900
|
+
"""
|
|
901
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
902
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
903
|
+
if file_format not in ("csv", "parquet"):
|
|
904
|
+
raise ValueError(f"file_format must be 'csv' or 'parquet', got {file_format!r}")
|
|
905
|
+
export = (
|
|
906
|
+
client.datasets.export_zip_csv_to_file
|
|
907
|
+
if file_format == "csv"
|
|
908
|
+
else client.datasets.export_zip_parquet_to_file
|
|
909
|
+
)
|
|
910
|
+
try:
|
|
911
|
+
export(
|
|
912
|
+
dataset_id,
|
|
913
|
+
dataflow_id=dataflow_id,
|
|
914
|
+
dest=dest,
|
|
915
|
+
table_schema_id=table_schema_id,
|
|
916
|
+
timeout=timeout,
|
|
917
|
+
progress=progress,
|
|
918
|
+
)
|
|
919
|
+
except ForbiddenError as err:
|
|
920
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
921
|
+
return Path(dest)
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def export_table(
|
|
925
|
+
client: ReportnetClient,
|
|
926
|
+
dataset_id: int,
|
|
927
|
+
table: str,
|
|
928
|
+
dest: str | os.PathLike[str],
|
|
929
|
+
*,
|
|
930
|
+
file_format: str = "csv",
|
|
931
|
+
big_data: bool = True,
|
|
932
|
+
timeout: float = 300.0,
|
|
933
|
+
progress: Progress = None,
|
|
934
|
+
) -> Path:
|
|
935
|
+
"""Export one table (by name) to `dest` as csv/xlsx/..., waiting for the file.
|
|
936
|
+
|
|
937
|
+
Unlike `export_to_file()` this is not a job: the backend generates the file
|
|
938
|
+
in the background and only notifies the Reportnet UI, so the helper polls the
|
|
939
|
+
download until the file appears. Raises `reportnet_api.jobs.JobError` after
|
|
940
|
+
`timeout` seconds. It leaves a notification in the user's Reportnet UI.
|
|
941
|
+
Needs a Bearer token (an ApiKey gets 403). `big_data=True` (default) uses the
|
|
942
|
+
routes the Reportnet UI calls; the plain ones never produced a file live.
|
|
943
|
+
Check `dest` afterwards: a leftover file with the same name on the server
|
|
944
|
+
could be served instead. `progress=True` shows a "waiting for the file" line
|
|
945
|
+
with the elapsed seconds; a callable gets event dicts.
|
|
946
|
+
"""
|
|
947
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
948
|
+
if not file_format or " " in file_format or file_format.startswith("."):
|
|
949
|
+
raise ValueError(f"file_format must be a bare extension like 'csv', got {file_format!r}")
|
|
950
|
+
tables = list_tables(client, dataset_id)
|
|
951
|
+
match = tables[tables["name"] == table]
|
|
952
|
+
if match.empty:
|
|
953
|
+
raise ValueError(f"No table named {table!r}; the dataset has {sorted(tables['name'])}")
|
|
954
|
+
if len(match) > 1:
|
|
955
|
+
raise ValueError(f"The dataset has {len(match)} tables named {table!r}; can't pick one")
|
|
956
|
+
try:
|
|
957
|
+
client.datasets.export_table_to_file(
|
|
958
|
+
dataset_id,
|
|
959
|
+
table_name=table,
|
|
960
|
+
table_schema_id=str(match.iloc[0]["table_id"]),
|
|
961
|
+
dest=dest,
|
|
962
|
+
mime_type=file_format,
|
|
963
|
+
big_data=big_data,
|
|
964
|
+
timeout=timeout,
|
|
965
|
+
progress=progress,
|
|
966
|
+
)
|
|
967
|
+
except ForbiddenError as err:
|
|
968
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
969
|
+
return Path(dest)
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def import_file(
|
|
973
|
+
client: ReportnetClient,
|
|
974
|
+
dataset_id: int,
|
|
975
|
+
dataflow_id: int,
|
|
976
|
+
path: str | os.PathLike[str],
|
|
977
|
+
*,
|
|
978
|
+
table_schema_id: str | None = None,
|
|
979
|
+
replace: bool = False,
|
|
980
|
+
confirm: bool = False,
|
|
981
|
+
check: bool = True,
|
|
982
|
+
provider_id: int | None = None,
|
|
983
|
+
delimiter: str | None = None,
|
|
984
|
+
timeout: float = 300.0,
|
|
985
|
+
progress: Progress = None,
|
|
986
|
+
) -> dict[str, Any]:
|
|
987
|
+
"""Upload a CSV (or other import file) into a dataset and wait until it is in.
|
|
988
|
+
|
|
989
|
+
Before uploading a `.csv`, the file is checked against the dataset schema
|
|
990
|
+
(`check_import_file`): the name must be a table name and the header must name
|
|
991
|
+
every field of that table, else `ValueError` lists the problems and nothing is
|
|
992
|
+
sent (`check=False` skips this; so does an unreadable schema).
|
|
993
|
+
`replace=True` deletes data, so it also needs `confirm=True`.
|
|
994
|
+
|
|
995
|
+
The file name must match a table name in the dataset (``TestTable.csv`` for
|
|
996
|
+
table ``TestTable``); otherwise Reportnet cancels the job ("File name does not
|
|
997
|
+
match any table name") and nothing is imported.
|
|
998
|
+
|
|
999
|
+
`delimiter` is the file's field separator; leave it None for the dataset's own
|
|
1000
|
+
setting. A multi-column file whose separator differs from the dataset's is
|
|
1001
|
+
cancelled ("headers ... do not match"): live on the sandbox the delimiter is `|`.
|
|
1002
|
+
`replace=True` emptied the dataset's *other* tables too (live, 2026-11-07,
|
|
1003
|
+
the sandbox: a one-table file left the second table empty), so re-import the tables
|
|
1004
|
+
you want to keep afterwards.
|
|
1005
|
+
|
|
1006
|
+
Uploads with the async big-file import, waits for its job, and then
|
|
1007
|
+
confirms through the dataset's import statistics. Returns those statistics
|
|
1008
|
+
(``{table_id: {"numberOfRecordsImported": ..., ...}}``). This WRITES to the
|
|
1009
|
+
dataset; `replace=True` also deletes the existing data first. Raises
|
|
1010
|
+
`reportnet_api.jobs.JobError` if the job fails or `timeout` seconds pass
|
|
1011
|
+
(the upload may still finish later). `progress=True` shows one updating status
|
|
1012
|
+
line; a callable gets event dicts. Ctrl-C while waiting only stops the waiting:
|
|
1013
|
+
the import keeps running on the server and cannot be cancelled from here.
|
|
1014
|
+
"""
|
|
1015
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
1016
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
1017
|
+
if not Path(path).is_file():
|
|
1018
|
+
raise FileNotFoundError(f"No such file: {path}")
|
|
1019
|
+
if replace and not confirm:
|
|
1020
|
+
raise ValueError(
|
|
1021
|
+
"replace=True deletes the dataset's existing data first (live: also the other "
|
|
1022
|
+
"tables'); pass confirm=True to go ahead"
|
|
1023
|
+
)
|
|
1024
|
+
if check:
|
|
1025
|
+
problems = check_import_file(client, dataset_id, path, delimiter=delimiter)
|
|
1026
|
+
if problems:
|
|
1027
|
+
raise ValueError("The file would be rejected by Reportnet:\n- " + "\n- ".join(problems))
|
|
1028
|
+
try:
|
|
1029
|
+
return client.datasets.import_file_and_wait(
|
|
1030
|
+
dataset_id,
|
|
1031
|
+
path,
|
|
1032
|
+
dataflow_id=dataflow_id,
|
|
1033
|
+
provider_id=provider_id,
|
|
1034
|
+
table_schema_id=table_schema_id,
|
|
1035
|
+
replace=True if replace else None, # omit rather than send replace=false
|
|
1036
|
+
delimiter=delimiter,
|
|
1037
|
+
timeout=timeout,
|
|
1038
|
+
progress=progress,
|
|
1039
|
+
)
|
|
1040
|
+
except ForbiddenError as err:
|
|
1041
|
+
raise _hinted(err, _FORBIDDEN_HINT) from err
|
|
1042
|
+
except JobError as err:
|
|
1043
|
+
if "do not contain any records" in str(err) or "headers" in str(err):
|
|
1044
|
+
raise JobError(f"{err} {_HEADERS_HINT}") from err
|
|
1045
|
+
raise
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
_HEADERS_HINT = (
|
|
1049
|
+
"(The web app reports this as wrong headers: the first line must name every field "
|
|
1050
|
+
"of the table, separated by the dataset's delimiter; see check_import_file().)"
|
|
1051
|
+
)
|
|
1052
|
+
_DELIMITERS = (",", "|", ";", "\t")
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
def _schema_fields(client: ReportnetClient, dataset_id: int) -> dict[str, list[tuple[str, str]]]:
|
|
1056
|
+
"""``{table name: [(field name, DataType), ...]}`` in schema order."""
|
|
1057
|
+
schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
|
|
1058
|
+
out: dict[str, list[tuple[str, str]]] = {}
|
|
1059
|
+
for t in schema.tableSchemas or []:
|
|
1060
|
+
fields = t.recordSchema.fieldSchema if t.recordSchema else None
|
|
1061
|
+
out[t.nameTableSchema or ""] = [
|
|
1062
|
+
(f.name, str(getattr(f.type, "value", f.type))) for f in fields or [] if f.name
|
|
1063
|
+
]
|
|
1064
|
+
return out
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def check_import_file(
|
|
1068
|
+
client: ReportnetClient,
|
|
1069
|
+
dataset_id: int,
|
|
1070
|
+
path: str | os.PathLike[str],
|
|
1071
|
+
*,
|
|
1072
|
+
delimiter: str | None = None,
|
|
1073
|
+
) -> list[str]:
|
|
1074
|
+
"""Read-only pre-check of an import file; returns the problems found (empty = ok).
|
|
1075
|
+
|
|
1076
|
+
Only `.csv` files are checked (a ZIP or workbook is left to Reportnet). The name
|
|
1077
|
+
must match a table of the dataset (ignoring case: whether the backend is
|
|
1078
|
+
case-sensitive is unverified) and the first line must name exactly that table's
|
|
1079
|
+
fields, split by `delimiter` (or by any of `,` `|` `;` tab when None, since the
|
|
1080
|
+
dataset's own delimiter cannot be read). Live: a header with only some of the
|
|
1081
|
+
fields was cancelled (the sandbox). If the schema cannot be read, returns [].
|
|
1082
|
+
"""
|
|
1083
|
+
path = Path(path)
|
|
1084
|
+
if path.suffix.lower() != ".csv":
|
|
1085
|
+
return []
|
|
1086
|
+
try:
|
|
1087
|
+
tables = _schema_fields(client, dataset_id)
|
|
1088
|
+
except ReportnetAPIError:
|
|
1089
|
+
return []
|
|
1090
|
+
by_lower = {name.lower(): name for name in tables}
|
|
1091
|
+
table = by_lower.get(path.stem.lower())
|
|
1092
|
+
if table is None:
|
|
1093
|
+
return [f"File name {path.name!r} matches no table; tables: {sorted(tables)}"]
|
|
1094
|
+
wanted = [name for name, _ in tables[table]]
|
|
1095
|
+
with path.open("rb") as fh: # only the first line matters; the rest may be any encoding
|
|
1096
|
+
raw = fh.readline()
|
|
1097
|
+
first = raw.decode("utf-8-sig", errors="replace").rstrip("\r\n")
|
|
1098
|
+
best: tuple[set[str], set[str]] | None = None
|
|
1099
|
+
for d in (delimiter,) if delimiter else _DELIMITERS:
|
|
1100
|
+
cols = [c.strip().strip('"').strip() for c in first.split(d)]
|
|
1101
|
+
missing, extra = set(wanted) - set(cols), set(cols) - set(wanted)
|
|
1102
|
+
if not missing and not extra:
|
|
1103
|
+
return []
|
|
1104
|
+
if best is None or len(missing) + len(extra) < len(best[0]) + len(best[1]):
|
|
1105
|
+
best = (missing, extra)
|
|
1106
|
+
assert best is not None
|
|
1107
|
+
missing, extra = best
|
|
1108
|
+
parts = []
|
|
1109
|
+
if missing:
|
|
1110
|
+
parts.append(f"missing {sorted(missing)}")
|
|
1111
|
+
if extra:
|
|
1112
|
+
parts.append(f"unknown {sorted(extra)}")
|
|
1113
|
+
where = f"delimiter {delimiter!r}" if delimiter else "any of , | ; tab"
|
|
1114
|
+
return [
|
|
1115
|
+
f"Header of {path.name!r} does not match table {table!r} ({where}): " + ", ".join(parts)
|
|
1116
|
+
]
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _format_cell(value: Any, field_type: str) -> str:
|
|
1120
|
+
"""Text for one cell in the form the backend exports (see `read_tables`)."""
|
|
1121
|
+
pd = _pandas()
|
|
1122
|
+
if value is None or value is pd.NA:
|
|
1123
|
+
return ""
|
|
1124
|
+
is_real = isinstance(value, numbers.Real) and not isinstance(value, numbers.Integral | bool)
|
|
1125
|
+
if is_real and value != value: # NaN, also numpy's
|
|
1126
|
+
return ""
|
|
1127
|
+
if value is pd.NaT:
|
|
1128
|
+
return ""
|
|
1129
|
+
if field_type == "DATE" and hasattr(value, "strftime"):
|
|
1130
|
+
return str(value.strftime("%Y-%m-%d"))
|
|
1131
|
+
if field_type == "DATETIME" and hasattr(value, "strftime"):
|
|
1132
|
+
ts = pd.Timestamp(value)
|
|
1133
|
+
ts = ts.tz_convert("UTC") if ts.tzinfo else ts
|
|
1134
|
+
return str(ts.strftime("%Y-%m-%dT%H:%M:%SZ"))
|
|
1135
|
+
if field_type == "BOOLEAN" and isinstance(value, bool):
|
|
1136
|
+
return "true" if value else "false"
|
|
1137
|
+
if is_real: # python and numpy floats: plain digits, never 1e-07 or 1e+16
|
|
1138
|
+
if value in (float("inf"), float("-inf")):
|
|
1139
|
+
raise ValueError(f"{value} cannot be stored in a {field_type} field")
|
|
1140
|
+
text = format(Decimal(str(value)), "f") # str() is the shortest exact form
|
|
1141
|
+
if "." in text:
|
|
1142
|
+
text = text.rstrip("0").rstrip(".")
|
|
1143
|
+
return "0" if text in ("-0", "") else text
|
|
1144
|
+
return str(value)
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def import_dataframe(
|
|
1148
|
+
client: ReportnetClient,
|
|
1149
|
+
dataset_id: int,
|
|
1150
|
+
dataflow_id: int,
|
|
1151
|
+
table_name: str,
|
|
1152
|
+
df: pd.DataFrame,
|
|
1153
|
+
*,
|
|
1154
|
+
delimiter: str = ",",
|
|
1155
|
+
**kwargs: Any,
|
|
1156
|
+
) -> dict[str, Any]:
|
|
1157
|
+
"""Import a DataFrame into one table of a dataset (`import_file` underneath). WRITES.
|
|
1158
|
+
|
|
1159
|
+
Writes a temporary `<table_name>.csv` with a header for EVERY field of the table
|
|
1160
|
+
in schema order (fields missing from `df` are left empty; columns that are not
|
|
1161
|
+
fields raise `ValueError`), values formatted by field type (dates `YYYY-MM-DD`,
|
|
1162
|
+
datetimes UTC ISO, NA empty), and uploads it with `delimiter`. A value containing
|
|
1163
|
+
the delimiter or a line break raises `ValueError` (how the backend reads quotes is
|
|
1164
|
+
unverified). Other keyword arguments (`replace`, `confirm`, `provider_id`,
|
|
1165
|
+
`progress`, ...) go to `import_file`; `replace=True` needs `confirm=True`.
|
|
1166
|
+
|
|
1167
|
+
The delimiter is always sent with the upload, so it overrides the dataset's own
|
|
1168
|
+
setting. Live-tested once, on a sandbox dataset: a typed read, then
|
|
1169
|
+
`import_dataframe(..., delimiter="|", replace=True, confirm=True)` and a second read
|
|
1170
|
+
gave identical data. Other field types and datasets are not verified.
|
|
1171
|
+
"""
|
|
1172
|
+
_pandas()
|
|
1173
|
+
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
|
|
1174
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
1175
|
+
tables = _schema_fields(client, dataset_id)
|
|
1176
|
+
if table_name not in tables:
|
|
1177
|
+
raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}")
|
|
1178
|
+
fields = tables[table_name]
|
|
1179
|
+
duplicated = sorted({str(c) for c in df.columns if list(df.columns).count(c) > 1})
|
|
1180
|
+
if duplicated:
|
|
1181
|
+
raise ValueError(f"Duplicate column names {duplicated}: each field can appear only once")
|
|
1182
|
+
unknown = [c for c in df.columns if c not in {n for n, _ in fields}]
|
|
1183
|
+
if unknown:
|
|
1184
|
+
raise ValueError(
|
|
1185
|
+
f"Columns {unknown} are not fields of {table_name!r}: {[n for n, _ in fields]}"
|
|
1186
|
+
)
|
|
1187
|
+
rows = {
|
|
1188
|
+
name: [_format_cell(v, ftype) for v in df[name]] if name in df.columns else [""] * len(df)
|
|
1189
|
+
for name, ftype in fields
|
|
1190
|
+
}
|
|
1191
|
+
for name, cells in rows.items():
|
|
1192
|
+
bad = next((c for c in cells if delimiter in c or "\n" in c or "\r" in c), None)
|
|
1193
|
+
if bad is not None:
|
|
1194
|
+
raise ValueError(
|
|
1195
|
+
f"Column {name!r} has a value with the delimiter or a line break: {bad!r}"
|
|
1196
|
+
)
|
|
1197
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
1198
|
+
file = Path(tmp) / f"{table_name}.csv"
|
|
1199
|
+
lines = [delimiter.join(rows)] + [
|
|
1200
|
+
delimiter.join(cells) for cells in zip(*rows.values(), strict=True)
|
|
1201
|
+
]
|
|
1202
|
+
file.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
1203
|
+
return import_file(client, dataset_id, dataflow_id, file, delimiter=delimiter, **kwargs)
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
# What each credential class is *expected* to manage, per feature family, as
|
|
1207
|
+
# recorded in docs/live_testing_notes.md ("Credential requirement matrix",
|
|
1208
|
+
# sections) and the live-testing notes. Static text, not
|
|
1209
|
+
# probed: check_access() prints it labelled "expected". Update it with those notes.
|
|
1210
|
+
_EXPECTED_ACCESS: tuple[tuple[str, str, str], ...] = (
|
|
1211
|
+
# (feature family, ApiKey, Bearer)
|
|
1212
|
+
(
|
|
1213
|
+
"listing dataflows/datasets, schema reads",
|
|
1214
|
+
"worked live",
|
|
1215
|
+
"worked live (dataflow read, refreshed Bearer)",
|
|
1216
|
+
),
|
|
1217
|
+
(
|
|
1218
|
+
"table read / sync ETL export (read_table)",
|
|
1219
|
+
"worked live",
|
|
1220
|
+
"403 live for a Custodian/Steward user (ETL is gated separately)",
|
|
1221
|
+
),
|
|
1222
|
+
(
|
|
1223
|
+
"job exports (export_to_file, v4/v5)",
|
|
1224
|
+
"worked live",
|
|
1225
|
+
"403 live on v4 for a Custodian/Steward user",
|
|
1226
|
+
),
|
|
1227
|
+
("file import (import_file)", "worked live", "worked live"),
|
|
1228
|
+
(
|
|
1229
|
+
"schema ZIP export, list imported files",
|
|
1230
|
+
"403 live (needs Custodian/Steward)",
|
|
1231
|
+
"worked live for a Custodian/Steward user",
|
|
1232
|
+
),
|
|
1233
|
+
(
|
|
1234
|
+
"historic releases",
|
|
1235
|
+
"403 live on a Design dataset",
|
|
1236
|
+
"403 live on a Design dataset (cause unconfirmed)",
|
|
1237
|
+
),
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def _probe_failure(err: ReportnetAPIError) -> tuple[str, str]:
|
|
1242
|
+
"""(verdict, detail) for a failed probe; the detail is explain()'s second line."""
|
|
1243
|
+
lines = err.explain().splitlines()
|
|
1244
|
+
return err.__class__.__name__, lines[1] if len(lines) > 1 else lines[0]
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def check_access(
|
|
1248
|
+
client: ReportnetClient, dataflow_id: int | None = None, *, show: bool = True
|
|
1249
|
+
) -> list[dict[str, str]]:
|
|
1250
|
+
"""Read-only self-check of a credential: which kind, alive, dataflow reachable.
|
|
1251
|
+
|
|
1252
|
+
Makes one call (list dataflows, page size 1) and, with `dataflow_id`, one
|
|
1253
|
+
more (the dataflow's datasets). Nothing is written. Returns rows with
|
|
1254
|
+
``check``, ``result`` and ``detail`` and prints them as a table unless
|
|
1255
|
+
`show=False`. The credential itself is never printed.
|
|
1256
|
+
|
|
1257
|
+
The ``expected`` rows are static advice for the credential's scheme taken
|
|
1258
|
+
from the live-testing notes; they were not probed by this call.
|
|
1259
|
+
"""
|
|
1260
|
+
scheme = client._core.scheme
|
|
1261
|
+
rows: list[dict[str, str]] = [
|
|
1262
|
+
{
|
|
1263
|
+
"check": "credential",
|
|
1264
|
+
"result": scheme,
|
|
1265
|
+
"detail": f"sent as `Authorization: {scheme} ...`",
|
|
1266
|
+
}
|
|
1267
|
+
]
|
|
1268
|
+
alive = False
|
|
1269
|
+
try:
|
|
1270
|
+
client.dataflow.dataflows("reporting", page_size=1, page_num=0)
|
|
1271
|
+
alive = True
|
|
1272
|
+
rows.append({"check": "credential alive", "result": "yes", "detail": "list dataflows ok"})
|
|
1273
|
+
except ReportnetAPIError as err:
|
|
1274
|
+
verdict, detail = _probe_failure(err)
|
|
1275
|
+
if err.status_code == 401 or (
|
|
1276
|
+
err.status_code == 500 and _is_rejected_credential_500(err.response_body)
|
|
1277
|
+
):
|
|
1278
|
+
verdict = "no (rejected)"
|
|
1279
|
+
detail = "expired, revoked, or sent with the wrong scheme"
|
|
1280
|
+
if scheme == "ApiKey":
|
|
1281
|
+
detail += " (a Bearer token needs scheme='Bearer')"
|
|
1282
|
+
rows.append({"check": "credential alive", "result": verdict, "detail": detail})
|
|
1283
|
+
except httpx.TransportError as err:
|
|
1284
|
+
rows.append(
|
|
1285
|
+
{"check": "credential alive", "result": "unknown", "detail": f"network error: {err}"}
|
|
1286
|
+
)
|
|
1287
|
+
if dataflow_id is not None:
|
|
1288
|
+
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
|
|
1289
|
+
check = f"dataflow {dataflow_id}"
|
|
1290
|
+
if not alive:
|
|
1291
|
+
rows.append({"check": check, "result": "skipped", "detail": "credential not usable"})
|
|
1292
|
+
else:
|
|
1293
|
+
try:
|
|
1294
|
+
flow = client.dataflow.dataflow_datasets(dataflow_id)
|
|
1295
|
+
except ReportnetAPIError as err:
|
|
1296
|
+
if err.status_code == 403:
|
|
1297
|
+
verdict, detail = "no access", "the server refuses this credential (403)"
|
|
1298
|
+
elif "accessReferenceEntity" in err.response_body:
|
|
1299
|
+
verdict, detail = "not found", "unknown id, or not visible (reported as 500)"
|
|
1300
|
+
else:
|
|
1301
|
+
verdict, detail = _probe_failure(err)
|
|
1302
|
+
rows.append({"check": check, "result": verdict, "detail": detail})
|
|
1303
|
+
except httpx.TransportError as err:
|
|
1304
|
+
rows.append(
|
|
1305
|
+
{"check": check, "result": "unknown", "detail": f"network error: {err}"}
|
|
1306
|
+
)
|
|
1307
|
+
else:
|
|
1308
|
+
count = sum(len((flow or {}).get(group) or []) for group in _DATASET_GROUPS)
|
|
1309
|
+
rows.append(
|
|
1310
|
+
{"check": check, "result": "reachable", "detail": f"{count} dataset(s) listed"}
|
|
1311
|
+
if flow
|
|
1312
|
+
else {
|
|
1313
|
+
"check": check,
|
|
1314
|
+
"result": "unclear",
|
|
1315
|
+
"detail": "empty 200 answer (unknown id or no access)",
|
|
1316
|
+
}
|
|
1317
|
+
)
|
|
1318
|
+
column = 2 if scheme == "Bearer" else 1 if scheme == "ApiKey" else None
|
|
1319
|
+
for entry in _EXPECTED_ACCESS:
|
|
1320
|
+
advice = entry[column] if column else "no advice for this scheme"
|
|
1321
|
+
rows.append(
|
|
1322
|
+
{"check": f"expected: {entry[0]}", "result": advice, "detail": "not probed now"}
|
|
1323
|
+
)
|
|
1324
|
+
if show:
|
|
1325
|
+
width = max(len(r["check"]) for r in rows)
|
|
1326
|
+
for r in rows:
|
|
1327
|
+
print(f"{r['check']:<{width}} {r['result']} ({r['detail']})")
|
|
1328
|
+
return rows
|
|
1329
|
+
|
|
1330
|
+
|
|
1331
|
+
_GUIDE = """\
|
|
1332
|
+
Reportnet cheat sheet (full reference: docs/reference/index.md; terms: docs/glossary.md)
|
|
1333
|
+
|
|
1334
|
+
1. Connect client = connect_interactive() # hidden prompt; or connect()
|
|
1335
|
+
check_access(client) # read-only: credential kind, alive?
|
|
1336
|
+
2. Find ids list_dataflows(client) # dataflow ids and names
|
|
1337
|
+
list_datasets(client, dataflow_id) # dataset ids (NOT the dataflow id)
|
|
1338
|
+
flow = find_dataflow(client, id_or_name); flow.dataset(id_or_country)
|
|
1339
|
+
3. Read list_tables(client, dataset_id); read_table(client, dataset_id, dataflow_id)
|
|
1340
|
+
read_tables(...) for all tables; read_many(datasets, table) to stack
|
|
1341
|
+
4. Validation validation_summary / validation_errors; validate_and_wait (WRITES)
|
|
1342
|
+
export_validations(client, dataset_id, dest)
|
|
1343
|
+
5. Export export_table(...) one table to csv/xlsx; export_to_file(...) whole dataset
|
|
1344
|
+
6. Upload import_file(client, dataset_id, dataflow_id, path) (WRITES)
|
|
1345
|
+
7. Wrong? a ReportnetAPIError has .explain(); an ApiKey can only reach its own
|
|
1346
|
+
dataflow with Reporter rights, the ETL reads/exports need an ApiKey.
|
|
1347
|
+
8. Search reportnet_api.find("export") # any method or helper by keyword
|
|
1348
|
+
"""
|
|
1349
|
+
|
|
1350
|
+
|
|
1351
|
+
def guide(*, show: bool = True) -> str:
|
|
1352
|
+
"""Print a one-page cheat sheet of the helpers in the order a notebook uses them.
|
|
1353
|
+
|
|
1354
|
+
Returns the text; prints it unless `show=False`. Search every method and helper
|
|
1355
|
+
by keyword with `reportnet_api.find(text)`.
|
|
1356
|
+
"""
|
|
1357
|
+
if show:
|
|
1358
|
+
print(_GUIDE)
|
|
1359
|
+
return _GUIDE
|