reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
reportnet_api/handles.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Handles for a dataflow and its datasets (, stages 1-3).
|
|
2
|
+
|
|
3
|
+
A `Dataflow` or `Dataset` carries the ids a notebook user would otherwise pass by
|
|
4
|
+
hand (dataflow id, dataset id, provider id, the data-lake flag) and resolves
|
|
5
|
+
datasets by id, country code or name. Handles are immutable snapshots; call
|
|
6
|
+
`Dataflow.refresh()` for a new one. They delegate to the `easy` functions and add
|
|
7
|
+
no endpoint.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import numbers
|
|
13
|
+
import os
|
|
14
|
+
import warnings
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import TYPE_CHECKING, Any
|
|
19
|
+
|
|
20
|
+
from . import easy
|
|
21
|
+
from ._exceptions import AuthenticationError, ReportnetAPIError
|
|
22
|
+
from .client import ReportnetClient
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
import pandas as pd
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _int_or_none(value: Any) -> int | None:
|
|
29
|
+
"""Ints come back from DataFrames as floats with NaN for missing values."""
|
|
30
|
+
try:
|
|
31
|
+
return None if value is None or value != value else int(value)
|
|
32
|
+
except (TypeError, ValueError):
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _as_id(key: object) -> int | None:
|
|
37
|
+
"""An integer key (numpy integers included; bool is not an id), else None."""
|
|
38
|
+
if isinstance(key, numbers.Integral) and not isinstance(key, bool):
|
|
39
|
+
return int(key)
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _describe(row: dict[str, Any], codes: dict[int, str] | None) -> str:
|
|
44
|
+
pid = _int_or_none(row.get("provider_id"))
|
|
45
|
+
code = f", {codes[pid]}" if codes and pid in codes else ""
|
|
46
|
+
return f"{int(row['dataset_id'])} {row['name']} ({row['kind']}{code})"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class DataLakeError(RuntimeError):
|
|
50
|
+
"""A plain table read was refused: the dataset is in a data-lake dataflow."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Dataset:
|
|
55
|
+
"""One dataset: ids, kind and the country it belongs to (if any)."""
|
|
56
|
+
|
|
57
|
+
id: int
|
|
58
|
+
name: str
|
|
59
|
+
kind: str
|
|
60
|
+
provider_id: int | None
|
|
61
|
+
country_code: str | None
|
|
62
|
+
dataflow_id: int
|
|
63
|
+
client: ReportnetClient = field(repr=False, compare=False)
|
|
64
|
+
big_data: bool = False # True when the dataflow is a data-lake one
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def tables(self) -> pd.DataFrame:
|
|
68
|
+
"""The dataset's tables as a DataFrame (`easy.list_tables`)."""
|
|
69
|
+
return easy.list_tables(self.client, self.id)
|
|
70
|
+
|
|
71
|
+
def _refuse_data_lake(self, allow: bool) -> None:
|
|
72
|
+
if self.big_data and not allow:
|
|
73
|
+
raise DataLakeError(
|
|
74
|
+
f"Dataset {self.id} ({self.name}) is in a data-lake dataflow ({self.dataflow_id}): "
|
|
75
|
+
"the plain table read answers 200 with 0 rows there even when data exists. "
|
|
76
|
+
"Use .export(dest) for the data, or pass "
|
|
77
|
+
"allow_data_lake=True to try the plain read anyway."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def read_all(self, *, allow_data_lake: bool = False, **kwargs: Any) -> dict[str, pd.DataFrame]:
|
|
81
|
+
"""All tables as ``{table_name: DataFrame}`` (`easy.read_tables`).
|
|
82
|
+
|
|
83
|
+
The handle's `provider_id` is passed unless you give one. Other keyword
|
|
84
|
+
arguments (`page_size`, `progress`, `table_schema_id`) go to `read_tables`;
|
|
85
|
+
see it for the ETL credential caveats. Raises `DataLakeError` on a data-lake
|
|
86
|
+
dataflow (`big_data`), where this route returns no rows: use `export()`, or
|
|
87
|
+
`allow_data_lake=True` to try anyway.
|
|
88
|
+
"""
|
|
89
|
+
self._refuse_data_lake(allow_data_lake)
|
|
90
|
+
kwargs.setdefault("provider_id", self.provider_id)
|
|
91
|
+
return easy.read_tables(self.client, self.id, self.dataflow_id, **kwargs)
|
|
92
|
+
|
|
93
|
+
def read(
|
|
94
|
+
self, table: str | None = None, *, allow_data_lake: bool = False, **kwargs: Any
|
|
95
|
+
) -> pd.DataFrame:
|
|
96
|
+
"""One table as a DataFrame (`easy.read_table`).
|
|
97
|
+
|
|
98
|
+
Without `table` the dataset must hold exactly one table with data. Raises
|
|
99
|
+
`DataLakeError` on a data-lake dataflow, as `read_all` does.
|
|
100
|
+
"""
|
|
101
|
+
self._refuse_data_lake(allow_data_lake)
|
|
102
|
+
kwargs.setdefault("provider_id", self.provider_id)
|
|
103
|
+
return easy.read_table(self.client, self.id, self.dataflow_id, table, **kwargs)
|
|
104
|
+
|
|
105
|
+
def validation_errors(self, **kwargs: Any) -> pd.DataFrame:
|
|
106
|
+
"""Failed validations, one row per failing rule (`easy.validation_errors`)."""
|
|
107
|
+
return easy.validation_errors(self.client, self.id, self.dataflow_id, **kwargs)
|
|
108
|
+
|
|
109
|
+
def validation_summary(self) -> pd.DataFrame:
|
|
110
|
+
"""Failures counted per table and level (`easy.validation_summary`)."""
|
|
111
|
+
return easy.validation_summary(self.client, self.id, self.dataflow_id)
|
|
112
|
+
|
|
113
|
+
def export(self, dest: str | os.PathLike[str], **kwargs: Any) -> Path:
|
|
114
|
+
"""Export the dataset as a ZIP of CSV or Parquet files at `dest` (`easy.export_to_file`).
|
|
115
|
+
|
|
116
|
+
Starts an export job and leaves a job record behind. Keyword arguments
|
|
117
|
+
(`file_format`, `table_schema_id`, `timeout`, `progress`) go to `export_to_file`.
|
|
118
|
+
"""
|
|
119
|
+
return easy.export_to_file(self.client, self.id, self.dataflow_id, dest, **kwargs)
|
|
120
|
+
|
|
121
|
+
def upload(self, path: str | os.PathLike[str], **kwargs: Any) -> dict[str, Any]:
|
|
122
|
+
"""Import a file into the dataset and wait for it (`easy.import_file`). WRITES.
|
|
123
|
+
|
|
124
|
+
The file name must match a table name and the header must name every field
|
|
125
|
+
(checked first). `replace=True` deletes the existing data first and needs
|
|
126
|
+
`confirm=True`. The handle's `provider_id` is passed unless you give one.
|
|
127
|
+
"""
|
|
128
|
+
kwargs.setdefault("provider_id", self.provider_id)
|
|
129
|
+
return easy.import_file(self.client, self.id, self.dataflow_id, path, **kwargs)
|
|
130
|
+
|
|
131
|
+
def validate(self, **kwargs: Any) -> pd.DataFrame:
|
|
132
|
+
"""Start a validation, wait for it, return the failures (`easy.validate_and_wait`). WRITES.
|
|
133
|
+
|
|
134
|
+
Replaces the dataset's validation results and locks the dataset while it runs.
|
|
135
|
+
"""
|
|
136
|
+
return easy.validate_and_wait(self.client, self.id, self.dataflow_id, **kwargs)
|
|
137
|
+
|
|
138
|
+
def export_validations(self, dest: str | os.PathLike[str], **kwargs: Any) -> Path:
|
|
139
|
+
"""Save the validation failures as a CSV at `dest` (`easy.export_validations`).
|
|
140
|
+
|
|
141
|
+
Close any open Reportnet page for this dataset first: the backend deletes
|
|
142
|
+
the file once the web app downloads it.
|
|
143
|
+
"""
|
|
144
|
+
return easy.export_validations(self.client, self.id, dest, **kwargs)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@dataclass(frozen=True)
|
|
148
|
+
class Dataflow:
|
|
149
|
+
"""One dataflow. `big_data` is True for a data-lake dataflow."""
|
|
150
|
+
|
|
151
|
+
id: int
|
|
152
|
+
name: str
|
|
153
|
+
status: str | None
|
|
154
|
+
big_data: bool
|
|
155
|
+
client: ReportnetClient = field(repr=False, compare=False)
|
|
156
|
+
# Provider-code cache, filled on first lookup; None inside means unreadable.
|
|
157
|
+
_cache: dict[str, Any] = field(
|
|
158
|
+
default_factory=dict, init=False, repr=False, compare=False, hash=False
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def datasets(self) -> pd.DataFrame:
|
|
163
|
+
"""The dataflow's datasets as a DataFrame (`easy.list_datasets`)."""
|
|
164
|
+
return easy.list_datasets(self.client, self.id)
|
|
165
|
+
|
|
166
|
+
def refresh(self) -> Dataflow:
|
|
167
|
+
"""A new handle read afresh from the server; this one is left unchanged."""
|
|
168
|
+
return find_dataflow(self.client, self.id)
|
|
169
|
+
|
|
170
|
+
def _codes(self) -> dict[int, str] | None:
|
|
171
|
+
if "codes" not in self._cache:
|
|
172
|
+
try:
|
|
173
|
+
table = easy.list_providers(self.client)
|
|
174
|
+
self._cache["codes"] = {
|
|
175
|
+
int(r["provider_id"]): str(r["code"]) for r in table.to_dict("records")
|
|
176
|
+
}
|
|
177
|
+
except ReportnetAPIError: # 403, but a bad ApiKey also shows up as 401/500
|
|
178
|
+
self._cache["codes"] = None
|
|
179
|
+
codes: dict[int, str] | None = self._cache["codes"]
|
|
180
|
+
return codes
|
|
181
|
+
|
|
182
|
+
def dataset(self, key: int | str, *, kind: str | None = None) -> Dataset:
|
|
183
|
+
"""One dataset by id, country code or name; exactly one match or `LookupError`.
|
|
184
|
+
|
|
185
|
+
A string is tried as a provider code (case-insensitive), then as a name:
|
|
186
|
+
exact first, then substring. `kind` ("design", "reporting", "test", ...)
|
|
187
|
+
narrows the candidates first.
|
|
188
|
+
"""
|
|
189
|
+
rows: list[dict[str, Any]] = self.datasets.to_dict("records")
|
|
190
|
+
if kind is not None:
|
|
191
|
+
rows = [r for r in rows if r["kind"] == kind]
|
|
192
|
+
wanted = _as_id(key)
|
|
193
|
+
codes = self._codes()
|
|
194
|
+
if wanted is not None:
|
|
195
|
+
found = [r for r in rows if int(r["dataset_id"]) == wanted]
|
|
196
|
+
else:
|
|
197
|
+
k = str(key).strip().lower()
|
|
198
|
+
if not k:
|
|
199
|
+
raise ValueError("The dataset key is empty: give an id, a country code or a name")
|
|
200
|
+
if codes is None and len(k) <= 2:
|
|
201
|
+
# Looks like a country code, which cannot be told from a name fragment.
|
|
202
|
+
raise LookupError(
|
|
203
|
+
f"Cannot resolve {key!r} in dataflow {self.id}: provider codes could not "
|
|
204
|
+
"be read (no access to providers), so use the dataset id or its full name"
|
|
205
|
+
)
|
|
206
|
+
if codes and k in {c.lower() for c in codes.values()}:
|
|
207
|
+
by_provider = {pid for pid, c in codes.items() if c.lower() == k}
|
|
208
|
+
found = [r for r in rows if _int_or_none(r["provider_id"]) in by_provider]
|
|
209
|
+
if not found: # a real country with no dataset here: never fall back to names
|
|
210
|
+
listing = "; ".join(_describe(r, codes) for r in rows)
|
|
211
|
+
raise LookupError(
|
|
212
|
+
f"No dataset for provider {key!r} in dataflow {self.id}: {listing}"
|
|
213
|
+
)
|
|
214
|
+
else:
|
|
215
|
+
found = [r for r in rows if str(r["name"]).lower() == k]
|
|
216
|
+
if not found:
|
|
217
|
+
found = [r for r in rows if k in str(r["name"]).lower()]
|
|
218
|
+
if len(found) != 1:
|
|
219
|
+
listing = "; ".join(_describe(r, codes) for r in (found or rows))
|
|
220
|
+
note = (
|
|
221
|
+
""
|
|
222
|
+
if codes is not None
|
|
223
|
+
else " (provider codes could not be read: no access to providers)"
|
|
224
|
+
)
|
|
225
|
+
what = "Several datasets match" if found else "No dataset matches"
|
|
226
|
+
raise LookupError(f"{what} {key!r} in dataflow {self.id}: {listing}{note}")
|
|
227
|
+
row = found[0]
|
|
228
|
+
pid = _int_or_none(row["provider_id"])
|
|
229
|
+
return Dataset(
|
|
230
|
+
id=int(row["dataset_id"]),
|
|
231
|
+
name=str(row["name"]),
|
|
232
|
+
kind=str(row["kind"]),
|
|
233
|
+
provider_id=pid,
|
|
234
|
+
country_code=codes.get(pid) if codes and pid is not None else None,
|
|
235
|
+
dataflow_id=self.id,
|
|
236
|
+
client=self.client,
|
|
237
|
+
big_data=self.big_data,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def read_many(datasets: Iterable[Dataset], table: str, **kwargs: Any) -> pd.DataFrame:
|
|
242
|
+
"""One table from several datasets, stacked, with `dataset` and `country` columns.
|
|
243
|
+
|
|
244
|
+
Reads one dataset after another (no concurrency: concurrent ETL reads are not
|
|
245
|
+
verified against the backend) with `Dataset.read_all`, so
|
|
246
|
+
`kwargs` (`page_size`, `progress`, `allow_data_lake`) go there; a data-lake
|
|
247
|
+
dataset raises `DataLakeError`. A dataset without that table, or
|
|
248
|
+
with no rows in it, is skipped and named in a `RuntimeWarning`; no rows at all
|
|
249
|
+
gives an empty frame. `country` is the dataset's country code, None if it has none.
|
|
250
|
+
"""
|
|
251
|
+
import pandas as pd
|
|
252
|
+
|
|
253
|
+
frames: list[pd.DataFrame] = []
|
|
254
|
+
skipped: list[str] = []
|
|
255
|
+
for ds in datasets:
|
|
256
|
+
frame = ds.read_all(**kwargs).get(table)
|
|
257
|
+
if frame is None or frame.empty:
|
|
258
|
+
skipped.append(f"{ds.id} {ds.name}")
|
|
259
|
+
continue
|
|
260
|
+
frame = frame.copy()
|
|
261
|
+
frame.insert(0, "country", ds.country_code)
|
|
262
|
+
frame.insert(0, "dataset", ds.name)
|
|
263
|
+
frames.append(frame)
|
|
264
|
+
if skipped:
|
|
265
|
+
warnings.warn(
|
|
266
|
+
f"No rows of table {table!r} for: {'; '.join(skipped)}",
|
|
267
|
+
RuntimeWarning,
|
|
268
|
+
stacklevel=2,
|
|
269
|
+
)
|
|
270
|
+
return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def find_dataflow(client: ReportnetClient, query: int | str) -> Dataflow:
|
|
274
|
+
"""One dataflow by id or name text; exactly one match or `LookupError`.
|
|
275
|
+
|
|
276
|
+
An int (or a string of digits) is an id: it is read from `list_dataflows()`
|
|
277
|
+
when listed there, else accepted when `list_datasets()` can read it (the
|
|
278
|
+
handle is then named after the id). Any other string matches names
|
|
279
|
+
case-insensitively, exact first, then as a substring.
|
|
280
|
+
The dataflow's `bigData` flag is read once, here.
|
|
281
|
+
"""
|
|
282
|
+
if isinstance(query, str):
|
|
283
|
+
text = query.strip()
|
|
284
|
+
if not text:
|
|
285
|
+
raise ValueError("The dataflow query is empty: give an id or part of a name")
|
|
286
|
+
if text.isdigit(): # an id read from an env var or a command line
|
|
287
|
+
query = int(text)
|
|
288
|
+
rows: list[dict[str, Any]] = []
|
|
289
|
+
try:
|
|
290
|
+
rows = easy.list_dataflows(client).to_dict("records")
|
|
291
|
+
except ReportnetAPIError:
|
|
292
|
+
if _as_id(query) is None:
|
|
293
|
+
raise # a name search needs the listing; an id can still be checked below
|
|
294
|
+
wanted = _as_id(query)
|
|
295
|
+
if wanted is not None:
|
|
296
|
+
query = wanted
|
|
297
|
+
found = [r for r in rows if _int_or_none(r.get("id")) == query]
|
|
298
|
+
if not found:
|
|
299
|
+
try:
|
|
300
|
+
easy.list_datasets(client, query)
|
|
301
|
+
except AuthenticationError:
|
|
302
|
+
raise # an expired or bad credential is not "no such dataflow"
|
|
303
|
+
except (LookupError, ReportnetAPIError) as err:
|
|
304
|
+
raise LookupError(
|
|
305
|
+
f"Dataflow {query} is not listed and could not be read: {err}"
|
|
306
|
+
) from err
|
|
307
|
+
found = [{"id": query, "name": str(query), "status": None}]
|
|
308
|
+
else:
|
|
309
|
+
k = str(query).strip().lower()
|
|
310
|
+
found = [r for r in rows if str(r.get("name")).lower() == k]
|
|
311
|
+
if not found:
|
|
312
|
+
found = [r for r in rows if k in str(r.get("name")).lower()]
|
|
313
|
+
if len(found) != 1:
|
|
314
|
+
listing = "; ".join(f"{r.get('id')} {r.get('name')}" for r in (found or rows)[:20])
|
|
315
|
+
what = "Several dataflows match" if found else "No dataflow matches"
|
|
316
|
+
raise LookupError(f"{what} {query!r}: {listing}")
|
|
317
|
+
row = found[0]
|
|
318
|
+
flow_id = int(row["id"])
|
|
319
|
+
status = row.get("status")
|
|
320
|
+
return Dataflow(
|
|
321
|
+
id=flow_id,
|
|
322
|
+
name=str(row["name"]),
|
|
323
|
+
status=None if status is None or status != status else str(status),
|
|
324
|
+
big_data=easy.is_data_lake(client, flow_id),
|
|
325
|
+
client=client,
|
|
326
|
+
)
|