reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,326 @@
1
+ """Handles for a dataflow and its datasets (, stages 1-3).
2
+
3
+ A `Dataflow` or `Dataset` carries the ids a notebook user would otherwise pass by
4
+ hand (dataflow id, dataset id, provider id, the data-lake flag) and resolves
5
+ datasets by id, country code or name. Handles are immutable snapshots; call
6
+ `Dataflow.refresh()` for a new one. They delegate to the `easy` functions and add
7
+ no endpoint.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import numbers
13
+ import os
14
+ import warnings
15
+ from collections.abc import Iterable
16
+ from dataclasses import dataclass, field
17
+ from pathlib import Path
18
+ from typing import TYPE_CHECKING, Any
19
+
20
+ from . import easy
21
+ from ._exceptions import AuthenticationError, ReportnetAPIError
22
+ from .client import ReportnetClient
23
+
24
+ if TYPE_CHECKING:
25
+ import pandas as pd
26
+
27
+
28
+ def _int_or_none(value: Any) -> int | None:
29
+ """Ints come back from DataFrames as floats with NaN for missing values."""
30
+ try:
31
+ return None if value is None or value != value else int(value)
32
+ except (TypeError, ValueError):
33
+ return None
34
+
35
+
36
+ def _as_id(key: object) -> int | None:
37
+ """An integer key (numpy integers included; bool is not an id), else None."""
38
+ if isinstance(key, numbers.Integral) and not isinstance(key, bool):
39
+ return int(key)
40
+ return None
41
+
42
+
43
+ def _describe(row: dict[str, Any], codes: dict[int, str] | None) -> str:
44
+ pid = _int_or_none(row.get("provider_id"))
45
+ code = f", {codes[pid]}" if codes and pid in codes else ""
46
+ return f"{int(row['dataset_id'])} {row['name']} ({row['kind']}{code})"
47
+
48
+
49
+ class DataLakeError(RuntimeError):
50
+ """A plain table read was refused: the dataset is in a data-lake dataflow."""
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class Dataset:
55
+ """One dataset: ids, kind and the country it belongs to (if any)."""
56
+
57
+ id: int
58
+ name: str
59
+ kind: str
60
+ provider_id: int | None
61
+ country_code: str | None
62
+ dataflow_id: int
63
+ client: ReportnetClient = field(repr=False, compare=False)
64
+ big_data: bool = False # True when the dataflow is a data-lake one
65
+
66
+ @property
67
+ def tables(self) -> pd.DataFrame:
68
+ """The dataset's tables as a DataFrame (`easy.list_tables`)."""
69
+ return easy.list_tables(self.client, self.id)
70
+
71
+ def _refuse_data_lake(self, allow: bool) -> None:
72
+ if self.big_data and not allow:
73
+ raise DataLakeError(
74
+ f"Dataset {self.id} ({self.name}) is in a data-lake dataflow ({self.dataflow_id}): "
75
+ "the plain table read answers 200 with 0 rows there even when data exists. "
76
+ "Use .export(dest) for the data, or pass "
77
+ "allow_data_lake=True to try the plain read anyway."
78
+ )
79
+
80
+ def read_all(self, *, allow_data_lake: bool = False, **kwargs: Any) -> dict[str, pd.DataFrame]:
81
+ """All tables as ``{table_name: DataFrame}`` (`easy.read_tables`).
82
+
83
+ The handle's `provider_id` is passed unless you give one. Other keyword
84
+ arguments (`page_size`, `progress`, `table_schema_id`) go to `read_tables`;
85
+ see it for the ETL credential caveats. Raises `DataLakeError` on a data-lake
86
+ dataflow (`big_data`), where this route returns no rows: use `export()`, or
87
+ `allow_data_lake=True` to try anyway.
88
+ """
89
+ self._refuse_data_lake(allow_data_lake)
90
+ kwargs.setdefault("provider_id", self.provider_id)
91
+ return easy.read_tables(self.client, self.id, self.dataflow_id, **kwargs)
92
+
93
+ def read(
94
+ self, table: str | None = None, *, allow_data_lake: bool = False, **kwargs: Any
95
+ ) -> pd.DataFrame:
96
+ """One table as a DataFrame (`easy.read_table`).
97
+
98
+ Without `table` the dataset must hold exactly one table with data. Raises
99
+ `DataLakeError` on a data-lake dataflow, as `read_all` does.
100
+ """
101
+ self._refuse_data_lake(allow_data_lake)
102
+ kwargs.setdefault("provider_id", self.provider_id)
103
+ return easy.read_table(self.client, self.id, self.dataflow_id, table, **kwargs)
104
+
105
+ def validation_errors(self, **kwargs: Any) -> pd.DataFrame:
106
+ """Failed validations, one row per failing rule (`easy.validation_errors`)."""
107
+ return easy.validation_errors(self.client, self.id, self.dataflow_id, **kwargs)
108
+
109
+ def validation_summary(self) -> pd.DataFrame:
110
+ """Failures counted per table and level (`easy.validation_summary`)."""
111
+ return easy.validation_summary(self.client, self.id, self.dataflow_id)
112
+
113
+ def export(self, dest: str | os.PathLike[str], **kwargs: Any) -> Path:
114
+ """Export the dataset as a ZIP of CSV or Parquet files at `dest` (`easy.export_to_file`).
115
+
116
+ Starts an export job and leaves a job record behind. Keyword arguments
117
+ (`file_format`, `table_schema_id`, `timeout`, `progress`) go to `export_to_file`.
118
+ """
119
+ return easy.export_to_file(self.client, self.id, self.dataflow_id, dest, **kwargs)
120
+
121
+ def upload(self, path: str | os.PathLike[str], **kwargs: Any) -> dict[str, Any]:
122
+ """Import a file into the dataset and wait for it (`easy.import_file`). WRITES.
123
+
124
+ The file name must match a table name and the header must name every field
125
+ (checked first). `replace=True` deletes the existing data first and needs
126
+ `confirm=True`. The handle's `provider_id` is passed unless you give one.
127
+ """
128
+ kwargs.setdefault("provider_id", self.provider_id)
129
+ return easy.import_file(self.client, self.id, self.dataflow_id, path, **kwargs)
130
+
131
+ def validate(self, **kwargs: Any) -> pd.DataFrame:
132
+ """Start a validation, wait for it, return the failures (`easy.validate_and_wait`). WRITES.
133
+
134
+ Replaces the dataset's validation results and locks the dataset while it runs.
135
+ """
136
+ return easy.validate_and_wait(self.client, self.id, self.dataflow_id, **kwargs)
137
+
138
+ def export_validations(self, dest: str | os.PathLike[str], **kwargs: Any) -> Path:
139
+ """Save the validation failures as a CSV at `dest` (`easy.export_validations`).
140
+
141
+ Close any open Reportnet page for this dataset first: the backend deletes
142
+ the file once the web app downloads it.
143
+ """
144
+ return easy.export_validations(self.client, self.id, dest, **kwargs)
145
+
146
+
147
+ @dataclass(frozen=True)
148
+ class Dataflow:
149
+ """One dataflow. `big_data` is True for a data-lake dataflow."""
150
+
151
+ id: int
152
+ name: str
153
+ status: str | None
154
+ big_data: bool
155
+ client: ReportnetClient = field(repr=False, compare=False)
156
+ # Provider-code cache, filled on first lookup; None inside means unreadable.
157
+ _cache: dict[str, Any] = field(
158
+ default_factory=dict, init=False, repr=False, compare=False, hash=False
159
+ )
160
+
161
+ @property
162
+ def datasets(self) -> pd.DataFrame:
163
+ """The dataflow's datasets as a DataFrame (`easy.list_datasets`)."""
164
+ return easy.list_datasets(self.client, self.id)
165
+
166
+ def refresh(self) -> Dataflow:
167
+ """A new handle read afresh from the server; this one is left unchanged."""
168
+ return find_dataflow(self.client, self.id)
169
+
170
+ def _codes(self) -> dict[int, str] | None:
171
+ if "codes" not in self._cache:
172
+ try:
173
+ table = easy.list_providers(self.client)
174
+ self._cache["codes"] = {
175
+ int(r["provider_id"]): str(r["code"]) for r in table.to_dict("records")
176
+ }
177
+ except ReportnetAPIError: # 403, but a bad ApiKey also shows up as 401/500
178
+ self._cache["codes"] = None
179
+ codes: dict[int, str] | None = self._cache["codes"]
180
+ return codes
181
+
182
+ def dataset(self, key: int | str, *, kind: str | None = None) -> Dataset:
183
+ """One dataset by id, country code or name; exactly one match or `LookupError`.
184
+
185
+ A string is tried as a provider code (case-insensitive), then as a name:
186
+ exact first, then substring. `kind` ("design", "reporting", "test", ...)
187
+ narrows the candidates first.
188
+ """
189
+ rows: list[dict[str, Any]] = self.datasets.to_dict("records")
190
+ if kind is not None:
191
+ rows = [r for r in rows if r["kind"] == kind]
192
+ wanted = _as_id(key)
193
+ codes = self._codes()
194
+ if wanted is not None:
195
+ found = [r for r in rows if int(r["dataset_id"]) == wanted]
196
+ else:
197
+ k = str(key).strip().lower()
198
+ if not k:
199
+ raise ValueError("The dataset key is empty: give an id, a country code or a name")
200
+ if codes is None and len(k) <= 2:
201
+ # Looks like a country code, which cannot be told from a name fragment.
202
+ raise LookupError(
203
+ f"Cannot resolve {key!r} in dataflow {self.id}: provider codes could not "
204
+ "be read (no access to providers), so use the dataset id or its full name"
205
+ )
206
+ if codes and k in {c.lower() for c in codes.values()}:
207
+ by_provider = {pid for pid, c in codes.items() if c.lower() == k}
208
+ found = [r for r in rows if _int_or_none(r["provider_id"]) in by_provider]
209
+ if not found: # a real country with no dataset here: never fall back to names
210
+ listing = "; ".join(_describe(r, codes) for r in rows)
211
+ raise LookupError(
212
+ f"No dataset for provider {key!r} in dataflow {self.id}: {listing}"
213
+ )
214
+ else:
215
+ found = [r for r in rows if str(r["name"]).lower() == k]
216
+ if not found:
217
+ found = [r for r in rows if k in str(r["name"]).lower()]
218
+ if len(found) != 1:
219
+ listing = "; ".join(_describe(r, codes) for r in (found or rows))
220
+ note = (
221
+ ""
222
+ if codes is not None
223
+ else " (provider codes could not be read: no access to providers)"
224
+ )
225
+ what = "Several datasets match" if found else "No dataset matches"
226
+ raise LookupError(f"{what} {key!r} in dataflow {self.id}: {listing}{note}")
227
+ row = found[0]
228
+ pid = _int_or_none(row["provider_id"])
229
+ return Dataset(
230
+ id=int(row["dataset_id"]),
231
+ name=str(row["name"]),
232
+ kind=str(row["kind"]),
233
+ provider_id=pid,
234
+ country_code=codes.get(pid) if codes and pid is not None else None,
235
+ dataflow_id=self.id,
236
+ client=self.client,
237
+ big_data=self.big_data,
238
+ )
239
+
240
+
241
+ def read_many(datasets: Iterable[Dataset], table: str, **kwargs: Any) -> pd.DataFrame:
242
+ """One table from several datasets, stacked, with `dataset` and `country` columns.
243
+
244
+ Reads one dataset after another (no concurrency: concurrent ETL reads are not
245
+ verified against the backend) with `Dataset.read_all`, so
246
+ `kwargs` (`page_size`, `progress`, `allow_data_lake`) go there; a data-lake
247
+ dataset raises `DataLakeError`. A dataset without that table, or
248
+ with no rows in it, is skipped and named in a `RuntimeWarning`; no rows at all
249
+ gives an empty frame. `country` is the dataset's country code, None if it has none.
250
+ """
251
+ import pandas as pd
252
+
253
+ frames: list[pd.DataFrame] = []
254
+ skipped: list[str] = []
255
+ for ds in datasets:
256
+ frame = ds.read_all(**kwargs).get(table)
257
+ if frame is None or frame.empty:
258
+ skipped.append(f"{ds.id} {ds.name}")
259
+ continue
260
+ frame = frame.copy()
261
+ frame.insert(0, "country", ds.country_code)
262
+ frame.insert(0, "dataset", ds.name)
263
+ frames.append(frame)
264
+ if skipped:
265
+ warnings.warn(
266
+ f"No rows of table {table!r} for: {'; '.join(skipped)}",
267
+ RuntimeWarning,
268
+ stacklevel=2,
269
+ )
270
+ return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
271
+
272
+
273
+ def find_dataflow(client: ReportnetClient, query: int | str) -> Dataflow:
274
+ """One dataflow by id or name text; exactly one match or `LookupError`.
275
+
276
+ An int (or a string of digits) is an id: it is read from `list_dataflows()`
277
+ when listed there, else accepted when `list_datasets()` can read it (the
278
+ handle is then named after the id). Any other string matches names
279
+ case-insensitively, exact first, then as a substring.
280
+ The dataflow's `bigData` flag is read once, here.
281
+ """
282
+ if isinstance(query, str):
283
+ text = query.strip()
284
+ if not text:
285
+ raise ValueError("The dataflow query is empty: give an id or part of a name")
286
+ if text.isdigit(): # an id read from an env var or a command line
287
+ query = int(text)
288
+ rows: list[dict[str, Any]] = []
289
+ try:
290
+ rows = easy.list_dataflows(client).to_dict("records")
291
+ except ReportnetAPIError:
292
+ if _as_id(query) is None:
293
+ raise # a name search needs the listing; an id can still be checked below
294
+ wanted = _as_id(query)
295
+ if wanted is not None:
296
+ query = wanted
297
+ found = [r for r in rows if _int_or_none(r.get("id")) == query]
298
+ if not found:
299
+ try:
300
+ easy.list_datasets(client, query)
301
+ except AuthenticationError:
302
+ raise # an expired or bad credential is not "no such dataflow"
303
+ except (LookupError, ReportnetAPIError) as err:
304
+ raise LookupError(
305
+ f"Dataflow {query} is not listed and could not be read: {err}"
306
+ ) from err
307
+ found = [{"id": query, "name": str(query), "status": None}]
308
+ else:
309
+ k = str(query).strip().lower()
310
+ found = [r for r in rows if str(r.get("name")).lower() == k]
311
+ if not found:
312
+ found = [r for r in rows if k in str(r.get("name")).lower()]
313
+ if len(found) != 1:
314
+ listing = "; ".join(f"{r.get('id')} {r.get('name')}" for r in (found or rows)[:20])
315
+ what = "Several dataflows match" if found else "No dataflow matches"
316
+ raise LookupError(f"{what} {query!r}: {listing}")
317
+ row = found[0]
318
+ flow_id = int(row["id"])
319
+ status = row.get("status")
320
+ return Dataflow(
321
+ id=flow_id,
322
+ name=str(row["name"]),
323
+ status=None if status is None or status != status else str(status),
324
+ big_data=easy.is_data_lake(client, flow_id),
325
+ client=client,
326
+ )