reportnet-api 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reportnet_api/easy.py ADDED
@@ -0,0 +1,1359 @@
1
+ """Hand-written convenience layer for notebooks and non-developers.
2
+
3
+ Thin wrappers over `ReportnetClient` that hide API plumbing: credentials from
4
+ the environment, flat listings, and tables as pandas DataFrames. The full
5
+ client stays available for anything not covered here. pandas is optional
6
+ (`pip install "reportnet-api[pandas]"`); it is only imported when a function
7
+ returning a DataFrame is called.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import getpass
13
+ import json
14
+ import numbers
15
+ import operator
16
+ import os
17
+ import re
18
+ import tempfile
19
+ import time
20
+ import warnings
21
+ from collections.abc import Callable, Iterable
22
+ from decimal import Decimal
23
+ from pathlib import Path
24
+ from typing import TYPE_CHECKING, Any, Literal
25
+
26
+ import httpx
27
+
28
+ from ._core import RetryPolicy, caller_stacklevel
29
+ from ._exceptions import ForbiddenError, ReportnetAPIError, _is_rejected_credential_500
30
+ from .client import ReportnetClient
31
+ from .demo import demo_client as _demo_client
32
+ from .jobs import JobError, Progress, progress_event, progress_scope
33
+ from .resources.dataflow_ext import DataflowKind
34
+
35
+ if TYPE_CHECKING:
36
+ import pandas as pd
37
+
38
+ from .handles import Dataflow, Dataset
39
+
40
+ DEFAULT_BASE_URL = "https://api.reportnet.europa.eu"
41
+
42
+ _DATASET_GROUPS = {
43
+ "designDatasets": "design",
44
+ "reportingDatasets": "reporting",
45
+ "dataCollections": "data collection",
46
+ "euDatasets": "eu",
47
+ "referenceDatasets": "reference",
48
+ "testDatasets": "test",
49
+ }
50
+
51
+
52
+ # Three non-empty base64url parts: header.payload.signature.
53
+ _JWT_SHAPE = re.compile(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")
54
+ _SCHEMES = {"apikey": "ApiKey", "bearer": "Bearer"}
55
+
56
+
57
+ def connect(
58
+ token: str | None = None,
59
+ *,
60
+ base_url: str | None = None,
61
+ scheme: str | None = None,
62
+ token_provider: Callable[[], str] | None = None,
63
+ retry: RetryPolicy | bool | None = None,
64
+ ) -> ReportnetClient:
65
+ """Build a client from arguments or environment variables.
66
+
67
+ `token` falls back to `REPORTNET_API_KEY`, `base_url` to `REPORTNET_BASE_URL`
68
+ and then production. The scheme is chosen from the value: a ``"Bearer "``
69
+ prefix or a JWT-shaped value (three dot-separated base64url parts) is sent
70
+ as Bearer; anything else is a dataflow API key (``ApiKey`` scheme). Live,
71
+ both an ApiKey and a Bearer token are bare UUIDs and look alike, so for a
72
+ Bearer token pass `scheme="Bearer"` (or the ``"Bearer "`` prefix); sent as
73
+ an ApiKey it fails with a 500 "Access is denied". Which one you have
74
+ matters: API keys carry Reporter-level access only. Set `REPORTNET_TOKEN_SCHEME`
75
+ (``ApiKey`` or ``Bearer``) to fix the scheme once, e.g. next to a Bearer token
76
+ in `REPORTNET_API_KEY`; precedence is `scheme=`, a ``"Bearer "`` prefix, that
77
+ variable, then the guess from the value. The chosen scheme shows in the
78
+ client's repr; the secret never does.
79
+
80
+ `token_provider` is a zero-arg callable returning the current token, called
81
+ before every request (instead of `token`; the scheme defaults to Bearer).
82
+ `retry=True` retries transient failures with a default `RetryPolicy`; pass
83
+ your own policy for other settings.
84
+
85
+ A Bearer token expires after a few minutes. If you have your own way to get fresh
86
+ tokens, pass it as `token_provider`; `client.user.refresh_token()` renews a login
87
+ when you hold the refresh token issued with it. This library does not log in.
88
+ """
89
+ if scheme is not None:
90
+ if scheme.lower() not in _SCHEMES:
91
+ raise ValueError(f"scheme must be 'ApiKey' or 'Bearer', got {scheme!r}.")
92
+ scheme = _SCHEMES[scheme.lower()]
93
+ env_scheme = os.environ.get("REPORTNET_TOKEN_SCHEME", "").strip()
94
+ if env_scheme and env_scheme.lower() not in _SCHEMES:
95
+ raise ValueError(
96
+ f"REPORTNET_TOKEN_SCHEME must be 'ApiKey' or 'Bearer', got {env_scheme!r}."
97
+ )
98
+ if token_provider is not None:
99
+ if token:
100
+ raise ValueError("Pass either token or token_provider, not both.")
101
+ credential: str | Callable[[], str] = token_provider
102
+ scheme = scheme or _SCHEMES.get(env_scheme.lower()) or "Bearer"
103
+ else:
104
+ token = (token or os.environ.get("REPORTNET_API_KEY") or "").strip()
105
+ if not token:
106
+ raise ValueError(
107
+ "No credential: pass token=... or set the REPORTNET_API_KEY environment variable."
108
+ )
109
+ guessed = _SCHEMES.get(env_scheme.lower()) or "ApiKey"
110
+ if token.lower().startswith("bearer "):
111
+ guessed = "Bearer"
112
+ token = token[len("bearer ") :].strip()
113
+ elif _JWT_SHAPE.fullmatch(token) and not env_scheme:
114
+ guessed = "Bearer"
115
+ credential = token
116
+ scheme = scheme or guessed
117
+ policy = RetryPolicy() if retry is True else (retry or None)
118
+ url = base_url or os.environ.get("REPORTNET_BASE_URL") or DEFAULT_BASE_URL
119
+ return ReportnetClient(base_url=url, token=credential, token_scheme=scheme, retry=policy)
120
+
121
+
122
+ def demo_client(*, quiet: bool = False) -> ReportnetClient:
123
+ """A client that answers from built-in fictional data: no network, no credential.
124
+
125
+ For training and first tries (`list_dataflows(c)`, `list_datasets(c, 1)`,
126
+ `read_table(c, 102, 1)`, `find_dataflow(c, "demo")`); see `reportnet_api.demo`.
127
+ Prints a one-line notice unless `quiet=True`.
128
+ """
129
+ return _demo_client(quiet=quiet)
130
+
131
+
132
+ def connect_interactive(
133
+ *,
134
+ base_url: str | None = None,
135
+ scheme: str | None = None,
136
+ retry: RetryPolicy | bool | None = None,
137
+ prompt: str = "Reportnet API key or token (input hidden): ",
138
+ ) -> ReportnetClient:
139
+ """Like `connect()`, but asks for the credential with a hidden prompt.
140
+
141
+ Use it in a notebook so the key is typed, not pasted into a cell that gets
142
+ saved with the notebook. Same `scheme`/`retry` options as `connect()`.
143
+ """
144
+ return connect(getpass.getpass(prompt), base_url=base_url, scheme=scheme, retry=retry)
145
+
146
+
147
+ def _pandas() -> Any:
148
+ try:
149
+ import pandas
150
+ except ImportError:
151
+ raise ImportError(
152
+ "pandas is required for DataFrame output: install it with `pip install pandas` "
153
+ "(or use read_records() for plain dicts)"
154
+ ) from None
155
+ return pandas
156
+
157
+
158
+ def _require_id(value: object, name: str, hint: str) -> int:
159
+ if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
160
+ raise ValueError(f"{name} must be a positive integer id, got {value!r}. {hint}")
161
+ return value
162
+
163
+
164
+ def _hinted(err: ReportnetAPIError, hint: str) -> ReportnetAPIError:
165
+ """Same exception type and attributes, with a plain-language hint appended."""
166
+ return type(err)(
167
+ f"{err} - {hint}",
168
+ status_code=err.status_code,
169
+ response_body=err.response_body,
170
+ request=err.request,
171
+ response=err.response,
172
+ )
173
+
174
+
175
+ _DATAFLOW_HINT = "Pick an id from the list_dataflows() table."
176
+ _DATASET_HINT = "Use a dataset_id from list_datasets(), not the dataflow id."
177
+ _FORBIDDEN_HINT = "Also check that you did not mix up dataflow and dataset ids."
178
+
179
+
180
+ def list_dataflows(
181
+ client: ReportnetClient, kind: DataflowKind = "reporting", *, page_size: int = 200
182
+ ) -> pd.DataFrame:
183
+ """All dataflows visible to the credential, one row each (id, name, status, ...)."""
184
+ pd = _pandas()
185
+ rows: list[dict[str, Any]] = []
186
+ page = 0
187
+ while True:
188
+ result = client.dataflow.dataflows(kind, page_size=page_size, page_num=page)
189
+ batch = result.get("dataflows") or []
190
+ rows.extend(batch)
191
+ total = result.get("filteredRecords") or result.get("totalRecords") or 0
192
+ page += 1
193
+ if not batch or len(rows) >= total:
194
+ break
195
+ return pd.DataFrame(rows)
196
+
197
+
198
+ def find_dataflow(client: ReportnetClient, query: int | str) -> Dataflow:
199
+ """One dataflow handle by id or name text (see `handles.find_dataflow`).
200
+
201
+ The handle resolves datasets by id, country code or name:
202
+ `find_dataflow(c, "air quality").dataset("AT")`. Raises `LookupError` listing
203
+ the candidates when nothing or more than one matches.
204
+ """
205
+ from .handles import find_dataflow as _find
206
+
207
+ return _find(client, query)
208
+
209
+
210
+ def read_many(datasets: Iterable[Dataset], table: str, **kwargs: Any) -> pd.DataFrame:
211
+ """One table from several dataset handles, stacked (see `handles.read_many`).
212
+
213
+ Adds `dataset` and `country` columns; sequential; datasets without rows for the
214
+ table are skipped with a `RuntimeWarning`. Needs ETL access, like `read_tables()`.
215
+ """
216
+ from .handles import read_many as _read_many
217
+
218
+ return _read_many(datasets, table, **kwargs)
219
+
220
+
221
+ def list_datasets(
222
+ client: ReportnetClient, dataflow_id: int, *, provider_id: int | None = None
223
+ ) -> pd.DataFrame:
224
+ """Datasets of a dataflow as a flat table: dataset_id, name, kind, provider_id.
225
+
226
+ An unknown dataflow id answers HTTP 500 (`ServerError`, see `explain()`) and
227
+ no access answers 403 (live-tested). `LookupError` is a guard for an empty
228
+ 200 body, which the notes say the backend can send when it swallows an
229
+ error; not reproduced on this route.
230
+ """
231
+ pd = _pandas()
232
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
233
+ try:
234
+ flow = client.dataflow.dataflow_datasets(dataflow_id, provider_id=provider_id)
235
+ except ForbiddenError as err:
236
+ raise _hinted(err, _FORBIDDEN_HINT) from err
237
+ if not flow:
238
+ # The backend answers 200 with an empty body when it swallows an error.
239
+ raise LookupError(
240
+ f"Empty answer for dataflow {dataflow_id}: the id is unknown or this credential "
241
+ "has no access to it (the server does not say which). "
242
+ f"{_DATAFLOW_HINT} Reporters also need provider_id."
243
+ )
244
+ rows = []
245
+ for group, kind in _DATASET_GROUPS.items():
246
+ for ds in flow.get(group) or []:
247
+ rows.append(
248
+ {
249
+ "dataset_id": ds.get("id"),
250
+ "name": ds.get("dataSetName"),
251
+ "kind": kind,
252
+ "provider_id": ds.get("dataProviderId"),
253
+ }
254
+ )
255
+ return pd.DataFrame(rows, columns=["dataset_id", "name", "kind", "provider_id"])
256
+
257
+
258
+ def is_data_lake(client: ReportnetClient, dataflow_id: int) -> bool:
259
+ """True when the dataflow is a data-lake one (`bigData: true` in its dataset listing).
260
+
261
+ Data-lake dataflows keep validation results and table data in the lake, so the
262
+ old per-dataset routes answer an empty `200` there.
263
+ """
264
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
265
+ try:
266
+ flow = client.dataflow.dataflow_datasets(dataflow_id)
267
+ except ForbiddenError as err:
268
+ raise _hinted(err, _FORBIDDEN_HINT) from err
269
+ return bool((flow or {}).get("bigData"))
270
+
271
+
272
+ _ERROR_COLUMNS = [
273
+ "short_code",
274
+ "level",
275
+ "entity",
276
+ "table",
277
+ "field",
278
+ "message",
279
+ "records",
280
+ "rule_id",
281
+ ]
282
+
283
+
284
+ def validation_errors(
285
+ client: ReportnetClient,
286
+ dataset_id: int,
287
+ dataflow_id: int,
288
+ *,
289
+ level_errors: list[str] | None = None,
290
+ type_entities: list[str] | None = None,
291
+ page_size: int = 100,
292
+ data_lake: bool | None = None,
293
+ ) -> pd.DataFrame:
294
+ """Failed validations of a dataset, one row per failing rule (what the UI shows).
295
+
296
+ Columns: short_code, level, entity, table, field, message, records (number of
297
+ failing records), rule_id. Picks the data-lake route (`list_group_validations_dl`)
298
+ when the dataflow has `bigData: true`, else `list_group_validations`; the old
299
+ route answers an empty list on a lake dataset even when the UI shows failures.
300
+ Pass `data_lake=True`/`False` when the caller already knows (it skips the
301
+ `bigData` lookup, whose empty answer would read as not a lake); `None` detects it.
302
+ An empty frame means no failures, or a validation that has not run yet: this
303
+ does not start one. Needs a credential that may read validations (an ApiKey
304
+ got 403 on the sandbox).
305
+ """
306
+ pd = _pandas()
307
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
308
+ lake = is_data_lake(client, dataflow_id) if data_lake is None else data_lake
309
+ fetch = (
310
+ client.validation.list_group_validations_dl
311
+ if lake
312
+ else client.validation.list_group_validations
313
+ )
314
+ rows: list[dict[str, Any]] = []
315
+ page = 0
316
+ while True:
317
+ try:
318
+ result = fetch(
319
+ dataset_id,
320
+ page_num=page,
321
+ page_size=page_size,
322
+ level_errors=level_errors,
323
+ type_entities=type_entities,
324
+ )
325
+ except ForbiddenError as err:
326
+ raise _hinted(err, _FORBIDDEN_HINT) from err
327
+ batch = (result or {}).get("errors") or []
328
+ for e in batch:
329
+ rows.append(
330
+ {
331
+ "short_code": e.get("shortCode"),
332
+ "level": e.get("levelError"),
333
+ "entity": e.get("typeEntity"),
334
+ "table": e.get("nameTableSchema"),
335
+ "field": e.get("nameFieldSchema"),
336
+ "message": e.get("message"),
337
+ "records": e.get("numberOfRecords"),
338
+ "rule_id": e.get("idRule"),
339
+ }
340
+ )
341
+ total = (
342
+ (result or {}).get("totalFilteredRecords") or (result or {}).get("totalRecords") or 0
343
+ )
344
+ page += 1
345
+ if not batch or len(rows) >= total or page >= _MAX_PAGES:
346
+ break
347
+ return pd.DataFrame(rows, columns=_ERROR_COLUMNS)
348
+
349
+
350
+ def validation_summary(client: ReportnetClient, dataset_id: int, dataflow_id: int) -> pd.DataFrame:
351
+ """Failures counted per table and level: table, level, rules, records.
352
+
353
+ `rules` is the number of failing rules, `records` the sum of their failing
354
+ record counts (one record can fail several rules, so it is not a distinct
355
+ count). Built on `validation_errors()`, so it has the same lake handling and
356
+ does not start a validation. Empty frame: no failures, or none run yet.
357
+ """
358
+ pd = _pandas()
359
+ errors = validation_errors(client, dataset_id, dataflow_id)
360
+ cols = ["table", "level", "rules", "records"]
361
+ if errors.empty:
362
+ return pd.DataFrame(columns=cols)
363
+ grouped = (
364
+ errors.fillna({"table": "", "level": ""})
365
+ .groupby(["table", "level"], as_index=False)
366
+ .agg(rules=("short_code", "size"), records=("records", "sum"))
367
+ )
368
+ return grouped[cols]
369
+
370
+
371
+ def _latest_notification_ms(client: ReportnetClient, dataset_id: int, event_type: str) -> int:
372
+ """`insertDate` (ms) of the newest `event_type` user notification for the dataset, or 0.
373
+
374
+ Reads `GET /notification/findUserNotifications` (hidden route; the list the
375
+ Reportnet UI shows, newest first, for the calling user only).
376
+ """
377
+ answer = client.request(
378
+ "GET", "/notification/findUserNotifications", params={"pageNum": 0, "pageSize": 50}
379
+ ).json()
380
+ times = [
381
+ int(n.get("insertDate") or 0)
382
+ for n in (answer or {}).get("userNotifications") or []
383
+ if n.get("eventType") == event_type
384
+ and (n.get("content") or {}).get("datasetId") == dataset_id
385
+ ]
386
+ return max(times, default=0)
387
+
388
+
389
+ def validate_and_wait(
390
+ client: ReportnetClient,
391
+ dataset_id: int,
392
+ dataflow_id: int,
393
+ *,
394
+ released: bool = False,
395
+ timeout: float = 600.0,
396
+ poll_interval: float = 3.0,
397
+ data_lake: bool | None = None,
398
+ ) -> pd.DataFrame:
399
+ """Start a validation, wait until it finishes, return `validation_errors()`.
400
+
401
+ `data_lake` is passed to `validation_errors()` (None: detect from `bigData`).
402
+
403
+ WRITES: replaces the dataset's validation results and takes a dataset lock.
404
+ The completion signal is the user notification the backend creates when a run
405
+ ends (`VALIDATION_FINISHED_EVENT`, or `VALIDATION_RELEASE_FINISHED_EVENT` with
406
+ `released=True`), read from `/notification/findUserNotifications`: the dataset
407
+ has no status field to poll (live, the sandbox). A run took
408
+ about 35-55 s there. Only the calling user's notifications are visible, and a
409
+ failed run has no event of its own here, so it ends as a `JobError` timeout.
410
+ Notification times are server-side, so a newer one than any seen before the
411
+ start counts; a run started by someone else at the same moment could be
412
+ mistaken for this one. Raises `JobError` after `timeout` seconds.
413
+ """
414
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
415
+ event = "VALIDATION_RELEASE_FINISHED_EVENT" if released else "VALIDATION_FINISHED_EVENT"
416
+ baseline = _latest_notification_ms(client, dataset_id, event)
417
+ try:
418
+ client.validation.start_validation(dataset_id, released=released)
419
+ except ForbiddenError as err:
420
+ raise _hinted(err, _FORBIDDEN_HINT) from err
421
+ deadline = time.monotonic() + timeout
422
+ while _latest_notification_ms(client, dataset_id, event) <= baseline:
423
+ if time.monotonic() >= deadline:
424
+ raise JobError(
425
+ f"No {event} notification for dataset {dataset_id} within {timeout:g} "
426
+ "seconds (the validation may still be running, or it failed: a failed "
427
+ "run has no event of its own)"
428
+ )
429
+ time.sleep(poll_interval)
430
+ return validation_errors(client, dataset_id, dataflow_id, data_lake=data_lake)
431
+
432
+
433
+ def export_validations(
434
+ client: ReportnetClient,
435
+ dataset_id: int,
436
+ dest: str | os.PathLike[str],
437
+ *,
438
+ timeout: float = 300.0,
439
+ poll_interval: float = 2.0,
440
+ ) -> Path:
441
+ """Save a dataset's validation failures as a CSV at `dest` and return the path.
442
+
443
+ Generates the export and waits for it (`validation.export_validations_to_file`).
444
+ Columns: Entity, Table, Field, Code, QC Name, QC Description, Severity Level,
445
+ Message, Number of records. Works on data-lake datasets (live, the sandbox,
446
+ ). Close any open Reportnet page for the same dataset first: the
447
+ backend deletes the file when it is served, so the web app, which downloads it
448
+ on the notification, can take it before this does and the wait ends in a
449
+ `JobError` timeout. Leaves a notification in the user's Reportnet UI.
450
+ """
451
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
452
+ path = Path(dest)
453
+ try:
454
+ client.validation.export_validations_to_file(
455
+ dataset_id, path, timeout=timeout, poll_interval=poll_interval
456
+ )
457
+ except ForbiddenError as err:
458
+ raise _hinted(err, _FORBIDDEN_HINT) from err
459
+ return path
460
+
461
+
462
+ def list_providers(client: ReportnetClient, group_id: int = 1) -> pd.DataFrame:
463
+ """Data providers of a group as a table: provider_id, code, label, group_id.
464
+
465
+ A provider is a reporting country; `provider_id` is the `providerId` other
466
+ calls take and `code` its country code. Group `1` is the member
467
+ countries; `client.representative.data_provider_groups()` lists the groups.
468
+ """
469
+ pd = _pandas()
470
+ _require_id(group_id, "group_id", "Group 1 is the member countries.")
471
+ try:
472
+ providers = client.representative.data_providers(group_id) or []
473
+ except ForbiddenError as err:
474
+ raise _hinted(err, _FORBIDDEN_HINT) from err
475
+ rows = [
476
+ {
477
+ "provider_id": p.get("id"),
478
+ "code": p.get("code"),
479
+ "label": p.get("label"),
480
+ "group_id": p.get("groupId"),
481
+ }
482
+ for p in providers
483
+ ]
484
+ return pd.DataFrame(rows, columns=["provider_id", "code", "label", "group_id"])
485
+
486
+
487
+ def list_tables(client: ReportnetClient, dataset_id: int) -> pd.DataFrame:
488
+ """Tables defined in a dataset's schema: table_id and name."""
489
+ pd = _pandas()
490
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
491
+ try:
492
+ schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
493
+ except ForbiddenError as err:
494
+ raise _hinted(err, _FORBIDDEN_HINT) from err
495
+ rows = [
496
+ {"table_id": t.idTableSchema, "name": t.nameTableSchema} for t in schema.tableSchemas or []
497
+ ]
498
+ return pd.DataFrame(rows, columns=["table_id", "name"])
499
+
500
+
501
+ def _records_to_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
502
+ rows = []
503
+ for rec in records:
504
+ row: dict[str, Any] = {}
505
+ if rec.get("countryCode") is not None:
506
+ row["country_code"] = rec["countryCode"]
507
+ fields = rec.get("fields")
508
+ if isinstance(fields, list): # [{"fieldName": ..., "value": ...}, ...]
509
+ for f in fields:
510
+ row[f.get("fieldName")] = f.get("value")
511
+ elif isinstance(fields, dict):
512
+ row.update(fields)
513
+ else: # unknown shape: keep everything rather than silently dropping it
514
+ row.update({k: v for k, v in rec.items() if k != "countryCode"})
515
+ rows.append(row)
516
+ return rows
517
+
518
+
519
+ _MAX_PAGES = 10_000 # backstop against a backend that keeps returning full pages
520
+
521
+
522
+ def _schema_table_names(client: ReportnetClient, dataset_id: int) -> set[str] | None:
523
+ try:
524
+ return {
525
+ t.nameTableSchema
526
+ for t in client.dataset_schema.find_data_schema_by_dataset_id(
527
+ dataset_id=dataset_id
528
+ ).tableSchemas
529
+ or []
530
+ if t.nameTableSchema
531
+ }
532
+ except ReportnetAPIError:
533
+ return None # best-effort cross-check only
534
+
535
+
536
+ def _schema_field_types(client: ReportnetClient, dataset_id: int) -> dict[str, dict[str, str]]:
537
+ """``{table name: {field name: backend DataType}}`` from the dataset schema."""
538
+ schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
539
+ out: dict[str, dict[str, str]] = {}
540
+ for t in schema.tableSchemas or []:
541
+ fields = t.recordSchema.fieldSchema if t.recordSchema else None
542
+ out[t.nameTableSchema or ""] = {
543
+ f.name: str(getattr(f.type, "value", f.type)) for f in fields or [] if f.name
544
+ }
545
+ return out
546
+
547
+
548
+ _BOOL_TEXT = {"true": True, "false": False}
549
+
550
+
551
+ def _convert_column(series: pd.Series, field_type: str) -> pd.Series:
552
+ """Convert one text column by its schema type; raises ValueError if any value
553
+ does not fit. Empty text becomes NA."""
554
+ pd = _pandas()
555
+ text = series.astype("string").str.strip()
556
+ text = text.mask(text == "")
557
+ if field_type == "NUMBER_INTEGER":
558
+ num = pd.to_numeric(text, errors="raise")
559
+ if (num.dropna() % 1 != 0).any():
560
+ raise ValueError("non-integer value in an integer field")
561
+ return num.astype("Int64")
562
+ if field_type == "NUMBER_DECIMAL":
563
+ return pd.to_numeric(text, errors="raise").astype("Float64")
564
+ if field_type == "DATE":
565
+ return pd.to_datetime(text, format="%Y-%m-%d", errors="raise")
566
+ if field_type == "DATETIME":
567
+ return pd.to_datetime(text, format="ISO8601", errors="raise", utc=True)
568
+ if field_type == "BOOLEAN":
569
+ low = text.str.lower()
570
+ if not low.dropna().isin(_BOOL_TEXT).all():
571
+ raise ValueError("value is not true/false")
572
+ return low.map(_BOOL_TEXT).astype("boolean")
573
+ return series # text-like, geometry and codelist types stay as exported
574
+
575
+
576
+ def apply_schema_dtypes(
577
+ tables: dict[str, pd.DataFrame], field_types: dict[str, dict[str, str]]
578
+ ) -> dict[str, pd.DataFrame]:
579
+ """Convert columns to the types the dataset schema declares.
580
+
581
+ Handles NUMBER_INTEGER (`Int64`), NUMBER_DECIMAL (`Float64`), DATE, DATETIME (UTC)
582
+ and BOOLEAN (`true`/`false`, any case); every other type stays text. A column
583
+ whose values do not all fit keeps its original text and a `RuntimeWarning` names
584
+ it, so a wrong guess never silently changes data. Returns new frames.
585
+ """
586
+ out = {}
587
+ for name, df in tables.items():
588
+ df = df.copy()
589
+ types = field_types.get(name, {})
590
+ for col in df.columns:
591
+ ftype = types.get(col)
592
+ if ftype is None:
593
+ continue
594
+ try:
595
+ df[col] = _convert_column(df[col], ftype)
596
+ except (ValueError, TypeError) as err:
597
+ warnings.warn(
598
+ f"Table {name!r}, column {col!r} ({ftype}): left as text, {err}",
599
+ RuntimeWarning,
600
+ stacklevel=3,
601
+ )
602
+ out[name] = df
603
+ return out
604
+
605
+
606
+ def _collect_tables(
607
+ client: ReportnetClient,
608
+ dataset_id: int,
609
+ dataflow_id: int,
610
+ *,
611
+ table_schema_id: str | None,
612
+ provider_id: int | None,
613
+ page_size: int,
614
+ progress: Progress,
615
+ ) -> dict[str, list[dict[str, Any]]]:
616
+ """Page through the ETL export; ``{table name: rows}`` as plain dicts (no pandas)."""
617
+ try:
618
+ size = operator.index(page_size) # accepts numpy integers, rejects 2.5 and "10"
619
+ except TypeError:
620
+ size = 0
621
+ if isinstance(page_size, bool) or size < 1:
622
+ raise ValueError(f"page_size must be a positive integer, got {page_size!r}")
623
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
624
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
625
+ collected: dict[str, list[dict[str, Any]]] = {}
626
+ seen: dict[str, set[str]] = {}
627
+ totals: dict[str, int] = {}
628
+ duplicates: dict[str, int] = {}
629
+ page = 1 # counts requests, for progress and the page cap
630
+ offset = 0 # 0-based row offset: the backend's `offset` is not a page number
631
+ started = time.monotonic()
632
+ with progress_scope(progress) as report, tempfile.TemporaryDirectory() as tmp:
633
+ dest = Path(tmp) / "export.json"
634
+ while True:
635
+ try:
636
+ client.datasets.etl_export_dataset(
637
+ dataset_id,
638
+ dest,
639
+ limit=page_size,
640
+ offset=offset,
641
+ provider_id=provider_id,
642
+ table_schema_id=table_schema_id,
643
+ dataflow_id=dataflow_id,
644
+ )
645
+ except ForbiddenError as err:
646
+ raise _hinted(err, _FORBIDDEN_HINT) from err
647
+ payload = json.loads(dest.read_text(encoding="utf-8") or "{}")
648
+ more = False
649
+ for t in payload.get("tables") or []:
650
+ name = t.get("tableName") or "table"
651
+ rows = collected.setdefault(name, [])
652
+ ids = seen.setdefault(name, set())
653
+ recs = t.get("records") or []
654
+ fresh = []
655
+ for rec in recs:
656
+ rid = rec.get("id_record")
657
+ if rid is not None:
658
+ if rid in ids:
659
+ duplicates[name] = duplicates.get(name, 0) + 1
660
+ continue
661
+ ids.add(rid)
662
+ fresh.append(rec)
663
+ rows.extend(_records_to_rows(fresh))
664
+ if not fresh:
665
+ continue # nothing new on this page: do not ask for more
666
+ # The backend only sends `totalRecords` when a filter such as
667
+ # `tableSchemaId` is set (per its develop source; live it was
668
+ # present without one); without it, a full page is the only sign
669
+ # that more may follow.
670
+ if "totalRecords" in t:
671
+ totals[name] = int(t["totalRecords"] or 0)
672
+ more = more or len(rows) < totals[name]
673
+ else:
674
+ more = more or len(recs) >= page_size
675
+ if report:
676
+ rows_so_far = sum(len(r) for r in collected.values())
677
+ elapsed = time.monotonic() - started
678
+ report(progress_event("page", elapsed=elapsed, page=page, rows=rows_so_far))
679
+ if not more:
680
+ break
681
+ if page >= _MAX_PAGES:
682
+ warnings.warn(
683
+ f"read_tables stopped after {_MAX_PAGES} pages; the data is incomplete",
684
+ RuntimeWarning,
685
+ stacklevel=caller_stacklevel(),
686
+ )
687
+ break
688
+ page += 1
689
+ offset += page_size
690
+ for name, n in duplicates.items():
691
+ warnings.warn(
692
+ f"Table {name!r}: the backend returned {n} duplicate record(s) across pages; "
693
+ "they were dropped",
694
+ RuntimeWarning,
695
+ stacklevel=caller_stacklevel(),
696
+ )
697
+ for name, total in totals.items():
698
+ if len(collected[name]) != total:
699
+ warnings.warn(
700
+ f"Table {name!r}: got {len(collected[name])} rows but the backend reports "
701
+ f"totalRecords={total}; the result may be incomplete",
702
+ RuntimeWarning,
703
+ stacklevel=caller_stacklevel(),
704
+ )
705
+ if not any(collected.values()):
706
+ warnings.warn(
707
+ "The ETL export answered with no rows. If the dataset is not empty, it may be "
708
+ "in a data-lake dataflow (`bigData: true`), where this route answers 200 with "
709
+ "0 rows (live-tested); read it with `easy.export_to_file()` instead",
710
+ RuntimeWarning,
711
+ stacklevel=caller_stacklevel(),
712
+ )
713
+ if table_schema_id is None:
714
+ expected = _schema_table_names(client, dataset_id)
715
+ missing = sorted(expected - collected.keys()) if expected else []
716
+ if missing:
717
+ warnings.warn(
718
+ f"Tables {missing} are in the dataset schema but missing from the export",
719
+ RuntimeWarning,
720
+ stacklevel=caller_stacklevel(),
721
+ )
722
+ return collected
723
+
724
+
725
+ def read_tables(
726
+ client: ReportnetClient,
727
+ dataset_id: int,
728
+ dataflow_id: int,
729
+ *,
730
+ table_schema_id: str | None = None,
731
+ provider_id: int | None = None,
732
+ page_size: int = 10000,
733
+ dtypes: Literal["schema"] | None = None,
734
+ progress: Progress = None,
735
+ ) -> dict[str, pd.DataFrame]:
736
+ """Download a dataset's data as ``{table_name: DataFrame}``.
737
+
738
+ Pages through the synchronous ETL export until every row has been fetched.
739
+ Note: ETL access is credential-dependent (an API key works; some Bearer
740
+ users get 403) - see the README.
741
+
742
+ The backend's `offset` is a 0-based row offset (live-tested on production
743
+ and the sandbox), so pages are fetched at offsets 0,
744
+ `page_size`, 2 * `page_size`, ... Multi-table datasets are not verified. As a
745
+ guard, rows are de-duplicated by the backend's `id_record`, paging stops when
746
+ a page brings nothing new, and a `RuntimeWarning` is raised for duplicates, a
747
+ row count that differs from `totalRecords`, or tables of the schema missing
748
+ from the answer.
749
+
750
+ Data-lake caveat: for a dataflow with `bigData: true` this route answers `200`
751
+ with 0 rows even when data exists (live-tested), so a result
752
+ with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export)
753
+ for such a dataset.
754
+
755
+ Values arrive from the backend as text, empty cells as `''`. With
756
+ `dtypes="schema"` columns are converted using the field types in the dataset
757
+ schema (see `apply_schema_dtypes`: integers, decimals, dates, datetimes and
758
+ booleans; everything else stays text; a column that does not fit stays text with
759
+ a warning). Live-checked on the sandbox for integer, decimal, date and datetime
760
+ text; boolean text is unverified. The default (`None`) keeps all columns as text.
761
+
762
+ `progress=True` shows one updating line with the page number and rows so far;
763
+ a callable receives an event dict per page (`reportnet_api.jobs.progress_event`).
764
+ """
765
+ pd = _pandas()
766
+ if dtypes not in (None, "schema"):
767
+ raise ValueError('dtypes must be None or "schema"')
768
+ collected = _collect_tables(
769
+ client,
770
+ dataset_id,
771
+ dataflow_id,
772
+ table_schema_id=table_schema_id,
773
+ provider_id=provider_id,
774
+ page_size=page_size,
775
+ progress=progress,
776
+ )
777
+ frames = {name: pd.DataFrame(rows) for name, rows in collected.items()}
778
+ if dtypes == "schema":
779
+ try:
780
+ field_types = _schema_field_types(client, dataset_id)
781
+ except ReportnetAPIError as err:
782
+ warnings.warn(
783
+ f"dtypes='schema': could not read the dataset schema ({err}); columns left as text",
784
+ RuntimeWarning,
785
+ stacklevel=caller_stacklevel(),
786
+ )
787
+ return frames
788
+ frames = apply_schema_dtypes(frames, field_types)
789
+ return frames
790
+
791
+
792
+ def read_records(
793
+ client: ReportnetClient,
794
+ dataset_id: int,
795
+ dataflow_id: int,
796
+ table_name: str | None = None,
797
+ **kwargs: Any,
798
+ ) -> list[dict[str, Any]]:
799
+ """One table as a list of dicts, without pandas (for FME and plain scripts).
800
+
801
+ Same paging, de-duplication and warnings as `read_tables`; each row is a dict
802
+ ``{field name: text}`` (plus `country_code` when the backend sends one) and empty
803
+ cells are `''`. With no `table_name`, the dataset must contain exactly one table with
804
+ data. Keyword arguments (`table_schema_id`, `provider_id`, `page_size`, `progress`)
805
+ are those of `read_tables`.
806
+ """
807
+ tables = read_all_records(client, dataset_id, dataflow_id, **kwargs)
808
+ if table_name is not None:
809
+ try:
810
+ return tables[table_name]
811
+ except KeyError:
812
+ raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}") from None
813
+ if len(tables) != 1:
814
+ filled = {name: rows for name, rows in tables.items() if len(rows)}
815
+ if len(filled) == 1:
816
+ return next(iter(filled.values()))
817
+ if not filled:
818
+ raise ValueError(
819
+ f"All {len(tables)} tables are empty: {sorted(tables)}; "
820
+ "pass table_name=... to get one anyway"
821
+ )
822
+ raise ValueError(
823
+ f"Dataset has {len(filled)} tables with data {sorted(filled)}; pass table_name=..."
824
+ )
825
+ return next(iter(tables.values()))
826
+
827
+
828
+ def read_all_records(
829
+ client: ReportnetClient,
830
+ dataset_id: int,
831
+ dataflow_id: int,
832
+ *,
833
+ table_schema_id: str | None = None,
834
+ provider_id: int | None = None,
835
+ page_size: int = 10000,
836
+ progress: Progress = None,
837
+ ) -> dict[str, list[dict[str, Any]]]:
838
+ """Every table as ``{table name: list of row dicts}``, without pandas (see `read_records`)."""
839
+ return _collect_tables(
840
+ client,
841
+ dataset_id,
842
+ dataflow_id,
843
+ table_schema_id=table_schema_id,
844
+ provider_id=provider_id,
845
+ page_size=page_size,
846
+ progress=progress,
847
+ )
848
+
849
+
850
+ def read_table(
851
+ client: ReportnetClient,
852
+ dataset_id: int,
853
+ dataflow_id: int,
854
+ table_name: str | None = None,
855
+ **kwargs: Any,
856
+ ) -> pd.DataFrame:
857
+ """One table of a dataset as a DataFrame.
858
+
859
+ With no `table_name`, the dataset must contain exactly one table with data.
860
+ Extra keyword arguments go to `read_tables`, e.g. `dtypes="schema"` for typed columns.
861
+ """
862
+ tables = read_tables(client, dataset_id, dataflow_id, **kwargs)
863
+ if table_name is not None:
864
+ try:
865
+ return tables[table_name]
866
+ except KeyError:
867
+ raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}") from None
868
+ if len(tables) != 1:
869
+ filled = {name: rows for name, rows in tables.items() if len(rows)}
870
+ if len(filled) == 1:
871
+ return next(iter(filled.values()))
872
+ if not filled:
873
+ raise ValueError(
874
+ f"All {len(tables)} tables are empty: {sorted(tables)}; "
875
+ "pass table_name=... to get one anyway"
876
+ )
877
+ raise ValueError(
878
+ f"Dataset has {len(filled)} tables with data {sorted(filled)}; pass table_name=..."
879
+ )
880
+ return next(iter(tables.values()))
881
+
882
+
883
+ def export_to_file(
884
+ client: ReportnetClient,
885
+ dataset_id: int,
886
+ dataflow_id: int,
887
+ dest: str | os.PathLike[str],
888
+ *,
889
+ file_format: Literal["csv", "parquet"] = "csv",
890
+ table_schema_id: str | None = None,
891
+ timeout: float = 300.0,
892
+ progress: Progress = None,
893
+ ) -> Path:
894
+ """Export a dataset as a ZIP of CSV (or Parquet) files at `dest`.
895
+
896
+ Starts the async export job, waits for it to finish, then downloads the
897
+ ZIP. Raises `reportnet_api.jobs.JobError` if the job fails or `timeout`
898
+ seconds pass. Same ETL-access caveat as `read_tables()`. `progress=True` shows
899
+ one updating status line; a callable gets event dicts (see `wait_for_job()`).
900
+ """
901
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
902
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
903
+ if file_format not in ("csv", "parquet"):
904
+ raise ValueError(f"file_format must be 'csv' or 'parquet', got {file_format!r}")
905
+ export = (
906
+ client.datasets.export_zip_csv_to_file
907
+ if file_format == "csv"
908
+ else client.datasets.export_zip_parquet_to_file
909
+ )
910
+ try:
911
+ export(
912
+ dataset_id,
913
+ dataflow_id=dataflow_id,
914
+ dest=dest,
915
+ table_schema_id=table_schema_id,
916
+ timeout=timeout,
917
+ progress=progress,
918
+ )
919
+ except ForbiddenError as err:
920
+ raise _hinted(err, _FORBIDDEN_HINT) from err
921
+ return Path(dest)
922
+
923
+
924
+ def export_table(
925
+ client: ReportnetClient,
926
+ dataset_id: int,
927
+ table: str,
928
+ dest: str | os.PathLike[str],
929
+ *,
930
+ file_format: str = "csv",
931
+ big_data: bool = True,
932
+ timeout: float = 300.0,
933
+ progress: Progress = None,
934
+ ) -> Path:
935
+ """Export one table (by name) to `dest` as csv/xlsx/..., waiting for the file.
936
+
937
+ Unlike `export_to_file()` this is not a job: the backend generates the file
938
+ in the background and only notifies the Reportnet UI, so the helper polls the
939
+ download until the file appears. Raises `reportnet_api.jobs.JobError` after
940
+ `timeout` seconds. It leaves a notification in the user's Reportnet UI.
941
+ Needs a Bearer token (an ApiKey gets 403). `big_data=True` (default) uses the
942
+ routes the Reportnet UI calls; the plain ones never produced a file live.
943
+ Check `dest` afterwards: a leftover file with the same name on the server
944
+ could be served instead. `progress=True` shows a "waiting for the file" line
945
+ with the elapsed seconds; a callable gets event dicts.
946
+ """
947
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
948
+ if not file_format or " " in file_format or file_format.startswith("."):
949
+ raise ValueError(f"file_format must be a bare extension like 'csv', got {file_format!r}")
950
+ tables = list_tables(client, dataset_id)
951
+ match = tables[tables["name"] == table]
952
+ if match.empty:
953
+ raise ValueError(f"No table named {table!r}; the dataset has {sorted(tables['name'])}")
954
+ if len(match) > 1:
955
+ raise ValueError(f"The dataset has {len(match)} tables named {table!r}; can't pick one")
956
+ try:
957
+ client.datasets.export_table_to_file(
958
+ dataset_id,
959
+ table_name=table,
960
+ table_schema_id=str(match.iloc[0]["table_id"]),
961
+ dest=dest,
962
+ mime_type=file_format,
963
+ big_data=big_data,
964
+ timeout=timeout,
965
+ progress=progress,
966
+ )
967
+ except ForbiddenError as err:
968
+ raise _hinted(err, _FORBIDDEN_HINT) from err
969
+ return Path(dest)
970
+
971
+
972
+ def import_file(
973
+ client: ReportnetClient,
974
+ dataset_id: int,
975
+ dataflow_id: int,
976
+ path: str | os.PathLike[str],
977
+ *,
978
+ table_schema_id: str | None = None,
979
+ replace: bool = False,
980
+ confirm: bool = False,
981
+ check: bool = True,
982
+ provider_id: int | None = None,
983
+ delimiter: str | None = None,
984
+ timeout: float = 300.0,
985
+ progress: Progress = None,
986
+ ) -> dict[str, Any]:
987
+ """Upload a CSV (or other import file) into a dataset and wait until it is in.
988
+
989
+ Before uploading a `.csv`, the file is checked against the dataset schema
990
+ (`check_import_file`): the name must be a table name and the header must name
991
+ every field of that table, else `ValueError` lists the problems and nothing is
992
+ sent (`check=False` skips this; so does an unreadable schema).
993
+ `replace=True` deletes data, so it also needs `confirm=True`.
994
+
995
+ The file name must match a table name in the dataset (``TestTable.csv`` for
996
+ table ``TestTable``); otherwise Reportnet cancels the job ("File name does not
997
+ match any table name") and nothing is imported.
998
+
999
+ `delimiter` is the file's field separator; leave it None for the dataset's own
1000
+ setting. A multi-column file whose separator differs from the dataset's is
1001
+ cancelled ("headers ... do not match"): live on the sandbox the delimiter is `|`.
1002
+ `replace=True` emptied the dataset's *other* tables too (live, 2026-11-07,
1003
+ the sandbox: a one-table file left the second table empty), so re-import the tables
1004
+ you want to keep afterwards.
1005
+
1006
+ Uploads with the async big-file import, waits for its job, and then
1007
+ confirms through the dataset's import statistics. Returns those statistics
1008
+ (``{table_id: {"numberOfRecordsImported": ..., ...}}``). This WRITES to the
1009
+ dataset; `replace=True` also deletes the existing data first. Raises
1010
+ `reportnet_api.jobs.JobError` if the job fails or `timeout` seconds pass
1011
+ (the upload may still finish later). `progress=True` shows one updating status
1012
+ line; a callable gets event dicts. Ctrl-C while waiting only stops the waiting:
1013
+ the import keeps running on the server and cannot be cancelled from here.
1014
+ """
1015
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
1016
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
1017
+ if not Path(path).is_file():
1018
+ raise FileNotFoundError(f"No such file: {path}")
1019
+ if replace and not confirm:
1020
+ raise ValueError(
1021
+ "replace=True deletes the dataset's existing data first (live: also the other "
1022
+ "tables'); pass confirm=True to go ahead"
1023
+ )
1024
+ if check:
1025
+ problems = check_import_file(client, dataset_id, path, delimiter=delimiter)
1026
+ if problems:
1027
+ raise ValueError("The file would be rejected by Reportnet:\n- " + "\n- ".join(problems))
1028
+ try:
1029
+ return client.datasets.import_file_and_wait(
1030
+ dataset_id,
1031
+ path,
1032
+ dataflow_id=dataflow_id,
1033
+ provider_id=provider_id,
1034
+ table_schema_id=table_schema_id,
1035
+ replace=True if replace else None, # omit rather than send replace=false
1036
+ delimiter=delimiter,
1037
+ timeout=timeout,
1038
+ progress=progress,
1039
+ )
1040
+ except ForbiddenError as err:
1041
+ raise _hinted(err, _FORBIDDEN_HINT) from err
1042
+ except JobError as err:
1043
+ if "do not contain any records" in str(err) or "headers" in str(err):
1044
+ raise JobError(f"{err} {_HEADERS_HINT}") from err
1045
+ raise
1046
+
1047
+
1048
+ _HEADERS_HINT = (
1049
+ "(The web app reports this as wrong headers: the first line must name every field "
1050
+ "of the table, separated by the dataset's delimiter; see check_import_file().)"
1051
+ )
1052
+ _DELIMITERS = (",", "|", ";", "\t")
1053
+
1054
+
1055
+ def _schema_fields(client: ReportnetClient, dataset_id: int) -> dict[str, list[tuple[str, str]]]:
1056
+ """``{table name: [(field name, DataType), ...]}`` in schema order."""
1057
+ schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
1058
+ out: dict[str, list[tuple[str, str]]] = {}
1059
+ for t in schema.tableSchemas or []:
1060
+ fields = t.recordSchema.fieldSchema if t.recordSchema else None
1061
+ out[t.nameTableSchema or ""] = [
1062
+ (f.name, str(getattr(f.type, "value", f.type))) for f in fields or [] if f.name
1063
+ ]
1064
+ return out
1065
+
1066
+
1067
+ def check_import_file(
1068
+ client: ReportnetClient,
1069
+ dataset_id: int,
1070
+ path: str | os.PathLike[str],
1071
+ *,
1072
+ delimiter: str | None = None,
1073
+ ) -> list[str]:
1074
+ """Read-only pre-check of an import file; returns the problems found (empty = ok).
1075
+
1076
+ Only `.csv` files are checked (a ZIP or workbook is left to Reportnet). The name
1077
+ must match a table of the dataset (ignoring case: whether the backend is
1078
+ case-sensitive is unverified) and the first line must name exactly that table's
1079
+ fields, split by `delimiter` (or by any of `,` `|` `;` tab when None, since the
1080
+ dataset's own delimiter cannot be read). Live: a header with only some of the
1081
+ fields was cancelled (the sandbox). If the schema cannot be read, returns [].
1082
+ """
1083
+ path = Path(path)
1084
+ if path.suffix.lower() != ".csv":
1085
+ return []
1086
+ try:
1087
+ tables = _schema_fields(client, dataset_id)
1088
+ except ReportnetAPIError:
1089
+ return []
1090
+ by_lower = {name.lower(): name for name in tables}
1091
+ table = by_lower.get(path.stem.lower())
1092
+ if table is None:
1093
+ return [f"File name {path.name!r} matches no table; tables: {sorted(tables)}"]
1094
+ wanted = [name for name, _ in tables[table]]
1095
+ with path.open("rb") as fh: # only the first line matters; the rest may be any encoding
1096
+ raw = fh.readline()
1097
+ first = raw.decode("utf-8-sig", errors="replace").rstrip("\r\n")
1098
+ best: tuple[set[str], set[str]] | None = None
1099
+ for d in (delimiter,) if delimiter else _DELIMITERS:
1100
+ cols = [c.strip().strip('"').strip() for c in first.split(d)]
1101
+ missing, extra = set(wanted) - set(cols), set(cols) - set(wanted)
1102
+ if not missing and not extra:
1103
+ return []
1104
+ if best is None or len(missing) + len(extra) < len(best[0]) + len(best[1]):
1105
+ best = (missing, extra)
1106
+ assert best is not None
1107
+ missing, extra = best
1108
+ parts = []
1109
+ if missing:
1110
+ parts.append(f"missing {sorted(missing)}")
1111
+ if extra:
1112
+ parts.append(f"unknown {sorted(extra)}")
1113
+ where = f"delimiter {delimiter!r}" if delimiter else "any of , | ; tab"
1114
+ return [
1115
+ f"Header of {path.name!r} does not match table {table!r} ({where}): " + ", ".join(parts)
1116
+ ]
1117
+
1118
+
1119
+ def _format_cell(value: Any, field_type: str) -> str:
1120
+ """Text for one cell in the form the backend exports (see `read_tables`)."""
1121
+ pd = _pandas()
1122
+ if value is None or value is pd.NA:
1123
+ return ""
1124
+ is_real = isinstance(value, numbers.Real) and not isinstance(value, numbers.Integral | bool)
1125
+ if is_real and value != value: # NaN, also numpy's
1126
+ return ""
1127
+ if value is pd.NaT:
1128
+ return ""
1129
+ if field_type == "DATE" and hasattr(value, "strftime"):
1130
+ return str(value.strftime("%Y-%m-%d"))
1131
+ if field_type == "DATETIME" and hasattr(value, "strftime"):
1132
+ ts = pd.Timestamp(value)
1133
+ ts = ts.tz_convert("UTC") if ts.tzinfo else ts
1134
+ return str(ts.strftime("%Y-%m-%dT%H:%M:%SZ"))
1135
+ if field_type == "BOOLEAN" and isinstance(value, bool):
1136
+ return "true" if value else "false"
1137
+ if is_real: # python and numpy floats: plain digits, never 1e-07 or 1e+16
1138
+ if value in (float("inf"), float("-inf")):
1139
+ raise ValueError(f"{value} cannot be stored in a {field_type} field")
1140
+ text = format(Decimal(str(value)), "f") # str() is the shortest exact form
1141
+ if "." in text:
1142
+ text = text.rstrip("0").rstrip(".")
1143
+ return "0" if text in ("-0", "") else text
1144
+ return str(value)
1145
+
1146
+
1147
+ def import_dataframe(
1148
+ client: ReportnetClient,
1149
+ dataset_id: int,
1150
+ dataflow_id: int,
1151
+ table_name: str,
1152
+ df: pd.DataFrame,
1153
+ *,
1154
+ delimiter: str = ",",
1155
+ **kwargs: Any,
1156
+ ) -> dict[str, Any]:
1157
+ """Import a DataFrame into one table of a dataset (`import_file` underneath). WRITES.
1158
+
1159
+ Writes a temporary `<table_name>.csv` with a header for EVERY field of the table
1160
+ in schema order (fields missing from `df` are left empty; columns that are not
1161
+ fields raise `ValueError`), values formatted by field type (dates `YYYY-MM-DD`,
1162
+ datetimes UTC ISO, NA empty), and uploads it with `delimiter`. A value containing
1163
+ the delimiter or a line break raises `ValueError` (how the backend reads quotes is
1164
+ unverified). Other keyword arguments (`replace`, `confirm`, `provider_id`,
1165
+ `progress`, ...) go to `import_file`; `replace=True` needs `confirm=True`.
1166
+
1167
+ The delimiter is always sent with the upload, so it overrides the dataset's own
1168
+ setting. Live-tested once, on a sandbox dataset: a typed read, then
1169
+ `import_dataframe(..., delimiter="|", replace=True, confirm=True)` and a second read
1170
+ gave identical data. Other field types and datasets are not verified.
1171
+ """
1172
+ _pandas()
1173
+ _require_id(dataset_id, "dataset_id", _DATASET_HINT)
1174
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
1175
+ tables = _schema_fields(client, dataset_id)
1176
+ if table_name not in tables:
1177
+ raise KeyError(f"No table {table_name!r}; available: {sorted(tables)}")
1178
+ fields = tables[table_name]
1179
+ duplicated = sorted({str(c) for c in df.columns if list(df.columns).count(c) > 1})
1180
+ if duplicated:
1181
+ raise ValueError(f"Duplicate column names {duplicated}: each field can appear only once")
1182
+ unknown = [c for c in df.columns if c not in {n for n, _ in fields}]
1183
+ if unknown:
1184
+ raise ValueError(
1185
+ f"Columns {unknown} are not fields of {table_name!r}: {[n for n, _ in fields]}"
1186
+ )
1187
+ rows = {
1188
+ name: [_format_cell(v, ftype) for v in df[name]] if name in df.columns else [""] * len(df)
1189
+ for name, ftype in fields
1190
+ }
1191
+ for name, cells in rows.items():
1192
+ bad = next((c for c in cells if delimiter in c or "\n" in c or "\r" in c), None)
1193
+ if bad is not None:
1194
+ raise ValueError(
1195
+ f"Column {name!r} has a value with the delimiter or a line break: {bad!r}"
1196
+ )
1197
+ with tempfile.TemporaryDirectory() as tmp:
1198
+ file = Path(tmp) / f"{table_name}.csv"
1199
+ lines = [delimiter.join(rows)] + [
1200
+ delimiter.join(cells) for cells in zip(*rows.values(), strict=True)
1201
+ ]
1202
+ file.write_text("\n".join(lines) + "\n", encoding="utf-8")
1203
+ return import_file(client, dataset_id, dataflow_id, file, delimiter=delimiter, **kwargs)
1204
+
1205
+
1206
+ # What each credential class is *expected* to manage, per feature family, as
1207
+ # recorded in docs/live_testing_notes.md ("Credential requirement matrix",
1208
+ # sections) and the live-testing notes. Static text, not
1209
+ # probed: check_access() prints it labelled "expected". Update it with those notes.
1210
+ _EXPECTED_ACCESS: tuple[tuple[str, str, str], ...] = (
1211
+ # (feature family, ApiKey, Bearer)
1212
+ (
1213
+ "listing dataflows/datasets, schema reads",
1214
+ "worked live",
1215
+ "worked live (dataflow read, refreshed Bearer)",
1216
+ ),
1217
+ (
1218
+ "table read / sync ETL export (read_table)",
1219
+ "worked live",
1220
+ "403 live for a Custodian/Steward user (ETL is gated separately)",
1221
+ ),
1222
+ (
1223
+ "job exports (export_to_file, v4/v5)",
1224
+ "worked live",
1225
+ "403 live on v4 for a Custodian/Steward user",
1226
+ ),
1227
+ ("file import (import_file)", "worked live", "worked live"),
1228
+ (
1229
+ "schema ZIP export, list imported files",
1230
+ "403 live (needs Custodian/Steward)",
1231
+ "worked live for a Custodian/Steward user",
1232
+ ),
1233
+ (
1234
+ "historic releases",
1235
+ "403 live on a Design dataset",
1236
+ "403 live on a Design dataset (cause unconfirmed)",
1237
+ ),
1238
+ )
1239
+
1240
+
1241
+ def _probe_failure(err: ReportnetAPIError) -> tuple[str, str]:
1242
+ """(verdict, detail) for a failed probe; the detail is explain()'s second line."""
1243
+ lines = err.explain().splitlines()
1244
+ return err.__class__.__name__, lines[1] if len(lines) > 1 else lines[0]
1245
+
1246
+
1247
+ def check_access(
1248
+ client: ReportnetClient, dataflow_id: int | None = None, *, show: bool = True
1249
+ ) -> list[dict[str, str]]:
1250
+ """Read-only self-check of a credential: which kind, alive, dataflow reachable.
1251
+
1252
+ Makes one call (list dataflows, page size 1) and, with `dataflow_id`, one
1253
+ more (the dataflow's datasets). Nothing is written. Returns rows with
1254
+ ``check``, ``result`` and ``detail`` and prints them as a table unless
1255
+ `show=False`. The credential itself is never printed.
1256
+
1257
+ The ``expected`` rows are static advice for the credential's scheme taken
1258
+ from the live-testing notes; they were not probed by this call.
1259
+ """
1260
+ scheme = client._core.scheme
1261
+ rows: list[dict[str, str]] = [
1262
+ {
1263
+ "check": "credential",
1264
+ "result": scheme,
1265
+ "detail": f"sent as `Authorization: {scheme} ...`",
1266
+ }
1267
+ ]
1268
+ alive = False
1269
+ try:
1270
+ client.dataflow.dataflows("reporting", page_size=1, page_num=0)
1271
+ alive = True
1272
+ rows.append({"check": "credential alive", "result": "yes", "detail": "list dataflows ok"})
1273
+ except ReportnetAPIError as err:
1274
+ verdict, detail = _probe_failure(err)
1275
+ if err.status_code == 401 or (
1276
+ err.status_code == 500 and _is_rejected_credential_500(err.response_body)
1277
+ ):
1278
+ verdict = "no (rejected)"
1279
+ detail = "expired, revoked, or sent with the wrong scheme"
1280
+ if scheme == "ApiKey":
1281
+ detail += " (a Bearer token needs scheme='Bearer')"
1282
+ rows.append({"check": "credential alive", "result": verdict, "detail": detail})
1283
+ except httpx.TransportError as err:
1284
+ rows.append(
1285
+ {"check": "credential alive", "result": "unknown", "detail": f"network error: {err}"}
1286
+ )
1287
+ if dataflow_id is not None:
1288
+ _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
1289
+ check = f"dataflow {dataflow_id}"
1290
+ if not alive:
1291
+ rows.append({"check": check, "result": "skipped", "detail": "credential not usable"})
1292
+ else:
1293
+ try:
1294
+ flow = client.dataflow.dataflow_datasets(dataflow_id)
1295
+ except ReportnetAPIError as err:
1296
+ if err.status_code == 403:
1297
+ verdict, detail = "no access", "the server refuses this credential (403)"
1298
+ elif "accessReferenceEntity" in err.response_body:
1299
+ verdict, detail = "not found", "unknown id, or not visible (reported as 500)"
1300
+ else:
1301
+ verdict, detail = _probe_failure(err)
1302
+ rows.append({"check": check, "result": verdict, "detail": detail})
1303
+ except httpx.TransportError as err:
1304
+ rows.append(
1305
+ {"check": check, "result": "unknown", "detail": f"network error: {err}"}
1306
+ )
1307
+ else:
1308
+ count = sum(len((flow or {}).get(group) or []) for group in _DATASET_GROUPS)
1309
+ rows.append(
1310
+ {"check": check, "result": "reachable", "detail": f"{count} dataset(s) listed"}
1311
+ if flow
1312
+ else {
1313
+ "check": check,
1314
+ "result": "unclear",
1315
+ "detail": "empty 200 answer (unknown id or no access)",
1316
+ }
1317
+ )
1318
+ column = 2 if scheme == "Bearer" else 1 if scheme == "ApiKey" else None
1319
+ for entry in _EXPECTED_ACCESS:
1320
+ advice = entry[column] if column else "no advice for this scheme"
1321
+ rows.append(
1322
+ {"check": f"expected: {entry[0]}", "result": advice, "detail": "not probed now"}
1323
+ )
1324
+ if show:
1325
+ width = max(len(r["check"]) for r in rows)
1326
+ for r in rows:
1327
+ print(f"{r['check']:<{width}} {r['result']} ({r['detail']})")
1328
+ return rows
1329
+
1330
+
1331
+ _GUIDE = """\
1332
+ Reportnet cheat sheet (full reference: docs/reference/index.md; terms: docs/glossary.md)
1333
+
1334
+ 1. Connect client = connect_interactive() # hidden prompt; or connect()
1335
+ check_access(client) # read-only: credential kind, alive?
1336
+ 2. Find ids list_dataflows(client) # dataflow ids and names
1337
+ list_datasets(client, dataflow_id) # dataset ids (NOT the dataflow id)
1338
+ flow = find_dataflow(client, id_or_name); flow.dataset(id_or_country)
1339
+ 3. Read list_tables(client, dataset_id); read_table(client, dataset_id, dataflow_id)
1340
+ read_tables(...) for all tables; read_many(datasets, table) to stack
1341
+ 4. Validation validation_summary / validation_errors; validate_and_wait (WRITES)
1342
+ export_validations(client, dataset_id, dest)
1343
+ 5. Export export_table(...) one table to csv/xlsx; export_to_file(...) whole dataset
1344
+ 6. Upload import_file(client, dataset_id, dataflow_id, path) (WRITES)
1345
+ 7. Wrong? a ReportnetAPIError has .explain(); an ApiKey can only reach its own
1346
+ dataflow with Reporter rights, the ETL reads/exports need an ApiKey.
1347
+ 8. Search reportnet_api.find("export") # any method or helper by keyword
1348
+ """
1349
+
1350
+
1351
+ def guide(*, show: bool = True) -> str:
1352
+ """Print a one-page cheat sheet of the helpers in the order a notebook uses them.
1353
+
1354
+ Returns the text; prints it unless `show=False`. Search every method and helper
1355
+ by keyword with `reportnet_api.find(text)`.
1356
+ """
1357
+ if show:
1358
+ print(_GUIDE)
1359
+ return _GUIDE