reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
reportnet_api/cli.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Command line for schedulers, shell scripts and FME's system caller.
|
|
2
|
+
|
|
3
|
+
reportnet check [--dataflow ID]
|
|
4
|
+
reportnet dataflows
|
|
5
|
+
reportnet datasets DATAFLOW_ID
|
|
6
|
+
reportnet tables DATASET_ID
|
|
7
|
+
reportnet read DATASET_ID DATAFLOW_ID [--table NAME] [--out FILE.csv]
|
|
8
|
+
reportnet export DATASET_ID DATAFLOW_ID OUT.zip [--format csv|parquet]
|
|
9
|
+
reportnet find WORDS...
|
|
10
|
+
reportnet guide
|
|
11
|
+
|
|
12
|
+
The credential comes from `REPORTNET_API_KEY` (or `REPORTNET_TOKEN_SCHEME` and
|
|
13
|
+
`REPORTNET_BASE_URL`, see `easy.connect`), never from a command-line argument, so it
|
|
14
|
+
does not end up in shell history or process lists. `--demo` runs against built-in
|
|
15
|
+
fictional data instead. Every command here only reads or exports; there is deliberately
|
|
16
|
+
no command that writes or deletes data (use the Python API for that). Needs no pandas.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import csv
|
|
23
|
+
import os
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
from collections.abc import Sequence
|
|
27
|
+
from typing import Any, TextIO
|
|
28
|
+
|
|
29
|
+
import httpx
|
|
30
|
+
|
|
31
|
+
from . import easy
|
|
32
|
+
from ._exceptions import ReportnetAPIError
|
|
33
|
+
from .client import ReportnetClient
|
|
34
|
+
from .demo import demo_client
|
|
35
|
+
from .discover import find
|
|
36
|
+
from .jobs import JobError, ProgressPrinter
|
|
37
|
+
|
|
38
|
+
_DATASET_GROUPS = easy._DATASET_GROUPS
|
|
39
|
+
_UMASK = os.umask(0)
|
|
40
|
+
os.umask(_UMASK) # read the process umask without changing it
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _writer(out: TextIO) -> Any:
|
|
44
|
+
return csv.writer(out, lineterminator="\n")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _table(rows: list[list[Any]], header: list[str], out: TextIO) -> None:
|
|
48
|
+
w = _writer(out)
|
|
49
|
+
w.writerow(header)
|
|
50
|
+
w.writerows(rows)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _dataflows(client: ReportnetClient, out: TextIO) -> None:
|
|
54
|
+
rows: list[list[Any]] = []
|
|
55
|
+
page = 0
|
|
56
|
+
while True:
|
|
57
|
+
result = client.dataflow.dataflows("reporting", page_size=200, page_num=page)
|
|
58
|
+
batch = result.get("dataflows") or []
|
|
59
|
+
rows.extend([[d.get("id"), d.get("name"), d.get("status")] for d in batch])
|
|
60
|
+
total = result.get("filteredRecords") or result.get("totalRecords") or 0
|
|
61
|
+
page += 1
|
|
62
|
+
if not batch or len(rows) >= total:
|
|
63
|
+
break
|
|
64
|
+
_table(rows, ["id", "name", "status"], out)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _datasets(client: ReportnetClient, dataflow_id: int, out: TextIO) -> None:
|
|
68
|
+
flow = client.dataflow.dataflow_datasets(dataflow_id)
|
|
69
|
+
if not flow:
|
|
70
|
+
raise LookupError(f"Empty answer for dataflow {dataflow_id}: unknown id or no access")
|
|
71
|
+
rows = [
|
|
72
|
+
[ds.get("id"), ds.get("dataSetName"), kind, ds.get("dataProviderId") or ""]
|
|
73
|
+
for group, kind in _DATASET_GROUPS.items()
|
|
74
|
+
for ds in flow.get(group) or []
|
|
75
|
+
]
|
|
76
|
+
_table(rows, ["dataset_id", "name", "kind", "provider_id"], out)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _tables(client: ReportnetClient, dataset_id: int, out: TextIO) -> None:
|
|
80
|
+
schema = client.dataset_schema.find_data_schema_by_dataset_id(dataset_id=dataset_id)
|
|
81
|
+
rows = [[t.idTableSchema, t.nameTableSchema] for t in schema.tableSchemas or []]
|
|
82
|
+
_table(rows, ["table_id", "name"], out)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _read(client: ReportnetClient, args: argparse.Namespace, out: TextIO) -> None:
|
|
86
|
+
records = easy.read_records(client, args.dataset_id, args.dataflow_id, args.table)
|
|
87
|
+
columns: list[str] = []
|
|
88
|
+
for rec in records:
|
|
89
|
+
columns.extend(c for c in rec if c not in columns)
|
|
90
|
+
_table([[rec.get(c, "") for c in columns] for rec in records], columns, out)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
94
|
+
p = argparse.ArgumentParser(
|
|
95
|
+
prog="reportnet",
|
|
96
|
+
description="Read-only command line for the Reportnet 3 API "
|
|
97
|
+
"(credential from REPORTNET_API_KEY).",
|
|
98
|
+
)
|
|
99
|
+
p.add_argument("--demo", action="store_true", help="use built-in fictional data, no network")
|
|
100
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
101
|
+
c = sub.add_parser("check", help="read-only check of the credential")
|
|
102
|
+
c.add_argument("--dataflow", type=int, default=None)
|
|
103
|
+
sub.add_parser("dataflows", help="list dataflows (CSV)")
|
|
104
|
+
d = sub.add_parser("datasets", help="list a dataflow's datasets (CSV)")
|
|
105
|
+
d.add_argument("dataflow_id", type=int)
|
|
106
|
+
t = sub.add_parser("tables", help="list a dataset's tables (CSV)")
|
|
107
|
+
t.add_argument("dataset_id", type=int)
|
|
108
|
+
r = sub.add_parser("read", help="print or save one table as CSV")
|
|
109
|
+
r.add_argument("dataset_id", type=int)
|
|
110
|
+
r.add_argument("dataflow_id", type=int)
|
|
111
|
+
r.add_argument("--table", default=None, help="table name (needed if the dataset has several)")
|
|
112
|
+
r.add_argument("--out", default=None, help="write to this file instead of stdout")
|
|
113
|
+
e = sub.add_parser("export", help="export a whole dataset to a ZIP of CSV/Parquet files")
|
|
114
|
+
e.add_argument("dataset_id", type=int)
|
|
115
|
+
e.add_argument("dataflow_id", type=int)
|
|
116
|
+
e.add_argument("dest")
|
|
117
|
+
e.add_argument("--format", choices=["csv", "parquet"], default="csv")
|
|
118
|
+
f = sub.add_parser("find", help="search methods and helpers by keyword")
|
|
119
|
+
f.add_argument("words", nargs="+")
|
|
120
|
+
sub.add_parser("guide", help="print the cheat sheet")
|
|
121
|
+
return p
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def main(argv: Sequence[str] | None = None, out: TextIO | None = None) -> int:
|
|
125
|
+
args = build_parser().parse_args(argv)
|
|
126
|
+
out = out or sys.stdout
|
|
127
|
+
try:
|
|
128
|
+
if args.command == "find":
|
|
129
|
+
print(find(" ".join(args.words)), file=out)
|
|
130
|
+
return 0
|
|
131
|
+
if args.command == "guide":
|
|
132
|
+
print(easy.guide(show=False), file=out)
|
|
133
|
+
return 0
|
|
134
|
+
if args.demo:
|
|
135
|
+
client = demo_client(quiet=True)
|
|
136
|
+
else:
|
|
137
|
+
try:
|
|
138
|
+
client = easy.connect()
|
|
139
|
+
except ValueError as err:
|
|
140
|
+
print(
|
|
141
|
+
f"error: {err}\nSet REPORTNET_API_KEY (and REPORTNET_BASE_URL for another "
|
|
142
|
+
"host), or try --demo.",
|
|
143
|
+
file=sys.stderr,
|
|
144
|
+
)
|
|
145
|
+
return 2
|
|
146
|
+
try:
|
|
147
|
+
if args.command == "check":
|
|
148
|
+
rows = easy.check_access(client, args.dataflow, show=False)
|
|
149
|
+
for r in rows:
|
|
150
|
+
print(f"{r['check']}: {r['result']} ({r['detail']})", file=out)
|
|
151
|
+
# Only the probed rows decide the exit code, not the static "expected" advice.
|
|
152
|
+
probed = [
|
|
153
|
+
r
|
|
154
|
+
for r in rows
|
|
155
|
+
if r["check"] == "credential alive" or r["check"].startswith("dataflow ")
|
|
156
|
+
]
|
|
157
|
+
if any(r["result"] not in {"yes", "reachable"} for r in probed):
|
|
158
|
+
return 1
|
|
159
|
+
elif args.command == "dataflows":
|
|
160
|
+
_dataflows(client, out)
|
|
161
|
+
elif args.command == "datasets":
|
|
162
|
+
_datasets(client, args.dataflow_id, out)
|
|
163
|
+
elif args.command == "tables":
|
|
164
|
+
_tables(client, args.dataset_id, out)
|
|
165
|
+
elif args.command == "read":
|
|
166
|
+
if args.out:
|
|
167
|
+
# Write next to the target and rename, so a failure leaves no empty file.
|
|
168
|
+
folder = os.path.dirname(os.path.abspath(args.out))
|
|
169
|
+
fd, tmp = tempfile.mkstemp(dir=folder, suffix=".part")
|
|
170
|
+
try:
|
|
171
|
+
with os.fdopen(fd, "w", newline="", encoding="utf-8") as fh:
|
|
172
|
+
_read(client, args, fh)
|
|
173
|
+
os.chmod(tmp, 0o666 & ~_UMASK) # mkstemp makes 0600; use the usual mode
|
|
174
|
+
os.replace(tmp, args.out)
|
|
175
|
+
except BaseException:
|
|
176
|
+
os.unlink(tmp)
|
|
177
|
+
raise
|
|
178
|
+
else:
|
|
179
|
+
_read(client, args, out)
|
|
180
|
+
elif args.command == "export":
|
|
181
|
+
printer = ProgressPrinter(sys.stderr)
|
|
182
|
+
try:
|
|
183
|
+
path = easy.export_to_file(
|
|
184
|
+
client,
|
|
185
|
+
args.dataset_id,
|
|
186
|
+
args.dataflow_id,
|
|
187
|
+
args.dest,
|
|
188
|
+
file_format=args.format,
|
|
189
|
+
progress=printer,
|
|
190
|
+
)
|
|
191
|
+
finally:
|
|
192
|
+
printer.close() # end the status line before the path or an error
|
|
193
|
+
print(path, file=out)
|
|
194
|
+
finally:
|
|
195
|
+
client.close()
|
|
196
|
+
except ReportnetAPIError as err:
|
|
197
|
+
print(f"{err}\n{err.explain()}", file=sys.stderr)
|
|
198
|
+
return 1
|
|
199
|
+
except (JobError, httpx.HTTPError) as err:
|
|
200
|
+
print(f"error: {err}", file=sys.stderr)
|
|
201
|
+
return 1
|
|
202
|
+
except OSError as err:
|
|
203
|
+
print(f"error: {err}", file=sys.stderr)
|
|
204
|
+
return 2
|
|
205
|
+
except (ValueError, LookupError, KeyError, RuntimeError) as err:
|
|
206
|
+
print(f"error: {err}", file=sys.stderr)
|
|
207
|
+
return 2
|
|
208
|
+
return 0
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
if __name__ == "__main__": # pragma: no cover
|
|
212
|
+
raise SystemExit(main())
|
reportnet_api/client.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import time
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from ._auth import TokenLike, TokenProvider
|
|
10
|
+
from ._core import (
|
|
11
|
+
ACCEPT_ANY,
|
|
12
|
+
DEFAULT_TIMEOUT,
|
|
13
|
+
RequestCore,
|
|
14
|
+
RetryPolicy,
|
|
15
|
+
ensure_same_origin,
|
|
16
|
+
request_is_replayable,
|
|
17
|
+
warn_if_insecure,
|
|
18
|
+
)
|
|
19
|
+
from .resources.dataflow_ext import DataflowResource
|
|
20
|
+
from .resources.dataset_schema_ext import DatasetSchemaResource
|
|
21
|
+
from .resources.dataset_snapshot import DatasetSnapshotResource
|
|
22
|
+
from .resources.datasets_ext import DatasetsResource
|
|
23
|
+
from .resources.eu_datasets import EuDatasetsResource
|
|
24
|
+
from .resources.reference_dataset_ext import ReferenceDatasetResource
|
|
25
|
+
from .resources.release_receipts import ReleaseReceiptsResource
|
|
26
|
+
from .resources.representative_ext import RepresentativeResource
|
|
27
|
+
from .resources.user_ext import UserResource
|
|
28
|
+
from .resources.validation_ext import ValidationResource
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ReportnetClient:
|
|
32
|
+
"""Sync client for the EEA Reportnet 3 REST API."""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
base_url: str,
|
|
37
|
+
token: TokenLike,
|
|
38
|
+
*,
|
|
39
|
+
token_scheme: str = "Bearer",
|
|
40
|
+
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
|
|
41
|
+
retry: RetryPolicy | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
self._retry = retry
|
|
44
|
+
self._core = RequestCore(TokenProvider(token, scheme=token_scheme))
|
|
45
|
+
self._http = httpx.Client(base_url=base_url, timeout=timeout)
|
|
46
|
+
warn_if_insecure(self._http.base_url)
|
|
47
|
+
|
|
48
|
+
self.dataflow = DataflowResource(self)
|
|
49
|
+
self.dataset_schema = DatasetSchemaResource(self)
|
|
50
|
+
self.datasets = DatasetsResource(self)
|
|
51
|
+
self.dataset_snapshot = DatasetSnapshotResource(self)
|
|
52
|
+
self.eu_datasets = EuDatasetsResource(self)
|
|
53
|
+
self.release_receipts = ReleaseReceiptsResource(self)
|
|
54
|
+
self.reference_dataset = ReferenceDatasetResource(self)
|
|
55
|
+
self.representative = RepresentativeResource(self)
|
|
56
|
+
self.user = UserResource(self)
|
|
57
|
+
self.validation = ValidationResource(self)
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
# Never include the token: only the base URL and the Authorization scheme.
|
|
61
|
+
return (
|
|
62
|
+
f"ReportnetClient(base_url={str(self._http.base_url)!r}, scheme={self._core.scheme!r})"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
|
66
|
+
ensure_same_origin(self._http.base_url, self._http.build_request(method, path).url)
|
|
67
|
+
headers = kwargs.pop("headers", None)
|
|
68
|
+
# A file handle or one-shot stream is consumed by the first attempt.
|
|
69
|
+
retry = self._retry if request_is_replayable(kwargs) else None
|
|
70
|
+
attempt = 0
|
|
71
|
+
while True:
|
|
72
|
+
kwargs["headers"] = self._core.build_headers(headers)
|
|
73
|
+
try:
|
|
74
|
+
response = self._http.request(method, path, **kwargs)
|
|
75
|
+
except httpx.TransportError as exc:
|
|
76
|
+
delay = retry.delay(attempt, method, error=exc) if retry else None
|
|
77
|
+
if delay is None:
|
|
78
|
+
raise
|
|
79
|
+
else:
|
|
80
|
+
failed = not response.is_success
|
|
81
|
+
delay = (
|
|
82
|
+
retry.delay(attempt, method, response=response) if retry and failed else None
|
|
83
|
+
)
|
|
84
|
+
if delay is None:
|
|
85
|
+
return self._core.handle_response(response)
|
|
86
|
+
time.sleep(delay)
|
|
87
|
+
attempt += 1
|
|
88
|
+
|
|
89
|
+
def download(
|
|
90
|
+
self, method: str, path: str, *, dest: str | os.PathLike[str], **kwargs: Any
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Stream a response body to `dest` without buffering it in memory."""
|
|
93
|
+
ensure_same_origin(self._http.base_url, self._http.build_request(method, path).url)
|
|
94
|
+
# Download endpoints declare `produces = octet-stream`; the JSON default
|
|
95
|
+
# would 406, so default to */* (callers can still override `headers=`).
|
|
96
|
+
kwargs["headers"] = self._core.build_headers(
|
|
97
|
+
{**ACCEPT_ANY, **(kwargs.get("headers") or {})}
|
|
98
|
+
)
|
|
99
|
+
with self._http.stream(method, path, **kwargs) as response:
|
|
100
|
+
if not response.is_success:
|
|
101
|
+
response.read()
|
|
102
|
+
self._core.handle_response(response)
|
|
103
|
+
with open(dest, "wb") as fh:
|
|
104
|
+
for chunk in response.iter_bytes():
|
|
105
|
+
fh.write(chunk)
|
|
106
|
+
|
|
107
|
+
def close(self) -> None:
|
|
108
|
+
self._http.close()
|
|
109
|
+
|
|
110
|
+
def __enter__(self) -> ReportnetClient:
|
|
111
|
+
return self
|
|
112
|
+
|
|
113
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
114
|
+
self.close()
|
reportnet_api/demo.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""A demo client that answers from built-in, fictional data (no network, no credential).
|
|
2
|
+
|
|
3
|
+
`demo_client()` returns a normal `ReportnetClient` whose transport is replaced by a tiny
|
|
4
|
+
in-process fake of the few Reportnet routes the notebook helpers use, so training
|
|
5
|
+
sessions and first-time users can try `list_dataflows`, `list_datasets`, `list_tables`,
|
|
6
|
+
`list_providers`, `find_dataflow` and `read_table` immediately.
|
|
7
|
+
|
|
8
|
+
Everything it returns is invented ("DEMO DATA"); only these read routes exist, and
|
|
9
|
+
anything else answers 501 (not implemented) with a message saying so. It is not a model
|
|
10
|
+
of Reportnet's behaviour, only of the response shapes the helpers read.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
from .client import ReportnetClient
|
|
22
|
+
|
|
23
|
+
DEMO_BASE_URL = "https://demo.invalid"
|
|
24
|
+
DEMO_DATAFLOW_ID = 1
|
|
25
|
+
DEMO_DATASET_IDS = {"design": 101, "AT": 102, "BE": 103}
|
|
26
|
+
|
|
27
|
+
_PROVIDERS = [
|
|
28
|
+
{"id": 2, "code": "AT", "label": "Austria", "groupId": 1},
|
|
29
|
+
{"id": 3, "code": "BE", "label": "Belgium", "groupId": 1},
|
|
30
|
+
]
|
|
31
|
+
_FIELDS = [
|
|
32
|
+
("Id", "NUMBER_INTEGER"),
|
|
33
|
+
("Station", "TEXT"),
|
|
34
|
+
("Latitude", "NUMBER_DECIMAL"),
|
|
35
|
+
("Commissioned", "DATE"),
|
|
36
|
+
]
|
|
37
|
+
_ROWS = {
|
|
38
|
+
102: [
|
|
39
|
+
("1", "Vienna Centre", "48.2082", "2019-04-01"),
|
|
40
|
+
("2", "Graz South", "47.0707", "2021-09-15"),
|
|
41
|
+
("3", "Linz Harbour", "", "2022-01-20"),
|
|
42
|
+
],
|
|
43
|
+
103: [
|
|
44
|
+
("1", "Brussels Ring", "50.8503", "2018-06-30"),
|
|
45
|
+
("2", "Ghent Canal", "51.0543", ""),
|
|
46
|
+
],
|
|
47
|
+
}
|
|
48
|
+
_CODES = {102: "AT", 103: "BE"}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _json(data: Any, status: int = 200) -> httpx.Response:
|
|
52
|
+
return httpx.Response(status, json=data)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _handler(request: httpx.Request) -> httpx.Response:
|
|
56
|
+
path = request.url.path
|
|
57
|
+
method = request.method
|
|
58
|
+
|
|
59
|
+
if method == "POST" and path == "/dataflow/getDataflows":
|
|
60
|
+
flow = {
|
|
61
|
+
"id": DEMO_DATAFLOW_ID,
|
|
62
|
+
"name": "Demo dataflow: air quality stations (DEMO DATA)",
|
|
63
|
+
"status": "OPEN",
|
|
64
|
+
"description": "Invented data for trying the library without a credential.",
|
|
65
|
+
}
|
|
66
|
+
return _json({"dataflows": [flow], "totalRecords": 1, "filteredRecords": 1})
|
|
67
|
+
|
|
68
|
+
if method == "GET" and (m := re.fullmatch(r"/dataflow/v1/(\d+)", path)):
|
|
69
|
+
if int(m.group(1)) != DEMO_DATAFLOW_ID:
|
|
70
|
+
return _json({"message": "Demo mode only knows dataflow 1"}, 404)
|
|
71
|
+
return _json(
|
|
72
|
+
{
|
|
73
|
+
"id": DEMO_DATAFLOW_ID,
|
|
74
|
+
"bigData": False,
|
|
75
|
+
"designDatasets": [{"id": DEMO_DATASET_IDS["design"], "dataSetName": "Design"}],
|
|
76
|
+
"reportingDatasets": [
|
|
77
|
+
{"id": DEMO_DATASET_IDS["AT"], "dataSetName": "Austria", "dataProviderId": 2},
|
|
78
|
+
{"id": DEMO_DATASET_IDS["BE"], "dataSetName": "Belgium", "dataProviderId": 3},
|
|
79
|
+
],
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
if method == "GET" and path == "/representative/dataProvider/countryGroups":
|
|
84
|
+
return _json([{"id": 1, "label": "Member countries"}])
|
|
85
|
+
if method == "GET" and path == "/representative/dataProvider/1":
|
|
86
|
+
return _json(_PROVIDERS)
|
|
87
|
+
|
|
88
|
+
if method == "GET" and (m := re.fullmatch(r"/dataschema/v1/datasetId/(\d+)", path)):
|
|
89
|
+
if int(m.group(1)) not in DEMO_DATASET_IDS.values():
|
|
90
|
+
return _json({"message": "Demo mode only knows datasets 101, 102 and 103"}, 404)
|
|
91
|
+
return _json(
|
|
92
|
+
{
|
|
93
|
+
"idDataSetSchema": "demo-schema",
|
|
94
|
+
"tableSchemas": [
|
|
95
|
+
{
|
|
96
|
+
"idTableSchema": "demo-table-stations",
|
|
97
|
+
"nameTableSchema": "Stations",
|
|
98
|
+
"recordSchema": {
|
|
99
|
+
"idRecordSchema": "demo-record",
|
|
100
|
+
"fieldSchema": [
|
|
101
|
+
{"id": f"demo-field-{i}", "name": name, "type": ftype}
|
|
102
|
+
for i, (name, ftype) in enumerate(_FIELDS)
|
|
103
|
+
],
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
],
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
if method == "GET" and (m := re.fullmatch(r"/dataset/v1/(\d+)/etlExport", path)):
|
|
111
|
+
rows = _ROWS.get(int(m.group(1)), [])
|
|
112
|
+
offset = int(request.url.params.get("offset", 0))
|
|
113
|
+
limit = int(request.url.params.get("limit", 10000))
|
|
114
|
+
records = [
|
|
115
|
+
{
|
|
116
|
+
"id_record": f"demo-{m.group(1)}-{row[0]}",
|
|
117
|
+
"countryCode": _CODES[int(m.group(1))],
|
|
118
|
+
"fields": [
|
|
119
|
+
{"fieldName": name, "value": value}
|
|
120
|
+
for (name, _), value in zip(_FIELDS, row, strict=True)
|
|
121
|
+
],
|
|
122
|
+
}
|
|
123
|
+
for row in rows[offset : offset + limit]
|
|
124
|
+
]
|
|
125
|
+
table = {"tableName": "Stations", "totalRecords": len(rows), "records": records}
|
|
126
|
+
return httpx.Response(
|
|
127
|
+
200,
|
|
128
|
+
content=json.dumps({"tables": [table]}).encode(),
|
|
129
|
+
headers={"content-type": "application/json"},
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# 501 rather than 404, so the error does not read as "wrong id".
|
|
133
|
+
return _json(
|
|
134
|
+
{"message": f"Demo mode has no {method} {path}: it only fakes the read helpers"}, 501
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def demo_client(*, quiet: bool = False) -> ReportnetClient:
|
|
139
|
+
"""A `ReportnetClient` backed by built-in fictional data; see the module docstring.
|
|
140
|
+
|
|
141
|
+
Try ``easy.list_dataflows(c)``, ``easy.list_datasets(c, 1)``, ``easy.list_tables(c, 102)``
|
|
142
|
+
and ``easy.read_table(c, 102, 1)``. Prints a one-line notice unless `quiet=True`.
|
|
143
|
+
"""
|
|
144
|
+
if not quiet:
|
|
145
|
+
print("DEMO MODE: fictional data, no network, no credential (dataflow 1).")
|
|
146
|
+
client = ReportnetClient(DEMO_BASE_URL, "demo")
|
|
147
|
+
client._http = httpx.Client(base_url=DEMO_BASE_URL, transport=httpx.MockTransport(_handler))
|
|
148
|
+
return client
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Search the client's methods and helpers by keyword.
|
|
2
|
+
|
|
3
|
+
`find("export")` lists every resource method, `easy` helper and handle member whose
|
|
4
|
+
name or first docstring line contains each word of the text, as a small table you can
|
|
5
|
+
print or iterate. It reads the live classes, so it never goes stale; the per-function
|
|
6
|
+
details (arguments, required rights, live-tested caveats) are in `docs/reference/`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import inspect
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from types import FunctionType
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from . import easy, jobs
|
|
17
|
+
from .client import ReportnetClient
|
|
18
|
+
from .handles import Dataflow, Dataset
|
|
19
|
+
|
|
20
|
+
_RESOURCES = (
|
|
21
|
+
"dataflow",
|
|
22
|
+
"dataset_schema",
|
|
23
|
+
"datasets",
|
|
24
|
+
"dataset_snapshot",
|
|
25
|
+
"eu_datasets",
|
|
26
|
+
"release_receipts",
|
|
27
|
+
"reference_dataset",
|
|
28
|
+
"representative",
|
|
29
|
+
"user",
|
|
30
|
+
"validation",
|
|
31
|
+
)
|
|
32
|
+
_REFERENCE_PAGES = {"jobs": "jobs", "easy": "easy"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class Match:
|
|
37
|
+
"""One searchable function: how to call it and what it does."""
|
|
38
|
+
|
|
39
|
+
call: str # e.g. "client.datasets.import_file_data" or "easy.read_table"
|
|
40
|
+
summary: str # first docstring line, may be empty
|
|
41
|
+
page: str # docs/reference page name
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def doc_link(self) -> str:
|
|
45
|
+
return f"docs/reference/{self.page}.md#{self.call.rsplit('.', 1)[-1].lower()}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Matches(list[Match]):
|
|
49
|
+
"""A list of `Match` that prints as an aligned table."""
|
|
50
|
+
|
|
51
|
+
def __str__(self) -> str:
|
|
52
|
+
if not self:
|
|
53
|
+
return "No match. Try a shorter or different word, e.g. find('export')."
|
|
54
|
+
width = max(len(m.call) for m in self)
|
|
55
|
+
return "\n".join(f"{m.call.ljust(width)} {m.summary}".rstrip() for m in self)
|
|
56
|
+
|
|
57
|
+
__repr__ = __str__
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _first_line(obj: Any, name: str = "") -> str:
|
|
61
|
+
"""First docstring line; generated methods start with a roles table, so for those
|
|
62
|
+
(and for no docstring) the method name in plain words is the summary."""
|
|
63
|
+
line = (inspect.getdoc(obj) or "").split("\n", 1)[0].strip()
|
|
64
|
+
if not line or line.lower().startswith("allowed roles"):
|
|
65
|
+
return name.replace("_", " ")
|
|
66
|
+
return line
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _catalog() -> list[Match]:
|
|
70
|
+
out: list[Match] = []
|
|
71
|
+
client = ReportnetClient("https://example.invalid", "x")
|
|
72
|
+
try:
|
|
73
|
+
for attr in _RESOURCES:
|
|
74
|
+
res = getattr(client, attr)
|
|
75
|
+
for name, fn in inspect.getmembers(type(res), inspect.isfunction):
|
|
76
|
+
if not name.startswith("_"):
|
|
77
|
+
out.append(Match(f"client.{attr}.{name}", _first_line(fn, name), attr))
|
|
78
|
+
finally:
|
|
79
|
+
client.close()
|
|
80
|
+
for page, module in (("easy", easy), ("jobs", jobs)):
|
|
81
|
+
for name, fn in inspect.getmembers(module, inspect.isfunction):
|
|
82
|
+
if not name.startswith("_") and fn.__module__ == module.__name__:
|
|
83
|
+
out.append(Match(f"{page}.{name}", _first_line(fn, name), _REFERENCE_PAGES[page]))
|
|
84
|
+
for cls, var in ((Dataflow, "flow"), (Dataset, "ds")):
|
|
85
|
+
for name, member in vars(cls).items():
|
|
86
|
+
if name.startswith("_") or not isinstance(member, property | FunctionType):
|
|
87
|
+
continue
|
|
88
|
+
out.append(Match(f"{var}.{name}", _first_line(member, name), "handles"))
|
|
89
|
+
return sorted(out, key=lambda m: m.call)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def find(text: str) -> Matches:
|
|
93
|
+
"""Search method names and one-line summaries for every word in `text`.
|
|
94
|
+
|
|
95
|
+
Case-insensitive; `_` and spaces both separate words, so `find("import file")`
|
|
96
|
+
matches `import_file_data`. Returns a printable list of `Match`.
|
|
97
|
+
"""
|
|
98
|
+
words = text.lower().replace("_", " ").split()
|
|
99
|
+
hits = Matches()
|
|
100
|
+
if not words:
|
|
101
|
+
return hits
|
|
102
|
+
for m in _catalog():
|
|
103
|
+
hay = f"{m.call} {m.summary}".lower().replace("_", " ")
|
|
104
|
+
if all(w in hay for w in words):
|
|
105
|
+
hits.append(m)
|
|
106
|
+
return hits
|