reportnet-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reportnet_api/__init__.py +38 -0
- reportnet_api/_auth.py +30 -0
- reportnet_api/_core.py +222 -0
- reportnet_api/_exceptions.py +243 -0
- reportnet_api/async_client.py +113 -0
- reportnet_api/cli.py +212 -0
- reportnet_api/client.py +114 -0
- reportnet_api/demo.py +148 -0
- reportnet_api/discover.py +106 -0
- reportnet_api/easy.py +1359 -0
- reportnet_api/handles.py +326 -0
- reportnet_api/jobs.py +383 -0
- reportnet_api/models/__init__.py +0 -0
- reportnet_api/models/generated.py +555 -0
- reportnet_api/py.typed +0 -0
- reportnet_api/resources/__init__.py +0 -0
- reportnet_api/resources/dataflow_ext.py +239 -0
- reportnet_api/resources/dataset_schema.py +274 -0
- reportnet_api/resources/dataset_schema_ext.py +589 -0
- reportnet_api/resources/dataset_snapshot.py +173 -0
- reportnet_api/resources/datasets.py +1810 -0
- reportnet_api/resources/datasets_ext.py +1001 -0
- reportnet_api/resources/eu_datasets.py +44 -0
- reportnet_api/resources/reference_dataset_ext.py +75 -0
- reportnet_api/resources/release_receipts.py +122 -0
- reportnet_api/resources/representative_ext.py +366 -0
- reportnet_api/resources/user_ext.py +53 -0
- reportnet_api/resources/validation_ext.py +447 -0
- reportnet_api-0.1.0.dist-info/METADATA +307 -0
- reportnet_api-0.1.0.dist-info/RECORD +33 -0
- reportnet_api-0.1.0.dist-info/WHEEL +4 -0
- reportnet_api-0.1.0.dist-info/entry_points.txt +2 -0
- reportnet_api-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError
|
|
2
|
+
from importlib.metadata import version as _version
|
|
3
|
+
|
|
4
|
+
from ._core import RetryPolicy
|
|
5
|
+
from ._exceptions import (
|
|
6
|
+
AuthenticationError,
|
|
7
|
+
ForbiddenError,
|
|
8
|
+
LockedError,
|
|
9
|
+
NotFoundError,
|
|
10
|
+
ReportnetAPIError,
|
|
11
|
+
ServerError,
|
|
12
|
+
TooManyRequestsError,
|
|
13
|
+
ValidationError,
|
|
14
|
+
)
|
|
15
|
+
from .async_client import AsyncReportnetClient
|
|
16
|
+
from .client import ReportnetClient
|
|
17
|
+
from .discover import find
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
__version__ = _version("reportnet-api")
|
|
21
|
+
except PackageNotFoundError: # running from a source tree that was never installed
|
|
22
|
+
__version__ = "0+unknown"
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"AsyncReportnetClient",
|
|
26
|
+
"AuthenticationError",
|
|
27
|
+
"ForbiddenError",
|
|
28
|
+
"LockedError",
|
|
29
|
+
"NotFoundError",
|
|
30
|
+
"ReportnetAPIError",
|
|
31
|
+
"ReportnetClient",
|
|
32
|
+
"RetryPolicy",
|
|
33
|
+
"ServerError",
|
|
34
|
+
"TooManyRequestsError",
|
|
35
|
+
"ValidationError",
|
|
36
|
+
"__version__",
|
|
37
|
+
"find",
|
|
38
|
+
]
|
reportnet_api/_auth.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
|
|
5
|
+
TokenLike = str | Callable[[], str]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TokenProvider:
|
|
9
|
+
"""Wraps a static token/key string or a zero-arg callable that returns one.
|
|
10
|
+
|
|
11
|
+
`scheme` is the `Authorization` header scheme, e.g. `"Bearer"` for a JWT
|
|
12
|
+
or `"ApiKey"` for a Reportnet dataflow-scoped API key — the API uses
|
|
13
|
+
different schemes for different credential types.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, token: TokenLike, scheme: str = "Bearer") -> None:
|
|
17
|
+
self._token = token
|
|
18
|
+
self._scheme = scheme
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def scheme(self) -> str:
|
|
22
|
+
return self._scheme
|
|
23
|
+
|
|
24
|
+
def get_token(self) -> str:
|
|
25
|
+
if callable(self._token):
|
|
26
|
+
return self._token()
|
|
27
|
+
return self._token
|
|
28
|
+
|
|
29
|
+
def auth_header(self) -> dict[str, str]:
|
|
30
|
+
return {"Authorization": f"{self._scheme} {self.get_token()}"}
|
reportnet_api/_core.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ipaddress
|
|
4
|
+
import math
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from collections.abc import Iterator, Mapping
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import UTC, datetime
|
|
11
|
+
from email.utils import parsedate_to_datetime
|
|
12
|
+
from types import MappingProxyType
|
|
13
|
+
from typing import Any, BinaryIO
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from ._auth import TokenProvider
|
|
18
|
+
from ._exceptions import error_for_response
|
|
19
|
+
|
|
20
|
+
# httpx treats an explicit `timeout=None` as "no timeout at all", so the clients
|
|
21
|
+
# need a real default. `read` is per-chunk, not per-response, so streamed
|
|
22
|
+
# exports/downloads aren't cut off as long as data keeps arriving; it's also
|
|
23
|
+
# the wait for the first byte, which some server-side exports take a while for.
|
|
24
|
+
DEFAULT_TIMEOUT = httpx.Timeout(60.0, connect=10.0)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
_IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
|
28
|
+
# Transport errors worth repeating; permanent ones (UnsupportedProtocol,
|
|
29
|
+
# LocalProtocolError, ProxyError, ...) are deliberately not listed.
|
|
30
|
+
_TRANSIENT_ERRORS = (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)
|
|
31
|
+
# Raised before anything was sent, so repeating can't duplicate a write.
|
|
32
|
+
_NEVER_SENT_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout)
|
|
33
|
+
|
|
34
|
+
# Accept header for routes that produce octet-stream (downloads, file answers):
|
|
35
|
+
# the default `application/json` gets 406 there. Copy it (`dict(ACCEPT_ANY)`)
|
|
36
|
+
# before changing it.
|
|
37
|
+
ACCEPT_ANY: Mapping[str, str] = MappingProxyType({"Accept": "*/*"})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def caller_stacklevel() -> int:
|
|
41
|
+
"""The `stacklevel` that points a warning at the first frame outside this package."""
|
|
42
|
+
package_dir = os.path.dirname(os.path.abspath(__file__)) + os.sep
|
|
43
|
+
level, frame = 1, sys._getframe(1)
|
|
44
|
+
while frame is not None and os.path.abspath(frame.f_code.co_filename).startswith(package_dir):
|
|
45
|
+
level += 1
|
|
46
|
+
frame = frame.f_back # type: ignore[assignment]
|
|
47
|
+
return level
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _is_local(host: str) -> bool:
|
|
51
|
+
if host == "localhost" or host.endswith(".localhost"):
|
|
52
|
+
return True
|
|
53
|
+
try:
|
|
54
|
+
return ipaddress.ip_address(host).is_loopback
|
|
55
|
+
except ValueError:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def ensure_same_origin(base_url: httpx.URL, target: httpx.URL) -> None:
|
|
60
|
+
"""Refuse to send the credential to a host other than the configured one.
|
|
61
|
+
|
|
62
|
+
A `path` may come from a server answer (a job's `pollingUrl`), so an absolute or
|
|
63
|
+
protocol-relative URL on another origin must never receive the token.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def origin(url: httpx.URL) -> str:
|
|
67
|
+
host = f"[{url.host}]" if ":" in url.host else url.host
|
|
68
|
+
port = f":{url.port}" if url.port else ""
|
|
69
|
+
return f"{url.scheme}://{host}{port}"
|
|
70
|
+
|
|
71
|
+
if origin(target) != origin(base_url):
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"Refusing to send the credential to {origin(target)}: it is not the configured "
|
|
74
|
+
f"base URL's origin ({origin(base_url)})"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def warn_if_insecure(base_url: httpx.URL) -> None:
|
|
79
|
+
"""Warn when the credential would travel unencrypted (plain http to a non-local host)."""
|
|
80
|
+
if base_url.scheme == "http" and not _is_local(base_url.host):
|
|
81
|
+
import warnings
|
|
82
|
+
|
|
83
|
+
warnings.warn(
|
|
84
|
+
f"base_url {str(base_url)!r} uses plain http: the credential would be sent "
|
|
85
|
+
"unencrypted. Use https.",
|
|
86
|
+
UserWarning,
|
|
87
|
+
stacklevel=caller_stacklevel(),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def request_is_replayable(kwargs: dict[str, Any]) -> bool:
|
|
92
|
+
"""False if the request body can't be sent twice (file handle or one-shot stream)."""
|
|
93
|
+
if "files" in kwargs:
|
|
94
|
+
return False
|
|
95
|
+
content, data = kwargs.get("content"), kwargs.get("data")
|
|
96
|
+
return (content is None or isinstance(content, (bytes, str))) and (
|
|
97
|
+
data is None or isinstance(data, dict)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _retry_after_seconds(response: httpx.Response) -> float | None:
|
|
102
|
+
"""`Retry-After` as seconds (number or HTTP-date), or None if absent or unusable."""
|
|
103
|
+
value = response.headers.get("Retry-After")
|
|
104
|
+
if value is None:
|
|
105
|
+
return None
|
|
106
|
+
try:
|
|
107
|
+
seconds = float(value)
|
|
108
|
+
except ValueError:
|
|
109
|
+
try:
|
|
110
|
+
when = parsedate_to_datetime(value)
|
|
111
|
+
except (TypeError, ValueError):
|
|
112
|
+
return None
|
|
113
|
+
if when.tzinfo is None:
|
|
114
|
+
when = when.replace(tzinfo=UTC)
|
|
115
|
+
seconds = (when - datetime.now(UTC)).total_seconds()
|
|
116
|
+
return seconds if math.isfinite(seconds) else None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True)
|
|
120
|
+
class RetryPolicy:
|
|
121
|
+
"""Opt-in retries for transient failures; pass as `retry=` to a client.
|
|
122
|
+
|
|
123
|
+
Without a policy the clients never retry. What is retried, and why:
|
|
124
|
+
|
|
125
|
+
- Status codes in `statuses` (429/502/503/504 by default), for GET/HEAD/OPTIONS only.
|
|
126
|
+
- Connection failures and pool timeouts, where the request was never sent,
|
|
127
|
+
for any method.
|
|
128
|
+
- Other transient transport errors (timeouts, dropped connections), for
|
|
129
|
+
GET/HEAD/OPTIONS only: the server may already have acted on a write.
|
|
130
|
+
|
|
131
|
+
Never retried: 500 (every 500 seen live was deterministic, e.g. "Access is
|
|
132
|
+
denied"), 423 (wait for the other job instead, e.g. `wait_for_job`),
|
|
133
|
+
permanent errors such as a malformed `base_url`, and requests whose body
|
|
134
|
+
can't be replayed (file uploads, streamed bodies). Waits grow as
|
|
135
|
+
`backoff * 2**attempt` up to `max_delay`; a `Retry-After` (seconds or
|
|
136
|
+
HTTP-date) replaces that (also capped).
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
retries: int = 3
|
|
140
|
+
backoff: float = 0.5
|
|
141
|
+
max_delay: float = 30.0
|
|
142
|
+
statuses: frozenset[int] = frozenset({429, 502, 503, 504})
|
|
143
|
+
|
|
144
|
+
def __post_init__(self) -> None:
|
|
145
|
+
if self.retries < 0:
|
|
146
|
+
raise ValueError(f"retries must be >= 0, got {self.retries}")
|
|
147
|
+
for name in ("backoff", "max_delay"):
|
|
148
|
+
value = getattr(self, name)
|
|
149
|
+
if not math.isfinite(value) or value < 0:
|
|
150
|
+
raise ValueError(f"{name} must be a finite number >= 0, got {value}")
|
|
151
|
+
|
|
152
|
+
def delay(
|
|
153
|
+
self,
|
|
154
|
+
attempt: int,
|
|
155
|
+
method: str,
|
|
156
|
+
*,
|
|
157
|
+
response: httpx.Response | None = None,
|
|
158
|
+
error: httpx.TransportError | None = None,
|
|
159
|
+
) -> float | None:
|
|
160
|
+
"""Seconds to wait before retry number `attempt + 1`, or None to give up."""
|
|
161
|
+
if attempt >= self.retries:
|
|
162
|
+
return None
|
|
163
|
+
idempotent = method.upper() in _IDEMPOTENT_METHODS
|
|
164
|
+
if error is not None:
|
|
165
|
+
if not isinstance(error, _TRANSIENT_ERRORS):
|
|
166
|
+
return None
|
|
167
|
+
if not (isinstance(error, _NEVER_SENT_ERRORS) or idempotent):
|
|
168
|
+
return None
|
|
169
|
+
elif response is None or response.status_code not in self.statuses or not idempotent:
|
|
170
|
+
return None
|
|
171
|
+
wait: float = min(self.backoff * 2**attempt, self.max_delay)
|
|
172
|
+
if response is not None:
|
|
173
|
+
asked = _retry_after_seconds(response)
|
|
174
|
+
if asked is not None:
|
|
175
|
+
wait = min(max(asked, 0.0), self.max_delay)
|
|
176
|
+
return wait
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class RequestCore:
|
|
180
|
+
"""Request/response plumbing shared by ReportnetClient and AsyncReportnetClient.
|
|
181
|
+
|
|
182
|
+
Builds auth headers and maps HTTP error responses to `ReportnetAPIError`
|
|
183
|
+
subclasses, so this logic is never duplicated between the sync and async
|
|
184
|
+
transports.
|
|
185
|
+
"""
|
|
186
|
+
|
|
187
|
+
def __init__(self, auth: TokenProvider) -> None:
|
|
188
|
+
self._auth = auth
|
|
189
|
+
|
|
190
|
+
@property
|
|
191
|
+
def scheme(self) -> str:
|
|
192
|
+
return self._auth.scheme
|
|
193
|
+
|
|
194
|
+
def build_headers(self, headers: dict[str, str] | None = None) -> dict[str, str]:
|
|
195
|
+
merged = {"Accept": "application/json"}
|
|
196
|
+
merged.update(self._auth.auth_header())
|
|
197
|
+
for name, value in (headers or {}).items():
|
|
198
|
+
# HTTP header names are case-insensitive: drop any default that
|
|
199
|
+
# the caller's key would otherwise duplicate (e.g. "accept").
|
|
200
|
+
for existing in [k for k in merged if k.lower() == name.lower()]:
|
|
201
|
+
del merged[existing]
|
|
202
|
+
merged[name] = value
|
|
203
|
+
return merged
|
|
204
|
+
|
|
205
|
+
def handle_response(self, response: httpx.Response) -> httpx.Response:
|
|
206
|
+
error = error_for_response(response)
|
|
207
|
+
if error is not None:
|
|
208
|
+
raise error
|
|
209
|
+
return response
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@contextmanager
|
|
213
|
+
def stream_upload_file(
|
|
214
|
+
field_name: str, file: str | os.PathLike[str]
|
|
215
|
+
) -> Iterator[dict[str, tuple[str, BinaryIO]]]:
|
|
216
|
+
"""Open `file` for a path-based streaming multipart upload.
|
|
217
|
+
|
|
218
|
+
httpx reads from the file handle lazily while sending the request, so
|
|
219
|
+
the file is never fully buffered in memory.
|
|
220
|
+
"""
|
|
221
|
+
with open(file, "rb") as fh:
|
|
222
|
+
yield {field_name: (os.path.basename(str(file)), fh)}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ReportnetAPIError(Exception):
|
|
10
|
+
"""Base exception for all Reportnet API errors."""
|
|
11
|
+
|
|
12
|
+
def __init__(
|
|
13
|
+
self,
|
|
14
|
+
message: str,
|
|
15
|
+
*,
|
|
16
|
+
status_code: int,
|
|
17
|
+
response_body: str,
|
|
18
|
+
request: httpx.Request,
|
|
19
|
+
response: httpx.Response,
|
|
20
|
+
) -> None:
|
|
21
|
+
super().__init__(message)
|
|
22
|
+
self.status_code = status_code
|
|
23
|
+
self.response_body = response_body
|
|
24
|
+
self.request = request
|
|
25
|
+
self.response = response
|
|
26
|
+
|
|
27
|
+
def explain(self) -> str:
|
|
28
|
+
"""A few plain-language lines: what happened, the likely cause, what to try.
|
|
29
|
+
|
|
30
|
+
Built from the status, the Authorization scheme used and the server's
|
|
31
|
+
message, including patterns seen live. Heuristic: the cause is the most
|
|
32
|
+
likely one, not a certainty.
|
|
33
|
+
"""
|
|
34
|
+
return "\n".join(_explain(self))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ValidationError(ReportnetAPIError):
|
|
38
|
+
"""400 / 422 — request rejected as invalid."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class AuthenticationError(ReportnetAPIError):
|
|
42
|
+
"""401 — missing or invalid credentials."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ForbiddenError(ReportnetAPIError):
|
|
46
|
+
"""403 — authenticated but not authorized for this operation."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class NotFoundError(ReportnetAPIError):
|
|
50
|
+
"""404 — resource does not exist."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class LockedError(ReportnetAPIError):
|
|
54
|
+
"""423 — another operation holds a lock on the dataflow or dataset."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class TooManyRequestsError(ReportnetAPIError):
|
|
58
|
+
"""429 — too many requests."""
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def retry_after(self) -> float | None:
|
|
62
|
+
"""Seconds from a numeric `Retry-After` header, or None if absent or not a number."""
|
|
63
|
+
try:
|
|
64
|
+
return float(self.response.headers["Retry-After"])
|
|
65
|
+
except (KeyError, ValueError):
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ServerError(ReportnetAPIError):
|
|
70
|
+
"""5xx — server-side failure."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
_STATUS_EXCEPTIONS: dict[int, type[ReportnetAPIError]] = {
|
|
74
|
+
400: ValidationError,
|
|
75
|
+
401: AuthenticationError,
|
|
76
|
+
403: ForbiddenError,
|
|
77
|
+
404: NotFoundError,
|
|
78
|
+
422: ValidationError,
|
|
79
|
+
423: LockedError,
|
|
80
|
+
429: TooManyRequestsError,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_APIKEY_HINT = (
|
|
85
|
+
"ApiKey credentials are Reporter-level only and only work for the dataflow "
|
|
86
|
+
"they were issued for; Custodian/Steward-gated endpoints need a Bearer JWT "
|
|
87
|
+
"for a user holding that role"
|
|
88
|
+
)
|
|
89
|
+
_BEARER_FORBIDDEN_HINT = (
|
|
90
|
+
"check that this user holds the required role (e.g. Custodian/Steward) on "
|
|
91
|
+
"this dataflow; some endpoints (e.g. the ETL family) are gated separately "
|
|
92
|
+
"from roles and can 403 even when the role is correct"
|
|
93
|
+
)
|
|
94
|
+
_BEARER_401_HINT = "the Bearer token may have expired or be invalid; get a fresh one"
|
|
95
|
+
_APIKEY_401_HINT = "check that the ApiKey is correct and has not been revoked"
|
|
96
|
+
_LOCKED_HINT = (
|
|
97
|
+
"another operation (for example an import job still QUEUED or IN_PROGRESS for this "
|
|
98
|
+
"dataset, or a clone or snapshot job) holds a lock on this dataflow or dataset; "
|
|
99
|
+
"wait for it to finish and try again"
|
|
100
|
+
)
|
|
101
|
+
_TOO_MANY_HINT = (
|
|
102
|
+
"slow down and retry later (see `retry_after` on the exception); the Reportnet "
|
|
103
|
+
"backend itself sets no rate limit, so this likely comes from infrastructure in "
|
|
104
|
+
"front of the API"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _hint(response: httpx.Response) -> str | None:
|
|
109
|
+
"""Short tip for 401/403 (tailored to the Authorization scheme used), 423 and 429."""
|
|
110
|
+
if response.status_code == 423:
|
|
111
|
+
return _LOCKED_HINT
|
|
112
|
+
if response.status_code == 429:
|
|
113
|
+
return _TOO_MANY_HINT
|
|
114
|
+
if response.status_code not in (401, 403):
|
|
115
|
+
return None
|
|
116
|
+
scheme = response.request.headers.get("Authorization", "").split(" ", 1)[0].lower()
|
|
117
|
+
if response.status_code == 401:
|
|
118
|
+
return _APIKEY_401_HINT if scheme == "apikey" else _BEARER_401_HINT
|
|
119
|
+
return _APIKEY_HINT if scheme == "apikey" else _BEARER_FORBIDDEN_HINT
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
_REASON_MAX = 200
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _server_reason(response: httpx.Response) -> str | None:
|
|
126
|
+
"""Server's JSON `message` (truncated), else a short plain-text snippet.
|
|
127
|
+
|
|
128
|
+
Only the first JSON object is read: a dead ApiKey gets two objects glued
|
|
129
|
+
together. An HTML body (gateway error page) gives nothing; it stays on
|
|
130
|
+
`.response_body`.
|
|
131
|
+
"""
|
|
132
|
+
text = response.text.strip()
|
|
133
|
+
if not text:
|
|
134
|
+
return None
|
|
135
|
+
try:
|
|
136
|
+
body, _ = json.JSONDecoder().raw_decode(text)
|
|
137
|
+
except ValueError:
|
|
138
|
+
if "<" in text:
|
|
139
|
+
return None
|
|
140
|
+
reason = re.sub(r"\s+", " ", text)
|
|
141
|
+
else:
|
|
142
|
+
message = body.get("message") if isinstance(body, dict) else None
|
|
143
|
+
if not isinstance(message, str) or not message.strip():
|
|
144
|
+
return None
|
|
145
|
+
reason = re.sub(r"\s+", " ", message.strip())
|
|
146
|
+
if len(reason) > _REASON_MAX:
|
|
147
|
+
reason = reason[:_REASON_MAX] + "..."
|
|
148
|
+
return reason
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _is_rejected_credential_500(body: str) -> bool:
|
|
152
|
+
"""Live bodies of a rejected credential answered as HTTP 500 (2026-10-26).
|
|
153
|
+
|
|
154
|
+
`/dataflow/v1/{id}` says "401 UNAUTHORIZED"; the listing route says "Access
|
|
155
|
+
is denied". A Bearer token sent as `ApiKey` gets the same answer.
|
|
156
|
+
"""
|
|
157
|
+
return "401 UNAUTHORIZED" in body or "Access is denied" in body
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _explain(err: ReportnetAPIError) -> list[str]:
|
|
161
|
+
status = err.status_code
|
|
162
|
+
request = err.request
|
|
163
|
+
scheme = request.headers.get("Authorization", "").split(" ", 1)[0].lower()
|
|
164
|
+
path = request.url.path
|
|
165
|
+
reason = _server_reason(err.response)
|
|
166
|
+
lines = [f"{request.method} {path} failed with HTTP {status}."]
|
|
167
|
+
if reason:
|
|
168
|
+
lines.append(f"The server said: {reason}")
|
|
169
|
+
if status == 500 and _is_rejected_credential_500(err.response_body):
|
|
170
|
+
lines.append(
|
|
171
|
+
"Likely cause: the credential was rejected upstream (a revoked or expired ApiKey "
|
|
172
|
+
"shows up as a 500 like this, even for plain reads), or it was sent with the "
|
|
173
|
+
"wrong scheme."
|
|
174
|
+
)
|
|
175
|
+
lines.append(
|
|
176
|
+
"Try: regenerate the ApiKey in the dataflow's Settings, or get a fresh Bearer token. "
|
|
177
|
+
"A Bearer token sent as an ApiKey fails the same way: Bearer tokens are bare "
|
|
178
|
+
'UUIDs here, not JWTs, so connect(token, scheme="Bearer").'
|
|
179
|
+
)
|
|
180
|
+
elif status == 500 and "accessReferenceEntity" in err.response_body:
|
|
181
|
+
lines.append(
|
|
182
|
+
"Likely cause: the dataflow id does not exist, or is not visible to this "
|
|
183
|
+
"credential (the gateway reports this as a 500, not a 404)."
|
|
184
|
+
)
|
|
185
|
+
lines.append("Try: check the dataflow id (list_dataflows) and that it is not a dataset id.")
|
|
186
|
+
elif status == 423:
|
|
187
|
+
lines.append(
|
|
188
|
+
"Likely cause: another operation (often an import job still QUEUED or "
|
|
189
|
+
"IN_PROGRESS) holds a lock on this dataset or dataflow; the server's own text "
|
|
190
|
+
"can be vague."
|
|
191
|
+
)
|
|
192
|
+
lines.append("Try: wait for that job to finish (see wait_for_job), then repeat the call.")
|
|
193
|
+
elif status == 403 and scheme == "bearer" and "/etl" in path:
|
|
194
|
+
lines.append(
|
|
195
|
+
"Likely cause: the ETL endpoints are gated separately from dataflow roles; a "
|
|
196
|
+
"Bearer user can get 403 here even as Custodian/Steward."
|
|
197
|
+
)
|
|
198
|
+
lines.append("Try: use a dataflow ApiKey for ETL calls, or ask the dataflow admins.")
|
|
199
|
+
elif status == 403 and scheme == "apikey":
|
|
200
|
+
lines.append(
|
|
201
|
+
"Likely cause: ApiKeys are Reporter-level and work only for the dataflow they "
|
|
202
|
+
"were issued for, so Custodian/Steward-only routes refuse them."
|
|
203
|
+
)
|
|
204
|
+
lines.append("Try: use a Bearer token of a user holding that role on this dataflow.")
|
|
205
|
+
elif (hint := _hint(err.response)) is not None:
|
|
206
|
+
lines.append(f"Hint: {hint}.")
|
|
207
|
+
elif status >= 500:
|
|
208
|
+
lines.append("Likely cause: a server-side failure; the request itself may be fine.")
|
|
209
|
+
lines.append("Try: check the server's message above, then retry later.")
|
|
210
|
+
elif status == 404:
|
|
211
|
+
lines.append("Likely cause: the id in the request does not exist for this credential.")
|
|
212
|
+
lines.append("Try: check dataflow and dataset ids are not mixed up.")
|
|
213
|
+
elif status in (400, 422):
|
|
214
|
+
lines.append("Likely cause: a parameter was missing or invalid.")
|
|
215
|
+
lines.append("Try: compare the arguments with the method's docstring.")
|
|
216
|
+
return lines
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def error_for_response(response: httpx.Response) -> ReportnetAPIError | None:
|
|
220
|
+
"""Return the exception matching `response`'s status, or None if successful."""
|
|
221
|
+
if response.is_success:
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
exc_cls = _STATUS_EXCEPTIONS.get(response.status_code)
|
|
225
|
+
if exc_cls is None and response.status_code >= 500:
|
|
226
|
+
exc_cls = ServerError
|
|
227
|
+
if exc_cls is None:
|
|
228
|
+
exc_cls = ReportnetAPIError
|
|
229
|
+
|
|
230
|
+
message = f"{response.request.method} {response.request.url} -> {response.status_code}"
|
|
231
|
+
reason = _server_reason(response)
|
|
232
|
+
if reason:
|
|
233
|
+
message += f": {reason}"
|
|
234
|
+
hint = _hint(response)
|
|
235
|
+
if hint:
|
|
236
|
+
message += f" (hint: {hint})"
|
|
237
|
+
return exc_cls(
|
|
238
|
+
message,
|
|
239
|
+
status_code=response.status_code,
|
|
240
|
+
response_body=response.text,
|
|
241
|
+
request=response.request,
|
|
242
|
+
response=response,
|
|
243
|
+
)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import os
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from ._auth import TokenLike, TokenProvider
|
|
10
|
+
from ._core import (
|
|
11
|
+
ACCEPT_ANY,
|
|
12
|
+
DEFAULT_TIMEOUT,
|
|
13
|
+
RequestCore,
|
|
14
|
+
RetryPolicy,
|
|
15
|
+
ensure_same_origin,
|
|
16
|
+
request_is_replayable,
|
|
17
|
+
warn_if_insecure,
|
|
18
|
+
)
|
|
19
|
+
from .resources.dataflow_ext import AsyncDataflowResource
|
|
20
|
+
from .resources.dataset_schema_ext import AsyncDatasetSchemaResource
|
|
21
|
+
from .resources.dataset_snapshot import AsyncDatasetSnapshotResource
|
|
22
|
+
from .resources.datasets_ext import AsyncDatasetsResource
|
|
23
|
+
from .resources.eu_datasets import AsyncEuDatasetsResource
|
|
24
|
+
from .resources.reference_dataset_ext import AsyncReferenceDatasetResource
|
|
25
|
+
from .resources.release_receipts import AsyncReleaseReceiptsResource
|
|
26
|
+
from .resources.representative_ext import AsyncRepresentativeResource
|
|
27
|
+
from .resources.user_ext import AsyncUserResource
|
|
28
|
+
from .resources.validation_ext import AsyncValidationResource
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AsyncReportnetClient:
|
|
32
|
+
"""Async client for the EEA Reportnet 3 REST API."""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
base_url: str,
|
|
37
|
+
token: TokenLike,
|
|
38
|
+
*,
|
|
39
|
+
token_scheme: str = "Bearer",
|
|
40
|
+
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
|
|
41
|
+
retry: RetryPolicy | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
self._retry = retry
|
|
44
|
+
self._core = RequestCore(TokenProvider(token, scheme=token_scheme))
|
|
45
|
+
self._http = httpx.AsyncClient(base_url=base_url, timeout=timeout)
|
|
46
|
+
warn_if_insecure(self._http.base_url)
|
|
47
|
+
|
|
48
|
+
self.dataflow = AsyncDataflowResource(self)
|
|
49
|
+
self.dataset_schema = AsyncDatasetSchemaResource(self)
|
|
50
|
+
self.datasets = AsyncDatasetsResource(self)
|
|
51
|
+
self.dataset_snapshot = AsyncDatasetSnapshotResource(self)
|
|
52
|
+
self.eu_datasets = AsyncEuDatasetsResource(self)
|
|
53
|
+
self.release_receipts = AsyncReleaseReceiptsResource(self)
|
|
54
|
+
self.reference_dataset = AsyncReferenceDatasetResource(self)
|
|
55
|
+
self.representative = AsyncRepresentativeResource(self)
|
|
56
|
+
self.user = AsyncUserResource(self)
|
|
57
|
+
self.validation = AsyncValidationResource(self)
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
# Never include the token: only the base URL and the Authorization scheme.
|
|
61
|
+
url, scheme = str(self._http.base_url), self._core.scheme
|
|
62
|
+
return f"AsyncReportnetClient(base_url={url!r}, scheme={scheme!r})"
|
|
63
|
+
|
|
64
|
+
async def request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
|
65
|
+
ensure_same_origin(self._http.base_url, self._http.build_request(method, path).url)
|
|
66
|
+
headers = kwargs.pop("headers", None)
|
|
67
|
+
# A file handle or one-shot stream is consumed by the first attempt.
|
|
68
|
+
retry = self._retry if request_is_replayable(kwargs) else None
|
|
69
|
+
attempt = 0
|
|
70
|
+
while True:
|
|
71
|
+
kwargs["headers"] = self._core.build_headers(headers)
|
|
72
|
+
try:
|
|
73
|
+
response = await self._http.request(method, path, **kwargs)
|
|
74
|
+
except httpx.TransportError as exc:
|
|
75
|
+
delay = retry.delay(attempt, method, error=exc) if retry else None
|
|
76
|
+
if delay is None:
|
|
77
|
+
raise
|
|
78
|
+
else:
|
|
79
|
+
failed = not response.is_success
|
|
80
|
+
delay = (
|
|
81
|
+
retry.delay(attempt, method, response=response) if retry and failed else None
|
|
82
|
+
)
|
|
83
|
+
if delay is None:
|
|
84
|
+
return self._core.handle_response(response)
|
|
85
|
+
await asyncio.sleep(delay)
|
|
86
|
+
attempt += 1
|
|
87
|
+
|
|
88
|
+
async def download(
|
|
89
|
+
self, method: str, path: str, *, dest: str | os.PathLike[str], **kwargs: Any
|
|
90
|
+
) -> None:
|
|
91
|
+
"""Stream a response body to `dest` without buffering it in memory."""
|
|
92
|
+
ensure_same_origin(self._http.base_url, self._http.build_request(method, path).url)
|
|
93
|
+
# Download endpoints declare `produces = octet-stream`; the JSON default
|
|
94
|
+
# would 406, so default to */* (callers can still override `headers=`).
|
|
95
|
+
kwargs["headers"] = self._core.build_headers(
|
|
96
|
+
{**ACCEPT_ANY, **(kwargs.get("headers") or {})}
|
|
97
|
+
)
|
|
98
|
+
async with self._http.stream(method, path, **kwargs) as response:
|
|
99
|
+
if not response.is_success:
|
|
100
|
+
await response.aread()
|
|
101
|
+
self._core.handle_response(response)
|
|
102
|
+
with open(dest, "wb") as fh:
|
|
103
|
+
async for chunk in response.aiter_bytes():
|
|
104
|
+
fh.write(chunk)
|
|
105
|
+
|
|
106
|
+
async def aclose(self) -> None:
|
|
107
|
+
await self._http.aclose()
|
|
108
|
+
|
|
109
|
+
async def __aenter__(self) -> AsyncReportnetClient:
|
|
110
|
+
return self
|
|
111
|
+
|
|
112
|
+
async def __aexit__(self, *exc_info: object) -> None:
|
|
113
|
+
await self.aclose()
|