internetdata 2.0.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {internetdata-2.0.0 → internetdata-2.2.0}/PKG-INFO +15 -3
- {internetdata-2.0.0 → internetdata-2.2.0}/README.md +14 -2
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/__init__.py +7 -1
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_core.py +156 -5
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/aio.py +21 -18
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/client.py +26 -20
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/models.py +32 -5
- {internetdata-2.0.0 → internetdata-2.2.0}/testdata/testdata.json +15 -1
- {internetdata-2.0.0 → internetdata-2.2.0}/tests/helpers.py +138 -1
- {internetdata-2.0.0 → internetdata-2.2.0}/tests/test_client.py +192 -1
- {internetdata-2.0.0 → internetdata-2.2.0}/tests/test_conformance.py +36 -14
- {internetdata-2.0.0 → internetdata-2.2.0}/tests/test_download.py +82 -6
- {internetdata-2.0.0 → internetdata-2.2.0}/.gitignore +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/LICENSE +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/pyproject.toml +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/scripts/download-spec.sh +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/scripts/generate.sh +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/scripts/publish.sh +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/scripts/v2_subset.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/spec/openapi.yaml +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/__init__.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/__init__.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/__init__.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/database_checksum_v2.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/database_metadata_v2.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/download_database_v2.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/list_databases.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/api/database_v_2/list_downloads.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/client.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/errors.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/__init__.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_checksum_v2_response_200.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_format.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_license_type_type_1.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_license_type_type_2_type_1.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_license_type_type_3_type_1.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata_column.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata_sample.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata_sample_additional_property_item.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata_schema.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_metadata_size.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/database_version.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/db_checksums.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/download.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/download_outcome.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/error.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/list_databases_response_200.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/list_downloads_response_200.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/models/standing.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/_generated/types.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/errors.py +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/src/internetdata/py.typed +0 -0
- {internetdata-2.0.0 → internetdata-2.2.0}/tests/conftest.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: internetdata
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Official Python client library for the InternetData API. Download and verify licensed IP datasets.
|
|
5
5
|
Project-URL: Homepage, https://internetdata.io
|
|
6
6
|
Project-URL: Documentation, https://docs.internetdata.io
|
|
@@ -72,7 +72,7 @@ with InternetData(api_key) as client:
|
|
|
72
72
|
|
|
73
73
|
### The catalog
|
|
74
74
|
|
|
75
|
-
`list` answers database FAMILIES. A
|
|
75
|
+
`list` answers database FAMILIES. A license is held against a family, while a download names a specific version, so the ids the other calls take come from each family's `versions`:
|
|
76
76
|
|
|
77
77
|
```python
|
|
78
78
|
for family in client.database.list():
|
|
@@ -82,7 +82,9 @@ for family in client.database.list():
|
|
|
82
82
|
print(version.id, version.formats) # 'bogon_ip_v1' ('csvgz', 'mmdb')
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
-
`standing` is `licensed`, `expired` or `unlicensed`, and `license_type` is what your
|
|
85
|
+
`standing` is `licensed`, `expired` or `unlicensed`, and `license_type` is what your license lets you do with the data (`evaluation`, `standard`, `redistribute`, or `None` when there is no license). A family you have never bought is still listed, as `unlicensed`, so you can see what else exists.
|
|
86
|
+
|
|
87
|
+
A rolling license also carries `renews_at`, when it next renews, and `notice_due_at`, the last day you can give notice of non-renewal for that term. Both are `None` when there is no license, when it has no defined term, or when `expires` sets a hard stop instead. `DATABASE_FORMATS`, `STANDINGS` and `LICENSE_TYPES` hold the published values at runtime, for checking one that came from a flag or a form before you make a call.
|
|
86
88
|
|
|
87
89
|
### What is inside a build
|
|
88
90
|
|
|
@@ -165,6 +167,16 @@ except InternetDataError as err:
|
|
|
165
167
|
|
|
166
168
|
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
167
169
|
|
|
170
|
+
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError` when the client is built:
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
client = InternetData(api_key, timeout=10)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
**Changed in 2.1.0:** the default was 10 seconds, and it bounded each read of a response rather than the whole attempt, so a response trickling in slowly could run past it for as long as the server kept sending.
|
|
177
|
+
|
|
178
|
+
**Changed in 2.2.0:** a timeout of 0 or less, NaN or a string used to be accepted, and failed every call.
|
|
179
|
+
|
|
168
180
|
Note that `rate_limited` and `quota_exceeded` both arrive as HTTP 429 and are not the same thing. A rate limit is the API facing a traffic burst, and retrying later works; a spent quota needs your allowance raised or the window to roll over. The library retries rate limits for you, and server and network failures, but never a spent quota or anything else you sent.
|
|
169
181
|
|
|
170
182
|
## Other Libraries
|
|
@@ -38,7 +38,7 @@ with InternetData(api_key) as client:
|
|
|
38
38
|
|
|
39
39
|
### The catalog
|
|
40
40
|
|
|
41
|
-
`list` answers database FAMILIES. A
|
|
41
|
+
`list` answers database FAMILIES. A license is held against a family, while a download names a specific version, so the ids the other calls take come from each family's `versions`:
|
|
42
42
|
|
|
43
43
|
```python
|
|
44
44
|
for family in client.database.list():
|
|
@@ -48,7 +48,9 @@ for family in client.database.list():
|
|
|
48
48
|
print(version.id, version.formats) # 'bogon_ip_v1' ('csvgz', 'mmdb')
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
-
`standing` is `licensed`, `expired` or `unlicensed`, and `license_type` is what your
|
|
51
|
+
`standing` is `licensed`, `expired` or `unlicensed`, and `license_type` is what your license lets you do with the data (`evaluation`, `standard`, `redistribute`, or `None` when there is no license). A family you have never bought is still listed, as `unlicensed`, so you can see what else exists.
|
|
52
|
+
|
|
53
|
+
A rolling license also carries `renews_at`, when it next renews, and `notice_due_at`, the last day you can give notice of non-renewal for that term. Both are `None` when there is no license, when it has no defined term, or when `expires` sets a hard stop instead. `DATABASE_FORMATS`, `STANDINGS` and `LICENSE_TYPES` hold the published values at runtime, for checking one that came from a flag or a form before you make a call.
|
|
52
54
|
|
|
53
55
|
### What is inside a build
|
|
54
56
|
|
|
@@ -131,6 +133,16 @@ except InternetDataError as err:
|
|
|
131
133
|
|
|
132
134
|
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
133
135
|
|
|
136
|
+
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError` when the client is built:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
client = InternetData(api_key, timeout=10)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
**Changed in 2.1.0:** the default was 10 seconds, and it bounded each read of a response rather than the whole attempt, so a response trickling in slowly could run past it for as long as the server kept sending.
|
|
143
|
+
|
|
144
|
+
**Changed in 2.2.0:** a timeout of 0 or less, NaN or a string used to be accepted, and failed every call.
|
|
145
|
+
|
|
134
146
|
Note that `rate_limited` and `quota_exceeded` both arrive as HTTP 429 and are not the same thing. A rate limit is the API facing a traffic burst, and retrying later works; a spent quota needs your allowance raised or the window to roll over. The library retries rate limits for you, and server and network failures, but never a spent quota or anything else you sent.
|
|
135
147
|
|
|
136
148
|
## Other Libraries
|
|
@@ -15,6 +15,9 @@ from .aio import AsyncDatabaseApi, AsyncInternetData
|
|
|
15
15
|
from .client import DatabaseApi, InternetData
|
|
16
16
|
from .errors import ErrorKind, InternetDataError
|
|
17
17
|
from .models import (
|
|
18
|
+
DATABASE_FORMATS,
|
|
19
|
+
LICENSE_TYPES,
|
|
20
|
+
STANDINGS,
|
|
18
21
|
Database,
|
|
19
22
|
DatabaseMetadata,
|
|
20
23
|
DatabaseVersion,
|
|
@@ -26,10 +29,13 @@ from .models import (
|
|
|
26
29
|
Standing,
|
|
27
30
|
)
|
|
28
31
|
|
|
29
|
-
__version__ = "2.
|
|
32
|
+
__version__ = "2.2.0"
|
|
30
33
|
|
|
31
34
|
__all__ = [
|
|
35
|
+
"DATABASE_FORMATS",
|
|
32
36
|
"DEFAULT_BASE_URL",
|
|
37
|
+
"LICENSE_TYPES",
|
|
38
|
+
"STANDINGS",
|
|
33
39
|
"AsyncDatabaseApi",
|
|
34
40
|
"AsyncInternetData",
|
|
35
41
|
"Database",
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
"""Plumbing the sync and the async client both need: transport wiring,
|
|
2
|
-
unwrapping, and the retry policy."""
|
|
1
|
+
"""Plumbing the sync and the async client both need: transport wiring, the deadline on
|
|
2
|
+
each attempt, response unwrapping, and the retry policy."""
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
+
import asyncio
|
|
6
7
|
import contextlib
|
|
8
|
+
import contextvars
|
|
7
9
|
import json
|
|
10
|
+
import math
|
|
8
11
|
import os
|
|
12
|
+
import threading
|
|
9
13
|
from collections.abc import Awaitable, Callable, Iterator
|
|
10
14
|
from pathlib import Path
|
|
15
|
+
from types import ModuleType
|
|
11
16
|
from typing import IO, Any, TypeVar, cast
|
|
12
17
|
|
|
13
18
|
import httpx
|
|
@@ -19,7 +24,7 @@ from .models import Database, Download, to_database, to_download
|
|
|
19
24
|
|
|
20
25
|
DEFAULT_BASE_URL = "https://internetdata.io"
|
|
21
26
|
DEFAULT_RETRIES = 2
|
|
22
|
-
DEFAULT_TIMEOUT =
|
|
27
|
+
DEFAULT_TIMEOUT = 30.0
|
|
23
28
|
DEFAULT_DOWNLOADS_LIMIT = 50
|
|
24
29
|
|
|
25
30
|
# One chunk of a transfer, and therefore the ceiling on what a download of any size
|
|
@@ -101,6 +106,30 @@ def build_async_transfer_client(
|
|
|
101
106
|
)
|
|
102
107
|
|
|
103
108
|
|
|
109
|
+
def check_timeout(timeout: float | None) -> float | None:
|
|
110
|
+
"""`timeout`, once it is a bound an attempt can meet.
|
|
111
|
+
|
|
112
|
+
Refused where it is SET, because nothing downstream refuses it: zero, a negative
|
|
113
|
+
number, NaN or a string reached the first call and failed it, and every call after,
|
|
114
|
+
as a retried `network` error after three seconds of backoff, or for a string as a
|
|
115
|
+
`server_error` blaming the API. Infinity is refused too, since the sync client's wait
|
|
116
|
+
cannot hold it (`OverflowError`); None is the spelling for no bound.
|
|
117
|
+
"""
|
|
118
|
+
if timeout is None:
|
|
119
|
+
return None
|
|
120
|
+
if (
|
|
121
|
+
isinstance(timeout, bool)
|
|
122
|
+
or not isinstance(timeout, int | float)
|
|
123
|
+
or not math.isfinite(timeout)
|
|
124
|
+
or timeout <= 0
|
|
125
|
+
):
|
|
126
|
+
raise ValueError(
|
|
127
|
+
f"timeout must be a number of seconds greater than 0, or None for no bound, "
|
|
128
|
+
f"not {timeout!r}"
|
|
129
|
+
)
|
|
130
|
+
return timeout
|
|
131
|
+
|
|
132
|
+
|
|
104
133
|
def storage_refusal(res: httpx.Response) -> InternetDataError:
|
|
105
134
|
"""What object storage refusing a download link becomes.
|
|
106
135
|
|
|
@@ -172,8 +201,10 @@ def send(call: Callable[[], Response[Any]]) -> Response[Any]:
|
|
|
172
201
|
something this client cannot read, which is a failed request rather than a bug in the
|
|
173
202
|
caller's code, so all three become the one error type here.
|
|
174
203
|
|
|
175
|
-
The
|
|
176
|
-
catch from swallowing a fault of our own.
|
|
204
|
+
The call does nothing but build one generated request, send it and hand the answer to
|
|
205
|
+
the generated decoder, which is what keeps this catch from swallowing a fault of our own.
|
|
206
|
+
An argument is converted before it, so a format this client does not publish is the
|
|
207
|
+
caller's `ValueError` rather than the server's failure.
|
|
177
208
|
"""
|
|
178
209
|
try:
|
|
179
210
|
return call()
|
|
@@ -189,6 +220,91 @@ async def send_async(call: Callable[[], Awaitable[Response[Any]]]) -> Response[A
|
|
|
189
220
|
raise malformed(exc) from exc
|
|
190
221
|
|
|
191
222
|
|
|
223
|
+
def request(
|
|
224
|
+
endpoint: ModuleType, client: AuthenticatedClient, bound: float | None, **params: Any
|
|
225
|
+
) -> Response[Any]:
|
|
226
|
+
"""`send` for one generated endpoint, one attempt of it finished within `bound` seconds,
|
|
227
|
+
or unbounded when that is None.
|
|
228
|
+
|
|
229
|
+
Assembled from the endpoint module's `_get_kwargs` and `_build_response` because its
|
|
230
|
+
`sync_detailed` sends through httpx alone, and httpx has no bound on the attempt.
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
def call() -> Response[Any]:
|
|
234
|
+
http = client.get_httpx_client()
|
|
235
|
+
req = http.build_request(**endpoint._get_kwargs(**params), timeout=httpx.Timeout(bound))
|
|
236
|
+
res = exchange(http, req, bound)
|
|
237
|
+
return cast(Response[Any], endpoint._build_response(client=client, response=res))
|
|
238
|
+
|
|
239
|
+
return send(call)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
async def request_async(
|
|
243
|
+
endpoint: ModuleType, client: AuthenticatedClient, bound: float | None, **params: Any
|
|
244
|
+
) -> Response[Any]:
|
|
245
|
+
"""`request`, awaited."""
|
|
246
|
+
|
|
247
|
+
async def call() -> Response[Any]:
|
|
248
|
+
http = client.get_async_httpx_client()
|
|
249
|
+
req = http.build_request(**endpoint._get_kwargs(**params), timeout=httpx.Timeout(bound))
|
|
250
|
+
res = await exchange_async(http, req, bound)
|
|
251
|
+
return cast(Response[Any], endpoint._build_response(client=client, response=res))
|
|
252
|
+
|
|
253
|
+
return await send_async(call)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def exchange(http: httpx.Client, req: httpx.Request, bound: float | None) -> httpx.Response:
|
|
257
|
+
"""One attempt at `req`, its whole body read, finished within `bound` seconds or failed
|
|
258
|
+
as a `network` error.
|
|
259
|
+
|
|
260
|
+
httpx bounds each PHASE of a request (connect, write, every read), not the attempt, so a
|
|
261
|
+
body trickling in a byte at a time outlasts any timeout it is given. The attempt runs on
|
|
262
|
+
a thread of its own and the caller waits at most `bound` for it; one abandoned stops at
|
|
263
|
+
its next chunk, or at the per-phase bound `req` also carries.
|
|
264
|
+
"""
|
|
265
|
+
if bound is None:
|
|
266
|
+
return http.send(req)
|
|
267
|
+
finished = threading.Event()
|
|
268
|
+
abandoned = threading.Event()
|
|
269
|
+
outcome: list[httpx.Response | BaseException] = []
|
|
270
|
+
context = contextvars.copy_context()
|
|
271
|
+
|
|
272
|
+
def attempt() -> None:
|
|
273
|
+
try:
|
|
274
|
+
outcome.append(context.run(_read_whole, http, req, abandoned))
|
|
275
|
+
except BaseException as exc: # noqa: BLE001 - raised again on the caller's thread
|
|
276
|
+
outcome.append(exc)
|
|
277
|
+
finally:
|
|
278
|
+
finished.set()
|
|
279
|
+
|
|
280
|
+
threading.Thread(target=attempt, name="internetdata-attempt", daemon=True).start()
|
|
281
|
+
try:
|
|
282
|
+
if not finished.wait(bound):
|
|
283
|
+
raise _deadline_passed(bound)
|
|
284
|
+
finally:
|
|
285
|
+
abandoned.set()
|
|
286
|
+
if isinstance(outcome[0], BaseException):
|
|
287
|
+
raise outcome[0]
|
|
288
|
+
return outcome[0]
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
async def exchange_async(
|
|
292
|
+
http: httpx.AsyncClient, req: httpx.Request, bound: float | None
|
|
293
|
+
) -> httpx.Response:
|
|
294
|
+
"""`exchange`, awaited. Cancelling the attempt closes its connection, so no thread is
|
|
295
|
+
needed to leave it behind."""
|
|
296
|
+
if bound is None:
|
|
297
|
+
return await http.send(req)
|
|
298
|
+
deadline = asyncio.timeout(bound)
|
|
299
|
+
try:
|
|
300
|
+
async with deadline:
|
|
301
|
+
return await http.send(req)
|
|
302
|
+
except TimeoutError:
|
|
303
|
+
if not deadline.expired():
|
|
304
|
+
raise
|
|
305
|
+
raise _deadline_passed(bound) from None
|
|
306
|
+
|
|
307
|
+
|
|
192
308
|
def malformed(exc: Exception) -> InternetDataError:
|
|
193
309
|
return InternetDataError("server_error", f"malformed response from the API: {exc}")
|
|
194
310
|
|
|
@@ -273,6 +389,41 @@ def retry_delay(err: InternetDataError, attempt: int, retries: int) -> float | N
|
|
|
273
389
|
return _BACKOFF_BASE * (2.0**attempt)
|
|
274
390
|
|
|
275
391
|
|
|
392
|
+
def _read_whole(
|
|
393
|
+
http: httpx.Client, req: httpx.Request, abandoned: threading.Event
|
|
394
|
+
) -> httpx.Response:
|
|
395
|
+
res = http.send(req, stream=True)
|
|
396
|
+
try:
|
|
397
|
+
if isinstance(res.stream, httpx.SyncByteStream):
|
|
398
|
+
res.stream = _Abandonable(res.stream, abandoned, req)
|
|
399
|
+
res.read()
|
|
400
|
+
finally:
|
|
401
|
+
res.close()
|
|
402
|
+
return res
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
class _Abandonable(httpx.SyncByteStream):
|
|
406
|
+
def __init__(
|
|
407
|
+
self, inner: httpx.SyncByteStream, abandoned: threading.Event, req: httpx.Request
|
|
408
|
+
) -> None:
|
|
409
|
+
self._inner = inner
|
|
410
|
+
self._abandoned = abandoned
|
|
411
|
+
self._req = req
|
|
412
|
+
|
|
413
|
+
def __iter__(self) -> Iterator[bytes]:
|
|
414
|
+
for chunk in self._inner:
|
|
415
|
+
if self._abandoned.is_set():
|
|
416
|
+
raise httpx.ReadError("abandoned once its deadline passed", request=self._req)
|
|
417
|
+
yield chunk
|
|
418
|
+
|
|
419
|
+
def close(self) -> None:
|
|
420
|
+
self._inner.close()
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _deadline_passed(bound: float) -> InternetDataError:
|
|
424
|
+
return InternetDataError("network", f"the request did not complete within {bound:g} seconds")
|
|
425
|
+
|
|
426
|
+
|
|
276
427
|
# The generated Response declares a plain MutableMapping, but always carries httpx's
|
|
277
428
|
# case-insensitive Headers. Rebuilding one keeps a header lookup case-blind whichever it
|
|
278
429
|
# turns out to be, which matters for `Retry-After`.
|
|
@@ -26,14 +26,15 @@ from ._core import (
|
|
|
26
26
|
assert_whole_transfer,
|
|
27
27
|
build_async_transfer_client,
|
|
28
28
|
build_client,
|
|
29
|
+
check_timeout,
|
|
29
30
|
checksums_of,
|
|
30
31
|
databases_of,
|
|
31
32
|
downloads_of,
|
|
32
33
|
parse_body,
|
|
33
34
|
part_file,
|
|
34
35
|
redirect_location,
|
|
36
|
+
request_async,
|
|
35
37
|
retry_delay,
|
|
36
|
-
send_async,
|
|
37
38
|
storage_refusal,
|
|
38
39
|
unwrap,
|
|
39
40
|
)
|
|
@@ -73,9 +74,11 @@ class AsyncInternetData:
|
|
|
73
74
|
timeout: float | None = DEFAULT_TIMEOUT,
|
|
74
75
|
transport: httpx.AsyncBaseTransport | None = None,
|
|
75
76
|
) -> None:
|
|
77
|
+
timeout = check_timeout(timeout)
|
|
76
78
|
self._client = build_client(api_key, base_url, timeout, transport)
|
|
77
79
|
self._transfer = build_async_transfer_client(timeout, transport)
|
|
78
80
|
self._retries = retries
|
|
81
|
+
self._timeout = timeout
|
|
79
82
|
self.database = AsyncDatabaseApi(self)
|
|
80
83
|
|
|
81
84
|
async def aclose(self) -> None:
|
|
@@ -128,7 +131,7 @@ class AsyncDatabaseApi:
|
|
|
128
131
|
"""
|
|
129
132
|
|
|
130
133
|
async def call() -> builtins.list[Database]:
|
|
131
|
-
res = await
|
|
134
|
+
res = await request_async(list_databases, self._client, self._owner._timeout)
|
|
132
135
|
return parse_body(unwrap(res), databases_of)
|
|
133
136
|
|
|
134
137
|
return await self._retrying(call)
|
|
@@ -137,8 +140,8 @@ class AsyncDatabaseApi:
|
|
|
137
140
|
"""What is inside one database: freshness, row count, columns, samples and sizes."""
|
|
138
141
|
|
|
139
142
|
async def call() -> DatabaseMetadata:
|
|
140
|
-
res = await
|
|
141
|
-
|
|
143
|
+
res = await request_async(
|
|
144
|
+
database_metadata_v2, self._client, self._owner._timeout, id=database_id
|
|
142
145
|
)
|
|
143
146
|
return parse_body(unwrap(res), to_metadata)
|
|
144
147
|
|
|
@@ -148,12 +151,12 @@ class AsyncDatabaseApi:
|
|
|
148
151
|
"""Every checksum published for one database file, keyed by algorithm."""
|
|
149
152
|
|
|
150
153
|
async def call() -> dict[str, str]:
|
|
151
|
-
res = await
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
)
|
|
154
|
+
res = await request_async(
|
|
155
|
+
database_checksum_v2,
|
|
156
|
+
self._client,
|
|
157
|
+
self._owner._timeout,
|
|
158
|
+
id=database_id,
|
|
159
|
+
format_=DatabaseFormat(format),
|
|
157
160
|
)
|
|
158
161
|
return parse_body(unwrap(res), checksums_of)
|
|
159
162
|
|
|
@@ -163,8 +166,8 @@ class AsyncDatabaseApi:
|
|
|
163
166
|
"""Your organization's recent download attempts, newest first."""
|
|
164
167
|
|
|
165
168
|
async def call() -> builtins.list[Download]:
|
|
166
|
-
res = await
|
|
167
|
-
|
|
169
|
+
res = await request_async(
|
|
170
|
+
list_downloads, self._client, self._owner._timeout, limit=limit
|
|
168
171
|
)
|
|
169
172
|
return parse_body(unwrap(res), downloads_of)
|
|
170
173
|
|
|
@@ -181,12 +184,12 @@ class AsyncDatabaseApi:
|
|
|
181
184
|
"""
|
|
182
185
|
|
|
183
186
|
async def call() -> str:
|
|
184
|
-
res = await
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
)
|
|
187
|
+
res = await request_async(
|
|
188
|
+
download_database_v2,
|
|
189
|
+
self._client,
|
|
190
|
+
self._owner._timeout,
|
|
191
|
+
id=database_id,
|
|
192
|
+
format_=DatabaseFormat(format),
|
|
190
193
|
)
|
|
191
194
|
return redirect_location(res)
|
|
192
195
|
|
|
@@ -21,14 +21,15 @@ from ._core import (
|
|
|
21
21
|
assert_whole_transfer,
|
|
22
22
|
build_client,
|
|
23
23
|
build_transfer_client,
|
|
24
|
+
check_timeout,
|
|
24
25
|
checksums_of,
|
|
25
26
|
databases_of,
|
|
26
27
|
downloads_of,
|
|
27
28
|
parse_body,
|
|
28
29
|
part_file,
|
|
29
30
|
redirect_location,
|
|
31
|
+
request,
|
|
30
32
|
retry_delay,
|
|
31
|
-
send,
|
|
32
33
|
storage_refusal,
|
|
33
34
|
unwrap,
|
|
34
35
|
)
|
|
@@ -55,9 +56,14 @@ class InternetData:
|
|
|
55
56
|
Every database published today is licensed, so create a key carrying the
|
|
56
57
|
`db.download` scope in the console and pass it in. The argument is optional
|
|
57
58
|
nonetheless, and an absent or empty one sends no `Authorization` header at all
|
|
58
|
-
rather than an empty one: what this API serves without a
|
|
59
|
+
rather than an empty one: what this API serves without a license is a product
|
|
59
60
|
decision, not the client's to refuse.
|
|
60
61
|
|
|
62
|
+
`timeout` is how long one attempt at a request may take, in seconds, body included, so a
|
|
63
|
+
call that is retried can take longer in total; None means no bound, and a database
|
|
64
|
+
transfer is exempt. Anything else that is not a finite number greater than 0 is a
|
|
65
|
+
`ValueError` here, rather than a failure of every call.
|
|
66
|
+
|
|
61
67
|
Holds an HTTP connection pool, so use it as a context manager or call `close()` when
|
|
62
68
|
you are done with it.
|
|
63
69
|
"""
|
|
@@ -74,9 +80,11 @@ class InternetData:
|
|
|
74
80
|
timeout: float | None = DEFAULT_TIMEOUT,
|
|
75
81
|
transport: httpx.BaseTransport | None = None,
|
|
76
82
|
) -> None:
|
|
83
|
+
timeout = check_timeout(timeout)
|
|
77
84
|
self._client = build_client(api_key, base_url, timeout, transport)
|
|
78
85
|
self._transfer = build_transfer_client(timeout, transport)
|
|
79
86
|
self._retries = retries
|
|
87
|
+
self._timeout = timeout
|
|
80
88
|
self.database = DatabaseApi(self)
|
|
81
89
|
|
|
82
90
|
def close(self) -> None:
|
|
@@ -128,12 +136,12 @@ class DatabaseApi:
|
|
|
128
136
|
customer. Nothing is cached and nothing is reconstructed here - what you get is
|
|
129
137
|
what the server sent for the key you are holding.
|
|
130
138
|
|
|
131
|
-
A
|
|
139
|
+
A license covers a FAMILY, while a download names a version, so the ids for the
|
|
132
140
|
other calls come from each entry's `versions`.
|
|
133
141
|
"""
|
|
134
142
|
|
|
135
143
|
def call() -> builtins.list[Database]:
|
|
136
|
-
res =
|
|
144
|
+
res = request(list_databases, self._client, self._owner._timeout)
|
|
137
145
|
return parse_body(unwrap(res), databases_of)
|
|
138
146
|
|
|
139
147
|
return self._retrying(call)
|
|
@@ -146,9 +154,7 @@ class DatabaseApi:
|
|
|
146
154
|
"""
|
|
147
155
|
|
|
148
156
|
def call() -> DatabaseMetadata:
|
|
149
|
-
res =
|
|
150
|
-
lambda: database_metadata_v2.sync_detailed(client=self._client, id=database_id)
|
|
151
|
-
)
|
|
157
|
+
res = request(database_metadata_v2, self._client, self._owner._timeout, id=database_id)
|
|
152
158
|
return parse_body(unwrap(res), to_metadata)
|
|
153
159
|
|
|
154
160
|
return self._retrying(call)
|
|
@@ -161,12 +167,12 @@ class DatabaseApi:
|
|
|
161
167
|
"""
|
|
162
168
|
|
|
163
169
|
def call() -> dict[str, str]:
|
|
164
|
-
res =
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
)
|
|
170
|
+
res = request(
|
|
171
|
+
database_checksum_v2,
|
|
172
|
+
self._client,
|
|
173
|
+
self._owner._timeout,
|
|
174
|
+
id=database_id,
|
|
175
|
+
format_=DatabaseFormat(format),
|
|
170
176
|
)
|
|
171
177
|
return parse_body(unwrap(res), checksums_of)
|
|
172
178
|
|
|
@@ -180,7 +186,7 @@ class DatabaseApi:
|
|
|
180
186
|
"""
|
|
181
187
|
|
|
182
188
|
def call() -> builtins.list[Download]:
|
|
183
|
-
res =
|
|
189
|
+
res = request(list_downloads, self._client, self._owner._timeout, limit=limit)
|
|
184
190
|
return parse_body(unwrap(res), downloads_of)
|
|
185
191
|
|
|
186
192
|
return self._retrying(call)
|
|
@@ -196,12 +202,12 @@ class DatabaseApi:
|
|
|
196
202
|
"""
|
|
197
203
|
|
|
198
204
|
def call() -> str:
|
|
199
|
-
res =
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
)
|
|
205
|
+
res = request(
|
|
206
|
+
download_database_v2,
|
|
207
|
+
self._client,
|
|
208
|
+
self._owner._timeout,
|
|
209
|
+
id=database_id,
|
|
210
|
+
format_=DatabaseFormat(format),
|
|
205
211
|
)
|
|
206
212
|
return redirect_location(res)
|
|
207
213
|
|
|
@@ -11,9 +11,12 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
import datetime
|
|
13
13
|
from dataclasses import dataclass, field
|
|
14
|
-
from typing import Any, Literal
|
|
14
|
+
from typing import Any, Literal, get_args
|
|
15
15
|
|
|
16
16
|
__all__ = [
|
|
17
|
+
"DATABASE_FORMATS",
|
|
18
|
+
"LICENSE_TYPES",
|
|
19
|
+
"STANDINGS",
|
|
17
20
|
"Database",
|
|
18
21
|
"DatabaseMetadata",
|
|
19
22
|
"DatabaseVersion",
|
|
@@ -37,7 +40,20 @@ Standing = Literal["licensed", "expired", "unlicensed"]
|
|
|
37
40
|
"""Where your organization stands with one database family."""
|
|
38
41
|
|
|
39
42
|
LicenseType = Literal["evaluation", "standard", "redistribute"]
|
|
40
|
-
"""What a
|
|
43
|
+
"""What a license permits you to do with the data. `None` when there is no license."""
|
|
44
|
+
|
|
45
|
+
DATABASE_FORMATS: tuple[Format, ...] = get_args(Format)
|
|
46
|
+
"""Every `Format`, at runtime.
|
|
47
|
+
|
|
48
|
+
A `Literal` is erased to nothing a program can check against, so a format read from a
|
|
49
|
+
flag, a form or a config file has these to be tested against before a call.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
STANDINGS: tuple[Standing, ...] = get_args(Standing)
|
|
53
|
+
"""Every `Standing`, at runtime."""
|
|
54
|
+
|
|
55
|
+
LICENSE_TYPES: tuple[LicenseType, ...] = get_args(LicenseType)
|
|
56
|
+
"""Every `LicenseType`, at runtime. `None`, for no license, is not one of them."""
|
|
41
57
|
|
|
42
58
|
Outcome = Literal["ok", "unauthorized", "denied", "expired", "unknown", "unavailable"]
|
|
43
59
|
"""How one download attempt ended, refusals included."""
|
|
@@ -49,7 +65,7 @@ class DatabaseVersion:
|
|
|
49
65
|
|
|
50
66
|
Old versions are frozen rather than migrated, so several stay downloadable at once.
|
|
51
67
|
`id` is what `download`, `checksums` and `metadata` take; the family `base` is what a
|
|
52
|
-
|
|
68
|
+
license is held against.
|
|
53
69
|
"""
|
|
54
70
|
|
|
55
71
|
id: str
|
|
@@ -60,13 +76,18 @@ class DatabaseVersion:
|
|
|
60
76
|
|
|
61
77
|
@dataclass(frozen=True, slots=True)
|
|
62
78
|
class Database:
|
|
63
|
-
"""One database FAMILY, with your organization's
|
|
79
|
+
"""One database FAMILY, with your organization's license beside it.
|
|
64
80
|
|
|
65
81
|
A family your organization has never licensed is still listed, with `standing` set to
|
|
66
82
|
`unlicensed`, so you can see what else exists. A family commissioned for a single
|
|
67
83
|
customer is a different matter: it is absent from this listing entirely for everyone
|
|
68
84
|
who does not license it. Absence here means "not yours to see", never "does not
|
|
69
85
|
exist", so the catalog is not the same document for every key.
|
|
86
|
+
|
|
87
|
+
A rolling license carries `renews_at`, when it next renews, and `notice_due_at`, the
|
|
88
|
+
last day notice of non-renewal can be given for the term ending then. Both are None
|
|
89
|
+
when there is no license, when it has no defined term, or when `expires` sets a hard
|
|
90
|
+
stop instead; `notice_due_at` is None too when the agreement records no notice period.
|
|
70
91
|
"""
|
|
71
92
|
|
|
72
93
|
base: str
|
|
@@ -76,6 +97,10 @@ class Database:
|
|
|
76
97
|
license_type: LicenseType | None
|
|
77
98
|
starts: datetime.datetime | None
|
|
78
99
|
expires: datetime.datetime | None
|
|
100
|
+
# Keyword-only with a default, so a Database built by hand before 2.2.0 still builds;
|
|
101
|
+
# the parser always sets both.
|
|
102
|
+
renews_at: datetime.datetime | None = field(default=None, kw_only=True)
|
|
103
|
+
notice_due_at: datetime.datetime | None = field(default=None, kw_only=True)
|
|
79
104
|
versions: tuple[DatabaseVersion, ...]
|
|
80
105
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
81
106
|
|
|
@@ -137,6 +162,8 @@ def to_database(body: dict[str, Any]) -> Database:
|
|
|
137
162
|
license_type=body["license_type"],
|
|
138
163
|
starts=_datetime(body["starts"]),
|
|
139
164
|
expires=_datetime(body["expires"]),
|
|
165
|
+
renews_at=_datetime(body["renews_at"]),
|
|
166
|
+
notice_due_at=_datetime(body["notice_due_at"]),
|
|
140
167
|
versions=tuple(to_version(v) for v in body["versions"]),
|
|
141
168
|
raw=body,
|
|
142
169
|
)
|
|
@@ -185,7 +212,7 @@ def to_download(body: dict[str, Any]) -> Download:
|
|
|
185
212
|
|
|
186
213
|
|
|
187
214
|
def _datetime(value: Any) -> datetime.datetime | None:
|
|
188
|
-
"""A NULLABLE wire timestamp. A
|
|
215
|
+
"""A NULLABLE wire timestamp. A license with no end date carries `expires: null`.
|
|
189
216
|
|
|
190
217
|
`fromisoformat` only learned to read a trailing `Z` in 3.11, which is one of the
|
|
191
218
|
three reasons this package floors there. A value that is neither null nor readable
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
}
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
|
-
"name": "
|
|
30
|
+
"name": "license-lapsed",
|
|
31
31
|
"status": 403,
|
|
32
32
|
"headers": {},
|
|
33
33
|
"body": {
|
|
@@ -78,6 +78,20 @@
|
|
|
78
78
|
"message": "INVALID_FORMAT"
|
|
79
79
|
}
|
|
80
80
|
},
|
|
81
|
+
{
|
|
82
|
+
"name": "unlisted-4xx-is-not-retryable",
|
|
83
|
+
"why": "The RANGE is the rule, not a list of statuses. An SDK that enumerated 400 and 404 passed every other case here while a 422 fell through to the retryable server_error default.",
|
|
84
|
+
"status": 422,
|
|
85
|
+
"headers": {},
|
|
86
|
+
"body": {
|
|
87
|
+
"rc": "INVALID_REQUEST"
|
|
88
|
+
},
|
|
89
|
+
"expect": {
|
|
90
|
+
"kind": "bad_request",
|
|
91
|
+
"retryable": false,
|
|
92
|
+
"message": "INVALID_REQUEST"
|
|
93
|
+
}
|
|
94
|
+
},
|
|
81
95
|
{
|
|
82
96
|
"name": "rate-limited-transient",
|
|
83
97
|
"status": 429,
|