internetdata 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {internetdata-2.2.0 → internetdata-2.3.0}/PKG-INFO +5 -2
- {internetdata-2.2.0 → internetdata-2.3.0}/README.md +4 -1
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/__init__.py +1 -1
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/aio.py +45 -13
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/client.py +43 -12
- {internetdata-2.2.0 → internetdata-2.3.0}/tests/helpers.py +19 -17
- {internetdata-2.2.0 → internetdata-2.3.0}/tests/test_client.py +101 -10
- {internetdata-2.2.0 → internetdata-2.3.0}/tests/test_conformance.py +17 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/.gitignore +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/LICENSE +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/pyproject.toml +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/scripts/download-spec.sh +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/scripts/generate.sh +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/scripts/publish.sh +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/scripts/v2_subset.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/spec/openapi.yaml +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_core.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/__init__.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/__init__.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/__init__.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/database_checksum_v2.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/database_metadata_v2.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/download_database_v2.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/list_databases.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/list_downloads.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/client.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/errors.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/__init__.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_checksum_v2_response_200.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_format.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_license_type_type_1.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_license_type_type_2_type_1.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_license_type_type_3_type_1.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata_column.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata_sample.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata_sample_additional_property_item.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata_schema.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata_size.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_version.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/db_checksums.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/download.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/download_outcome.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/error.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/list_databases_response_200.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/list_downloads_response_200.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/standing.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/types.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/errors.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/models.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/py.typed +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/testdata/testdata.json +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/tests/conftest.py +0 -0
- {internetdata-2.2.0 → internetdata-2.3.0}/tests/test_download.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: internetdata
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: Official Python client library for the InternetData API. Download and verify licensed IP datasets.
|
|
5
5
|
Project-URL: Homepage, https://internetdata.io
|
|
6
6
|
Project-URL: Documentation, https://docs.internetdata.io
|
|
@@ -167,12 +167,15 @@ except InternetDataError as err:
|
|
|
167
167
|
|
|
168
168
|
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
169
169
|
|
|
170
|
-
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError`
|
|
170
|
+
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError` where it is set:
|
|
171
171
|
|
|
172
172
|
```python
|
|
173
173
|
client = InternetData(api_key, timeout=10)
|
|
174
|
+
catalog = client.database.list(timeout=2)
|
|
174
175
|
```
|
|
175
176
|
|
|
177
|
+
From 2.3.0, `list`, `metadata`, `checksums`, `downloads` and `download_url` each take a keyword-only `timeout` in seconds, bounding each attempt at that one call in place of the client's; `None` there means the client's own rather than no bound. `download` and `download_bytes` deliberately take none and raise `TypeError` if handed one, rather than accepting it and quietly doing nothing: a transfer runs to gigabytes and minutes, so any bound that suits a JSON call would abandon a healthy download. `download_url` does take one, because minting the link is an ordinary API request - it bounds that request, not whatever you do with the link afterwards.
|
|
178
|
+
|
|
176
179
|
**Changed in 2.1.0:** the default was 10 seconds, and it bounded each read of a response rather than the whole attempt, so a response trickling in slowly could run past it for as long as the server kept sending.
|
|
177
180
|
|
|
178
181
|
**Changed in 2.2.0:** a timeout of 0 or less, NaN or a string used to be accepted, and failed every call.
|
|
@@ -133,12 +133,15 @@ except InternetDataError as err:
|
|
|
133
133
|
|
|
134
134
|
`kind` is one of `bad_request`, `unauthorized`, `forbidden`, `rate_limited`, `quota_exceeded`, `server_error` or `network`. `message` is the API's own result code, passed through as it was sent, so you can switch on `NOT_LICENSED` against `LICENSE_EXPIRED` without reading the status.
|
|
135
135
|
|
|
136
|
-
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError`
|
|
136
|
+
A request that runs past its `timeout` fails with `network`, and is retried like any other network failure. The default is 30 seconds per attempt, body included, so a retried call can take longer in total. A database transfer is exempt, so `download` and `download_bytes` are never cut off part way through a large file. Set it on the client, in seconds, or pass `None` for no bound. Anything else that is not a number greater than 0 raises `ValueError` where it is set:
|
|
137
137
|
|
|
138
138
|
```python
|
|
139
139
|
client = InternetData(api_key, timeout=10)
|
|
140
|
+
catalog = client.database.list(timeout=2)
|
|
140
141
|
```
|
|
141
142
|
|
|
143
|
+
From 2.3.0, `list`, `metadata`, `checksums`, `downloads` and `download_url` each take a keyword-only `timeout` in seconds, bounding each attempt at that one call in place of the client's; `None` there means the client's own rather than no bound. `download` and `download_bytes` deliberately take none and raise `TypeError` if handed one, rather than accepting it and quietly doing nothing: a transfer runs to gigabytes and minutes, so any bound that suits a JSON call would abandon a healthy download. `download_url` does take one, because minting the link is an ordinary API request - it bounds that request, not whatever you do with the link afterwards.
|
|
144
|
+
|
|
142
145
|
**Changed in 2.1.0:** the default was 10 seconds, and it bounded each read of a response rather than the whole attempt, so a response trickling in slowly could run past it for as long as the server kept sending.
|
|
143
146
|
|
|
144
147
|
**Changed in 2.2.0:** a timeout of 0 or less, NaN or a string used to be accepted, and failed every call.
|
|
@@ -96,6 +96,10 @@ class AsyncInternetData:
|
|
|
96
96
|
) -> None:
|
|
97
97
|
await self.aclose()
|
|
98
98
|
|
|
99
|
+
# A per-call timeout, checked, or the client's own when the call gave none.
|
|
100
|
+
def _bound(self, timeout: float | None) -> float | None:
|
|
101
|
+
return self._timeout if timeout is None else check_timeout(timeout)
|
|
102
|
+
|
|
99
103
|
async def _retrying(self, call: Callable[[], Awaitable[T]], retries: int) -> T:
|
|
100
104
|
attempt = 0
|
|
101
105
|
while True:
|
|
@@ -115,12 +119,17 @@ class AsyncDatabaseApi:
|
|
|
115
119
|
|
|
116
120
|
`list` is a method here, which shadows the builtin for everything else in the class
|
|
117
121
|
body, so the return annotations name `builtins.list` explicitly.
|
|
122
|
+
|
|
123
|
+
Every call here that asks the API a question takes `timeout`, in seconds, bounding
|
|
124
|
+
each ATTEMPT of that call alone and overriding the client's. The two transfers take
|
|
125
|
+
none and refuse one rather than ignoring it: a database runs to gigabytes and minutes,
|
|
126
|
+
so any bound that suits a JSON call would abandon a healthy download.
|
|
118
127
|
"""
|
|
119
128
|
|
|
120
129
|
def __init__(self, owner: AsyncInternetData) -> None:
|
|
121
130
|
self._owner = owner
|
|
122
131
|
|
|
123
|
-
async def list(self) -> builtins.list[Database]:
|
|
132
|
+
async def list(self, *, timeout: float | None = None) -> builtins.list[Database]:
|
|
124
133
|
"""The published catalog as YOUR organization may see it.
|
|
125
134
|
|
|
126
135
|
Every family carries a `standing`, so one you have never bought is listed as
|
|
@@ -128,33 +137,43 @@ class AsyncDatabaseApi:
|
|
|
128
137
|
single customer, which is absent entirely for everyone else. Nothing is cached
|
|
129
138
|
and nothing is reconstructed here - what you get is what the server sent for the
|
|
130
139
|
key you are holding.
|
|
140
|
+
|
|
141
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
131
142
|
"""
|
|
132
143
|
|
|
133
144
|
async def call() -> builtins.list[Database]:
|
|
134
|
-
res = await request_async(list_databases, self._client, self.
|
|
145
|
+
res = await request_async(list_databases, self._client, self._bound(timeout))
|
|
135
146
|
return parse_body(unwrap(res), databases_of)
|
|
136
147
|
|
|
137
148
|
return await self._retrying(call)
|
|
138
149
|
|
|
139
|
-
async def metadata(self, database_id: str) -> DatabaseMetadata:
|
|
140
|
-
"""What is inside one database: freshness, row count, columns, samples and sizes.
|
|
150
|
+
async def metadata(self, database_id: str, *, timeout: float | None = None) -> DatabaseMetadata:
|
|
151
|
+
"""What is inside one database: freshness, row count, columns, samples and sizes.
|
|
152
|
+
|
|
153
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
154
|
+
"""
|
|
141
155
|
|
|
142
156
|
async def call() -> DatabaseMetadata:
|
|
143
157
|
res = await request_async(
|
|
144
|
-
database_metadata_v2, self._client, self.
|
|
158
|
+
database_metadata_v2, self._client, self._bound(timeout), id=database_id
|
|
145
159
|
)
|
|
146
160
|
return parse_body(unwrap(res), to_metadata)
|
|
147
161
|
|
|
148
162
|
return await self._retrying(call)
|
|
149
163
|
|
|
150
|
-
async def checksums(
|
|
151
|
-
|
|
164
|
+
async def checksums(
|
|
165
|
+
self, database_id: str, format: Format, *, timeout: float | None = None
|
|
166
|
+
) -> dict[str, str]:
|
|
167
|
+
"""Every checksum published for one database file, keyed by algorithm.
|
|
168
|
+
|
|
169
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
170
|
+
"""
|
|
152
171
|
|
|
153
172
|
async def call() -> dict[str, str]:
|
|
154
173
|
res = await request_async(
|
|
155
174
|
database_checksum_v2,
|
|
156
175
|
self._client,
|
|
157
|
-
self.
|
|
176
|
+
self._bound(timeout),
|
|
158
177
|
id=database_id,
|
|
159
178
|
format_=DatabaseFormat(format),
|
|
160
179
|
)
|
|
@@ -162,18 +181,25 @@ class AsyncDatabaseApi:
|
|
|
162
181
|
|
|
163
182
|
return await self._retrying(call)
|
|
164
183
|
|
|
165
|
-
async def downloads(
|
|
166
|
-
|
|
184
|
+
async def downloads(
|
|
185
|
+
self, limit: int = DEFAULT_DOWNLOADS_LIMIT, *, timeout: float | None = None
|
|
186
|
+
) -> builtins.list[Download]:
|
|
187
|
+
"""Your organization's recent download attempts, newest first.
|
|
188
|
+
|
|
189
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
190
|
+
"""
|
|
167
191
|
|
|
168
192
|
async def call() -> builtins.list[Download]:
|
|
169
193
|
res = await request_async(
|
|
170
|
-
list_downloads, self._client, self.
|
|
194
|
+
list_downloads, self._client, self._bound(timeout), limit=limit
|
|
171
195
|
)
|
|
172
196
|
return parse_body(unwrap(res), downloads_of)
|
|
173
197
|
|
|
174
198
|
return await self._retrying(call)
|
|
175
199
|
|
|
176
|
-
async def download_url(
|
|
200
|
+
async def download_url(
|
|
201
|
+
self, database_id: str, format: Format, *, timeout: float | None = None
|
|
202
|
+
) -> str:
|
|
177
203
|
"""The time-limited URL for one database file.
|
|
178
204
|
|
|
179
205
|
The API answers `302` to object storage, and the link carries its own signature,
|
|
@@ -181,13 +207,16 @@ class AsyncDatabaseApi:
|
|
|
181
207
|
rather than followed so the caller decides how to move a file that reaches
|
|
182
208
|
gigabytes; the link authorizes the START of a transfer, so one already running is
|
|
183
209
|
not interrupted when it lapses.
|
|
210
|
+
|
|
211
|
+
`timeout` bounds each attempt at MINTING the link, which is an ordinary API
|
|
212
|
+
request, and says nothing about the transfer you then run with it.
|
|
184
213
|
"""
|
|
185
214
|
|
|
186
215
|
async def call() -> str:
|
|
187
216
|
res = await request_async(
|
|
188
217
|
download_database_v2,
|
|
189
218
|
self._client,
|
|
190
|
-
self.
|
|
219
|
+
self._bound(timeout),
|
|
191
220
|
id=database_id,
|
|
192
221
|
format_=DatabaseFormat(format),
|
|
193
222
|
)
|
|
@@ -266,5 +295,8 @@ class AsyncDatabaseApi:
|
|
|
266
295
|
def _client(self) -> AuthenticatedClient:
|
|
267
296
|
return self._owner._client
|
|
268
297
|
|
|
298
|
+
def _bound(self, timeout: float | None) -> float | None:
|
|
299
|
+
return self._owner._bound(timeout)
|
|
300
|
+
|
|
269
301
|
async def _retrying(self, call: Callable[[], Awaitable[T]]) -> T:
|
|
270
302
|
return await self._owner._retrying(call, self._owner._retries)
|
|
@@ -61,8 +61,10 @@ class InternetData:
|
|
|
61
61
|
|
|
62
62
|
`timeout` is how long one attempt at a request may take, in seconds, body included, so a
|
|
63
63
|
call that is retried can take longer in total; None means no bound, and a database
|
|
64
|
-
transfer is exempt.
|
|
65
|
-
|
|
64
|
+
transfer is exempt. Every `database` call but the two transfers also takes `timeout`,
|
|
65
|
+
which overrides the client's for that call alone. Anything else that is not a finite
|
|
66
|
+
number greater than 0 is a `ValueError` where it is set, rather than a failure of every
|
|
67
|
+
call.
|
|
66
68
|
|
|
67
69
|
Holds an HTTP connection pool, so use it as a context manager or call `close()` when
|
|
68
70
|
you are done with it.
|
|
@@ -102,6 +104,10 @@ class InternetData:
|
|
|
102
104
|
) -> None:
|
|
103
105
|
self.close()
|
|
104
106
|
|
|
107
|
+
# A per-call timeout, checked, or the client's own when the call gave none.
|
|
108
|
+
def _bound(self, timeout: float | None) -> float | None:
|
|
109
|
+
return self._timeout if timeout is None else check_timeout(timeout)
|
|
110
|
+
|
|
105
111
|
def _retrying(self, call: Callable[[], T], retries: int) -> T:
|
|
106
112
|
attempt = 0
|
|
107
113
|
while True:
|
|
@@ -121,12 +127,17 @@ class DatabaseApi:
|
|
|
121
127
|
|
|
122
128
|
`list` is a method here, which shadows the builtin for everything else in the class
|
|
123
129
|
body, so the return annotations name `builtins.list` explicitly.
|
|
130
|
+
|
|
131
|
+
Every call here that asks the API a question takes `timeout`, in seconds, bounding
|
|
132
|
+
each ATTEMPT of that call alone and overriding the client's. The two transfers take
|
|
133
|
+
none and refuse one rather than ignoring it: a database runs to gigabytes and minutes,
|
|
134
|
+
so any bound that suits a JSON call would abandon a healthy download.
|
|
124
135
|
"""
|
|
125
136
|
|
|
126
137
|
def __init__(self, owner: InternetData) -> None:
|
|
127
138
|
self._owner = owner
|
|
128
139
|
|
|
129
|
-
def list(self) -> builtins.list[Database]:
|
|
140
|
+
def list(self, *, timeout: float | None = None) -> builtins.list[Database]:
|
|
130
141
|
"""The published catalog as YOUR organization may see it.
|
|
131
142
|
|
|
132
143
|
Every family carries a `standing`, so one you have never bought is listed as
|
|
@@ -138,39 +149,47 @@ class DatabaseApi:
|
|
|
138
149
|
|
|
139
150
|
A license covers a FAMILY, while a download names a version, so the ids for the
|
|
140
151
|
other calls come from each entry's `versions`.
|
|
152
|
+
|
|
153
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
141
154
|
"""
|
|
142
155
|
|
|
143
156
|
def call() -> builtins.list[Database]:
|
|
144
|
-
res = request(list_databases, self._client, self.
|
|
157
|
+
res = request(list_databases, self._client, self._bound(timeout))
|
|
145
158
|
return parse_body(unwrap(res), databases_of)
|
|
146
159
|
|
|
147
160
|
return self._retrying(call)
|
|
148
161
|
|
|
149
|
-
def metadata(self, database_id: str) -> DatabaseMetadata:
|
|
162
|
+
def metadata(self, database_id: str, *, timeout: float | None = None) -> DatabaseMetadata:
|
|
150
163
|
"""What is inside one database: freshness, row count, columns, samples and sizes.
|
|
151
164
|
|
|
152
165
|
Cheap enough to poll: it answers `updated` and `entries` without moving the
|
|
153
166
|
build. `size` is the number to budget a transfer against before starting one.
|
|
167
|
+
|
|
168
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
154
169
|
"""
|
|
155
170
|
|
|
156
171
|
def call() -> DatabaseMetadata:
|
|
157
|
-
res = request(database_metadata_v2, self._client, self.
|
|
172
|
+
res = request(database_metadata_v2, self._client, self._bound(timeout), id=database_id)
|
|
158
173
|
return parse_body(unwrap(res), to_metadata)
|
|
159
174
|
|
|
160
175
|
return self._retrying(call)
|
|
161
176
|
|
|
162
|
-
def checksums(
|
|
177
|
+
def checksums(
|
|
178
|
+
self, database_id: str, format: Format, *, timeout: float | None = None
|
|
179
|
+
) -> dict[str, str]:
|
|
163
180
|
"""Every checksum published for one database file, keyed by algorithm.
|
|
164
181
|
|
|
165
182
|
Keyed rather than one digest because the API publishes md5, sha1, sha256 and
|
|
166
183
|
sha512 side by side and which of them you want is your verifier's business.
|
|
184
|
+
|
|
185
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
167
186
|
"""
|
|
168
187
|
|
|
169
188
|
def call() -> dict[str, str]:
|
|
170
189
|
res = request(
|
|
171
190
|
database_checksum_v2,
|
|
172
191
|
self._client,
|
|
173
|
-
self.
|
|
192
|
+
self._bound(timeout),
|
|
174
193
|
id=database_id,
|
|
175
194
|
format_=DatabaseFormat(format),
|
|
176
195
|
)
|
|
@@ -178,20 +197,26 @@ class DatabaseApi:
|
|
|
178
197
|
|
|
179
198
|
return self._retrying(call)
|
|
180
199
|
|
|
181
|
-
def downloads(
|
|
200
|
+
def downloads(
|
|
201
|
+
self, limit: int = DEFAULT_DOWNLOADS_LIMIT, *, timeout: float | None = None
|
|
202
|
+
) -> builtins.list[Download]:
|
|
182
203
|
"""Your organization's recent download attempts, newest first.
|
|
183
204
|
|
|
184
205
|
Refusals are listed too: a denial is what answers "it stopped working", and its
|
|
185
206
|
absence answers nothing.
|
|
207
|
+
|
|
208
|
+
`timeout` bounds each attempt at this call alone, in place of the client's.
|
|
186
209
|
"""
|
|
187
210
|
|
|
188
211
|
def call() -> builtins.list[Download]:
|
|
189
|
-
res = request(list_downloads, self._client, self.
|
|
212
|
+
res = request(list_downloads, self._client, self._bound(timeout), limit=limit)
|
|
190
213
|
return parse_body(unwrap(res), downloads_of)
|
|
191
214
|
|
|
192
215
|
return self._retrying(call)
|
|
193
216
|
|
|
194
|
-
def download_url(
|
|
217
|
+
def download_url(
|
|
218
|
+
self, database_id: str, format: Format, *, timeout: float | None = None
|
|
219
|
+
) -> str:
|
|
195
220
|
"""The time-limited URL for one database file.
|
|
196
221
|
|
|
197
222
|
The API answers `302` to object storage, and the link carries its own signature,
|
|
@@ -199,13 +224,16 @@ class DatabaseApi:
|
|
|
199
224
|
rather than followed so the caller decides how to move a file that reaches
|
|
200
225
|
gigabytes; the link authorizes the START of a transfer, so one already running is
|
|
201
226
|
not interrupted when it lapses.
|
|
227
|
+
|
|
228
|
+
`timeout` bounds each attempt at MINTING the link, which is an ordinary API
|
|
229
|
+
request, and says nothing about the transfer you then run with it.
|
|
202
230
|
"""
|
|
203
231
|
|
|
204
232
|
def call() -> str:
|
|
205
233
|
res = request(
|
|
206
234
|
download_database_v2,
|
|
207
235
|
self._client,
|
|
208
|
-
self.
|
|
236
|
+
self._bound(timeout),
|
|
209
237
|
id=database_id,
|
|
210
238
|
format_=DatabaseFormat(format),
|
|
211
239
|
)
|
|
@@ -279,5 +307,8 @@ class DatabaseApi:
|
|
|
279
307
|
def _client(self) -> AuthenticatedClient:
|
|
280
308
|
return self._owner._client
|
|
281
309
|
|
|
310
|
+
def _bound(self, timeout: float | None) -> float | None:
|
|
311
|
+
return self._owner._bound(timeout)
|
|
312
|
+
|
|
282
313
|
def _retrying(self, call: Callable[[], T]) -> T:
|
|
283
314
|
return self._owner._retrying(call, self._owner._retries)
|
|
@@ -83,32 +83,34 @@ class DatabaseAdapter:
|
|
|
83
83
|
def __init__(self, client: InternetData | AsyncInternetData) -> None:
|
|
84
84
|
self._client = client
|
|
85
85
|
|
|
86
|
-
def list(self) -> Any:
|
|
87
|
-
return self._call("list")
|
|
86
|
+
def list(self, **kwargs: Any) -> Any:
|
|
87
|
+
return self._call("list", **kwargs)
|
|
88
88
|
|
|
89
|
-
def metadata(self, database_id: str) -> Any:
|
|
90
|
-
return self._call("metadata", database_id)
|
|
89
|
+
def metadata(self, database_id: str, **kwargs: Any) -> Any:
|
|
90
|
+
return self._call("metadata", database_id, **kwargs)
|
|
91
91
|
|
|
92
|
-
def checksums(self, database_id: str, format: str) -> Any:
|
|
93
|
-
return self._call("checksums", database_id, format)
|
|
92
|
+
def checksums(self, database_id: str, format: str, **kwargs: Any) -> Any:
|
|
93
|
+
return self._call("checksums", database_id, format, **kwargs)
|
|
94
94
|
|
|
95
|
-
def downloads(self, *args: Any) -> Any:
|
|
96
|
-
return self._call("downloads", *args)
|
|
95
|
+
def downloads(self, *args: Any, **kwargs: Any) -> Any:
|
|
96
|
+
return self._call("downloads", *args, **kwargs)
|
|
97
97
|
|
|
98
|
-
def download_url(self, database_id: str, format: str) -> Any:
|
|
99
|
-
return self._call("download_url", database_id, format)
|
|
98
|
+
def download_url(self, database_id: str, format: str, **kwargs: Any) -> Any:
|
|
99
|
+
return self._call("download_url", database_id, format, **kwargs)
|
|
100
100
|
|
|
101
|
-
def download(self, database_id: str, format: str, path: Any) -> Any:
|
|
102
|
-
return self._call("download", database_id, format, path)
|
|
101
|
+
def download(self, database_id: str, format: str, path: Any, **kwargs: Any) -> Any:
|
|
102
|
+
return self._call("download", database_id, format, path, **kwargs)
|
|
103
103
|
|
|
104
|
-
def download_bytes(self, database_id: str, format: str) -> Any:
|
|
105
|
-
return self._call("download_bytes", database_id, format)
|
|
104
|
+
def download_bytes(self, database_id: str, format: str, **kwargs: Any) -> Any:
|
|
105
|
+
return self._call("download_bytes", database_id, format, **kwargs)
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
# Keyword arguments are forwarded untouched and none is named here: a `timeout`
|
|
108
|
+
# in this signature would swallow the TypeError a transfer must raise for one.
|
|
109
|
+
def _call(self, name: str, *args: Any, **kwargs: Any) -> Any:
|
|
108
110
|
method = getattr(self._client.database, name)
|
|
109
111
|
if isinstance(self._client, InternetData):
|
|
110
|
-
return method(*args)
|
|
111
|
-
return asyncio.run(method(*args))
|
|
112
|
+
return method(*args, **kwargs)
|
|
113
|
+
return asyncio.run(method(*args, **kwargs))
|
|
112
114
|
|
|
113
115
|
|
|
114
116
|
class Stub:
|
|
@@ -8,7 +8,8 @@ import datetime
|
|
|
8
8
|
import inspect
|
|
9
9
|
import time
|
|
10
10
|
from collections.abc import Callable
|
|
11
|
-
from
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, get_args
|
|
12
13
|
|
|
13
14
|
import attrs
|
|
14
15
|
import httpx
|
|
@@ -20,7 +21,6 @@ from helpers import (
|
|
|
20
21
|
DOWNLOADS_PATH,
|
|
21
22
|
LIST_PATH,
|
|
22
23
|
METADATA_PATH,
|
|
23
|
-
ClientAdapter,
|
|
24
24
|
ClientFactory,
|
|
25
25
|
SlowBody,
|
|
26
26
|
Stub,
|
|
@@ -37,6 +37,7 @@ from internetdata import (
|
|
|
37
37
|
InternetData,
|
|
38
38
|
InternetDataError,
|
|
39
39
|
MetadataColumn,
|
|
40
|
+
Outcome,
|
|
40
41
|
_core,
|
|
41
42
|
)
|
|
42
43
|
from internetdata._generated.models.database import Database as WireDatabase
|
|
@@ -48,6 +49,7 @@ from internetdata._generated.models.database_metadata_column import (
|
|
|
48
49
|
)
|
|
49
50
|
from internetdata._generated.models.database_version import DatabaseVersion as WireDatabaseVersion
|
|
50
51
|
from internetdata._generated.models.download import Download as WireDownload
|
|
52
|
+
from internetdata._generated.models.download_outcome import DownloadOutcome as WireDownloadOutcome
|
|
51
53
|
|
|
52
54
|
METADATA = {
|
|
53
55
|
"id": "bogon_ip_v1",
|
|
@@ -98,13 +100,28 @@ TIMEOUT = 0.3
|
|
|
98
100
|
TRICKLE = 0.02
|
|
99
101
|
# What starting a thread or an event loop may add to a deadline.
|
|
100
102
|
SLACK = 0.25
|
|
103
|
+
# The client's bound sits far above the call's, so an override accepted and ignored fails
|
|
104
|
+
# on how long the call took rather than passing on the timeout it hit anyway.
|
|
105
|
+
CLIENT_TIMEOUT = 0.6
|
|
106
|
+
CALL_TIMEOUT = 0.1
|
|
107
|
+
|
|
108
|
+
# Every call that asks the API a question, called with the keyword arguments given.
|
|
109
|
+
JSON_CALLS: dict[str, Callable[..., Any]] = {
|
|
110
|
+
"list": lambda client, **kw: client.database.list(**kw),
|
|
111
|
+
"metadata": lambda client, **kw: client.database.metadata("bogon_ip_v1", **kw),
|
|
112
|
+
"checksums": lambda client, **kw: client.database.checksums("bogon_ip_v1", "csvgz", **kw),
|
|
113
|
+
"downloads": lambda client, **kw: client.database.downloads(**kw),
|
|
114
|
+
"download_url": lambda client, **kw: client.database.download_url("bogon_ip_v1", "csvgz", **kw),
|
|
115
|
+
}
|
|
101
116
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
"
|
|
117
|
+
# The two transfers, which must REFUSE a per-call timeout rather than ignore one.
|
|
118
|
+
TRANSFERS: dict[str, Callable[..., Any]] = {
|
|
119
|
+
"download": lambda client, path, **kw: client.database.download(
|
|
120
|
+
"bogon_ip_v1", "csvgz", path, **kw
|
|
121
|
+
),
|
|
122
|
+
"download_bytes": lambda client, _path, **kw: client.database.download_bytes(
|
|
123
|
+
"bogon_ip_v1", "csvgz", **kw
|
|
124
|
+
),
|
|
108
125
|
}
|
|
109
126
|
|
|
110
127
|
|
|
@@ -205,6 +222,13 @@ def test_every_field_the_pinned_spec_serves_is_on_the_model(wire: Any, ours: Any
|
|
|
205
222
|
assert served - modeled == set(), f"{ours.__name__} lacks what the spec serves"
|
|
206
223
|
|
|
207
224
|
|
|
225
|
+
def test_the_outcome_vocabulary_is_the_pinned_specs() -> None:
|
|
226
|
+
"""The staleness pin on `Outcome`, the one hand-written Literal the corpus does not
|
|
227
|
+
carry: the generated enum comes from the same pinned spec, so a re-pin that adds an
|
|
228
|
+
outcome turns this red instead of leaving the type a member short."""
|
|
229
|
+
assert sorted(get_args(Outcome)) == sorted(member.value for member in WireDownloadOutcome)
|
|
230
|
+
|
|
231
|
+
|
|
208
232
|
def test_a_database_cannot_be_mutated(make_client: ClientFactory) -> None:
|
|
209
233
|
stub = Stub({LIST_PATH: {"body": {"databases": [database("bogon_ip")]}}})
|
|
210
234
|
client = make_client(transport=stub.transport)
|
|
@@ -349,6 +373,66 @@ def test_a_timed_out_attempt_is_retried_under_a_bound_of_its_own(
|
|
|
349
373
|
)
|
|
350
374
|
|
|
351
375
|
|
|
376
|
+
@pytest.mark.parametrize("call", JSON_CALLS)
|
|
377
|
+
def test_a_per_call_timeout_bounds_a_trickling_body_and_leaves_the_clients_own_alone(
|
|
378
|
+
make_client: ClientFactory, call: str
|
|
379
|
+
) -> None:
|
|
380
|
+
with SlowBody(trickle=TRICKLE) as server:
|
|
381
|
+
client = make_client(base_url=server.url, timeout=CLIENT_TIMEOUT, retries=0)
|
|
382
|
+
overridden, first = _timed(lambda: JSON_CALLS[call](client, timeout=CALL_TIMEOUT))
|
|
383
|
+
default, second = _timed(lambda: JSON_CALLS[call](client))
|
|
384
|
+
|
|
385
|
+
_assert_timed_out(first)
|
|
386
|
+
_assert_bounded_by(overridden, CALL_TIMEOUT)
|
|
387
|
+
# A per-call value written onto the one httpx client every call shares passes the
|
|
388
|
+
# call above and fails this one, which is what the second call is here for.
|
|
389
|
+
_assert_timed_out(second)
|
|
390
|
+
_assert_bounded_by(default, CLIENT_TIMEOUT)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf"), "30", True])
|
|
394
|
+
@pytest.mark.parametrize("call", JSON_CALLS)
|
|
395
|
+
def test_a_per_call_timeout_no_attempt_can_meet_is_refused_before_any_request(
|
|
396
|
+
make_client: ClientFactory, call: str, timeout: Any
|
|
397
|
+
) -> None:
|
|
398
|
+
"""Accepted, each of these failed the call instead, after the retries' backoff."""
|
|
399
|
+
stub = Stub({})
|
|
400
|
+
client = make_client(transport=stub.transport, retries=0)
|
|
401
|
+
|
|
402
|
+
with pytest.raises(ValueError, match="timeout"):
|
|
403
|
+
JSON_CALLS[call](client, timeout=timeout)
|
|
404
|
+
|
|
405
|
+
assert stub.requests == []
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
@pytest.mark.parametrize("call", TRANSFERS)
|
|
409
|
+
def test_a_transfer_refuses_a_per_call_timeout(
|
|
410
|
+
make_client: ClientFactory, call: str, tmp_path: Path
|
|
411
|
+
) -> None:
|
|
412
|
+
"""A transfer is exempt from the deadline, so the option is not in its signature and
|
|
413
|
+
Python refuses it for us. Accepted and quietly ignored, a caller would be told
|
|
414
|
+
nothing; honored, a bound that suits a JSON call would abandon a healthy download."""
|
|
415
|
+
stub = Stub({})
|
|
416
|
+
client = make_client(transport=stub.transport, retries=0)
|
|
417
|
+
path = tmp_path / "bogon_ip_v1.csv.gz"
|
|
418
|
+
|
|
419
|
+
with pytest.raises(TypeError, match="timeout"):
|
|
420
|
+
TRANSFERS[call](client, path, timeout=CALL_TIMEOUT)
|
|
421
|
+
|
|
422
|
+
assert stub.requests == []
|
|
423
|
+
assert not path.exists()
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@pytest.mark.parametrize("name", JSON_CALLS)
|
|
427
|
+
def test_every_json_call_takes_a_per_call_timeout(make_client: ClientFactory, name: str) -> None:
|
|
428
|
+
"""The other half of the refusal above, which without this would pass just as well on
|
|
429
|
+
a surface that had never been given a per-call timeout at all."""
|
|
430
|
+
timeout = inspect.signature(getattr(make_client().client.database, name)).parameters["timeout"]
|
|
431
|
+
|
|
432
|
+
assert timeout.kind is inspect.Parameter.KEYWORD_ONLY
|
|
433
|
+
assert timeout.default is None
|
|
434
|
+
|
|
435
|
+
|
|
352
436
|
def test_the_default_timeout_is_thirty_seconds() -> None:
|
|
353
437
|
for client in (InternetData, AsyncInternetData):
|
|
354
438
|
assert inspect.signature(client).parameters["timeout"].default == 30
|
|
@@ -424,6 +508,13 @@ def _assert_timed_out(outcome: Any) -> None:
|
|
|
424
508
|
|
|
425
509
|
|
|
426
510
|
def _assert_one_bound(elapsed: float) -> None:
|
|
427
|
-
|
|
428
|
-
|
|
511
|
+
_assert_bounded_by(elapsed, TIMEOUT)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _assert_bounded_by(elapsed: float, bound: float) -> None:
|
|
515
|
+
"""One bound's worth of waiting, with an UPPER and a LOWER limit: without the upper a
|
|
516
|
+
call that gave up at some other bound passes, and without the lower one abandoned at
|
|
517
|
+
its first chunk does."""
|
|
518
|
+
assert bound - 0.05 <= elapsed < bound + SLACK, (
|
|
519
|
+
f"gave up after {elapsed:.2f}s against a {bound}s bound"
|
|
429
520
|
)
|
|
@@ -200,6 +200,23 @@ def test_a_listing_is_never_reused_across_clients(make_client: ClientFactory) ->
|
|
|
200
200
|
assert len(stub.requests) == 3
|
|
201
201
|
|
|
202
202
|
|
|
203
|
+
# Each visibility rule the corpus names, and the test above that holds it. The three tests
|
|
204
|
+
# each check that their own rule is still in the corpus; this checks the other direction,
|
|
205
|
+
# so a rule added to the corpus fails here until something in this module holds it.
|
|
206
|
+
VISIBILITY_RULES = {
|
|
207
|
+
"listing-is-returned-as-served": test_the_listing_is_returned_exactly_as_served,
|
|
208
|
+
"no-catalog-is-compiled-into-the-client": test_no_catalog_is_compiled_into_the_client,
|
|
209
|
+
"a-listing-is-never-reused-across-clients": test_a_listing_is_never_reused_across_clients,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def test_every_visibility_rule_in_the_corpus_is_held_here() -> None:
|
|
214
|
+
rules = TESTDATA["visibility"]["clientRules"]
|
|
215
|
+
assert rules, "the corpus pins no visibility rules"
|
|
216
|
+
unheld = sorted(set(rules) - set(VISIBILITY_RULES))
|
|
217
|
+
assert unheld == [], f"the corpus adds {unheld} and this suite does not check it"
|
|
218
|
+
|
|
219
|
+
|
|
203
220
|
def test_checksums_unwrap_past_the_envelope(make_client: ClientFactory) -> None:
|
|
204
221
|
"""`checksums` nests under a key, beside `id` and `format`.
|
|
205
222
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/api/database_v_2/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_format.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_metadata.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/database_version.py
RENAMED
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/db_checksums.py
RENAMED
|
File without changes
|
|
File without changes
|
{internetdata-2.2.0 → internetdata-2.3.0}/src/internetdata/_generated/models/download_outcome.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|