newsdataapi 0.3.3__tar.gz → 0.3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/PKG-INFO +20 -8
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/README.md +19 -7
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/__init__.py +5 -1
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/client.py +19 -9
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/exceptions.py +16 -1
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/websocket.py +53 -16
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/tests/test_unit.py +34 -3
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/tests/test_websocket.py +137 -8
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/.github/workflows/ci.yml +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/.github/workflows/publish.yml +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/.gitignore +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/.python-version +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/LICENSE +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/examples/basic_usage.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/examples/websocket_usage.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/newsdata-logo.png +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/pyproject.toml +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/constants.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/csv_export.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/src/newsdataapi/py.typed +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/tests/__init__.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/tests/conftest.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/tests/test_integration.py +0 -0
- {newsdataapi-0.3.3 → newsdataapi-0.3.5}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: newsdataapi
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.5
|
|
4
4
|
Summary: Official Python client (SDK) for the Newsdata.io News API — fetch real-time, historical, crypto, and stock-market news via REST, with validation, retries, pagination, and typed exceptions.
|
|
5
5
|
Project-URL: Homepage, https://newsdata.io
|
|
6
6
|
Project-URL: Documentation, https://newsdata.io/documentation
|
|
@@ -221,6 +221,8 @@ ws = NewsDataApiWebSocket(
|
|
|
221
221
|
```python
|
|
222
222
|
from newsdataapi import (
|
|
223
223
|
NewsdataAPIError,
|
|
224
|
+
NewsdataApiKeyLimitExceededError,
|
|
225
|
+
NewsdataApiLimitExceededError,
|
|
224
226
|
NewsdataAuthError,
|
|
225
227
|
NewsdataNetworkError,
|
|
226
228
|
NewsdataRateLimitError,
|
|
@@ -230,6 +232,10 @@ try:
|
|
|
230
232
|
client.latest_api(q="news")
|
|
231
233
|
except NewsdataAuthError as e:
|
|
232
234
|
print(f"bad API key (HTTP {e.status_code})")
|
|
235
|
+
except NewsdataApiKeyLimitExceededError:
|
|
236
|
+
print("this API key's own credit cap is exhausted")
|
|
237
|
+
except NewsdataApiLimitExceededError:
|
|
238
|
+
print("account credit pool is exhausted")
|
|
233
239
|
except NewsdataRateLimitError as e:
|
|
234
240
|
print(f"rate limited; retry after {e.retry_after}s")
|
|
235
241
|
except NewsdataAPIError as e:
|
|
@@ -242,18 +248,24 @@ The full hierarchy:
|
|
|
242
248
|
|
|
243
249
|
```
|
|
244
250
|
NewsdataException
|
|
245
|
-
├── NewsdataValidationError
|
|
246
|
-
├── NewsdataAPIError
|
|
247
|
-
│ ├── NewsdataAuthError
|
|
248
|
-
│ ├── NewsdataRateLimitError
|
|
249
|
-
│
|
|
250
|
-
|
|
251
|
-
└──
|
|
251
|
+
├── NewsdataValidationError (also a ValueError; carries .param)
|
|
252
|
+
├── NewsdataAPIError (carries .status_code, .response_body)
|
|
253
|
+
│ ├── NewsdataAuthError (401 / 403)
|
|
254
|
+
│ ├── NewsdataRateLimitError (429; carries .retry_after)
|
|
255
|
+
│ │ ├── NewsdataApiLimitExceededError (account credit pool exhausted)
|
|
256
|
+
│ │ └── NewsdataApiKeyLimitExceededError (this API key's own credit cap exhausted)
|
|
257
|
+
│ └── NewsdataServerError (5xx)
|
|
258
|
+
├── NewsdataNetworkError (carries .original)
|
|
259
|
+
└── NewsdataWebSocketError (real-time stream)
|
|
252
260
|
└── NewsdataWebSocketAuthError (handshake 401 / 403, or policy-violation close 1008)
|
|
253
261
|
```
|
|
254
262
|
|
|
263
|
+
`NewsdataApiLimitExceededError` and `NewsdataApiKeyLimitExceededError` are both subclasses of `NewsdataRateLimitError`, so existing code that only catches `NewsdataRateLimitError` keeps working unchanged — catch the specific subclasses only if you want to react differently to each. Neither is ever retried.
|
|
264
|
+
|
|
255
265
|
`NewsdataException` is always a valid catch-all.
|
|
256
266
|
|
|
267
|
+
When a request succeeds and `include_headers=True`, the response dict's `response_headers` also carries `X-API-Limit-Remaining` (account pool) and, for API keys with a credit cap, `X-APIKEY-Limit-Remaining` (that key's own remaining credits).
|
|
268
|
+
|
|
257
269
|
## Save results to CSV
|
|
258
270
|
|
|
259
271
|
```python
|
|
@@ -169,6 +169,8 @@ ws = NewsDataApiWebSocket(
|
|
|
169
169
|
```python
|
|
170
170
|
from newsdataapi import (
|
|
171
171
|
NewsdataAPIError,
|
|
172
|
+
NewsdataApiKeyLimitExceededError,
|
|
173
|
+
NewsdataApiLimitExceededError,
|
|
172
174
|
NewsdataAuthError,
|
|
173
175
|
NewsdataNetworkError,
|
|
174
176
|
NewsdataRateLimitError,
|
|
@@ -178,6 +180,10 @@ try:
|
|
|
178
180
|
client.latest_api(q="news")
|
|
179
181
|
except NewsdataAuthError as e:
|
|
180
182
|
print(f"bad API key (HTTP {e.status_code})")
|
|
183
|
+
except NewsdataApiKeyLimitExceededError:
|
|
184
|
+
print("this API key's own credit cap is exhausted")
|
|
185
|
+
except NewsdataApiLimitExceededError:
|
|
186
|
+
print("account credit pool is exhausted")
|
|
181
187
|
except NewsdataRateLimitError as e:
|
|
182
188
|
print(f"rate limited; retry after {e.retry_after}s")
|
|
183
189
|
except NewsdataAPIError as e:
|
|
@@ -190,18 +196,24 @@ The full hierarchy:
|
|
|
190
196
|
|
|
191
197
|
```
|
|
192
198
|
NewsdataException
|
|
193
|
-
├── NewsdataValidationError
|
|
194
|
-
├── NewsdataAPIError
|
|
195
|
-
│ ├── NewsdataAuthError
|
|
196
|
-
│ ├── NewsdataRateLimitError
|
|
197
|
-
│
|
|
198
|
-
|
|
199
|
-
└──
|
|
199
|
+
├── NewsdataValidationError (also a ValueError; carries .param)
|
|
200
|
+
├── NewsdataAPIError (carries .status_code, .response_body)
|
|
201
|
+
│ ├── NewsdataAuthError (401 / 403)
|
|
202
|
+
│ ├── NewsdataRateLimitError (429; carries .retry_after)
|
|
203
|
+
│ │ ├── NewsdataApiLimitExceededError (account credit pool exhausted)
|
|
204
|
+
│ │ └── NewsdataApiKeyLimitExceededError (this API key's own credit cap exhausted)
|
|
205
|
+
│ └── NewsdataServerError (5xx)
|
|
206
|
+
├── NewsdataNetworkError (carries .original)
|
|
207
|
+
└── NewsdataWebSocketError (real-time stream)
|
|
200
208
|
└── NewsdataWebSocketAuthError (handshake 401 / 403, or policy-violation close 1008)
|
|
201
209
|
```
|
|
202
210
|
|
|
211
|
+
`NewsdataApiLimitExceededError` and `NewsdataApiKeyLimitExceededError` are both subclasses of `NewsdataRateLimitError`, so existing code that only catches `NewsdataRateLimitError` keeps working unchanged — catch the specific subclasses only if you want to react differently to each. Neither is ever retried.
|
|
212
|
+
|
|
203
213
|
`NewsdataException` is always a valid catch-all.
|
|
204
214
|
|
|
215
|
+
When a request succeeds and `include_headers=True`, the response dict's `response_headers` also carries `X-API-Limit-Remaining` (account pool) and, for API keys with a credit cap, `X-APIKEY-Limit-Remaining` (that key's own remaining credits).
|
|
216
|
+
|
|
205
217
|
## Save results to CSV
|
|
206
218
|
|
|
207
219
|
```python
|
|
@@ -4,6 +4,8 @@ from .client import NewsDataApiClient
|
|
|
4
4
|
from .csv_export import save_to_csv
|
|
5
5
|
from .exceptions import (
|
|
6
6
|
NewsdataAPIError,
|
|
7
|
+
NewsdataApiKeyLimitExceededError,
|
|
8
|
+
NewsdataApiLimitExceededError,
|
|
7
9
|
NewsdataAuthError,
|
|
8
10
|
NewsdataException,
|
|
9
11
|
NewsdataNetworkError,
|
|
@@ -15,12 +17,14 @@ from .exceptions import (
|
|
|
15
17
|
)
|
|
16
18
|
from .websocket import NewsDataApiWebSocket
|
|
17
19
|
|
|
18
|
-
__version__ = "0.3.
|
|
20
|
+
__version__ = "0.3.5"
|
|
19
21
|
|
|
20
22
|
__all__ = [
|
|
21
23
|
"NewsDataApiClient",
|
|
22
24
|
"NewsDataApiWebSocket",
|
|
23
25
|
"NewsdataAPIError",
|
|
26
|
+
"NewsdataApiKeyLimitExceededError",
|
|
27
|
+
"NewsdataApiLimitExceededError",
|
|
24
28
|
"NewsdataAuthError",
|
|
25
29
|
"NewsdataException",
|
|
26
30
|
"NewsdataNetworkError",
|
|
@@ -18,6 +18,8 @@ from requests.exceptions import RequestException
|
|
|
18
18
|
from . import constants, csv_export
|
|
19
19
|
from .exceptions import (
|
|
20
20
|
NewsdataAPIError,
|
|
21
|
+
NewsdataApiKeyLimitExceededError,
|
|
22
|
+
NewsdataApiLimitExceededError,
|
|
21
23
|
NewsdataAuthError,
|
|
22
24
|
NewsdataException,
|
|
23
25
|
NewsdataNetworkError,
|
|
@@ -85,7 +87,10 @@ _MUTEX_GROUPS: tuple[tuple[str, ...], ...] = (
|
|
|
85
87
|
("domain", "domainurl", "excludedomain", "excludedomainurl"),
|
|
86
88
|
)
|
|
87
89
|
|
|
88
|
-
|
|
90
|
+
_QUOTA_EXHAUSTED_ERRORS: dict[str, type[NewsdataRateLimitError]] = {
|
|
91
|
+
"ApiLimitExceeded": NewsdataApiLimitExceededError,
|
|
92
|
+
"ApiKeyLimitExceeded": NewsdataApiKeyLimitExceededError,
|
|
93
|
+
}
|
|
89
94
|
|
|
90
95
|
|
|
91
96
|
def _validate_params(user_params: Mapping[str, Any]) -> dict[str, Any]:
|
|
@@ -1190,13 +1195,19 @@ class NewsDataApiClient:
|
|
|
1190
1195
|
continue
|
|
1191
1196
|
|
|
1192
1197
|
elapsed = time.perf_counter() - t0
|
|
1193
|
-
|
|
1194
|
-
|
|
1198
|
+
apikey_limit_remaining = response.headers.get("X-APIKEY-Limit-Remaining")
|
|
1199
|
+
log_msg = "%d %s (%.2fs) X-API-Limit-Remaining=%s"
|
|
1200
|
+
log_args = [
|
|
1195
1201
|
response.status_code,
|
|
1196
1202
|
response.reason,
|
|
1197
1203
|
elapsed,
|
|
1198
1204
|
response.headers.get("X-API-Limit-Remaining"),
|
|
1199
|
-
|
|
1205
|
+
]
|
|
1206
|
+
if apikey_limit_remaining is not None:
|
|
1207
|
+
# Only sent for API keys that have a credit cap set.
|
|
1208
|
+
log_msg += " X-APIKEY-Limit-Remaining=%s"
|
|
1209
|
+
log_args.append(apikey_limit_remaining)
|
|
1210
|
+
logger.debug(log_msg, *log_args)
|
|
1200
1211
|
|
|
1201
1212
|
try:
|
|
1202
1213
|
body: Any = response.json()
|
|
@@ -1230,11 +1241,10 @@ class NewsDataApiClient:
|
|
|
1230
1241
|
# Rate limit.
|
|
1231
1242
|
if response.status_code == 429:
|
|
1232
1243
|
retry_after = _parse_retry_after(response.headers.get("Retry-After"))
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
raise NewsdataRateLimitError(
|
|
1244
|
+
error_code = _extract_error_code(body)
|
|
1245
|
+
exc_cls = _QUOTA_EXHAUSTED_ERRORS.get(error_code or "", NewsdataRateLimitError)
|
|
1246
|
+
if error_code in _QUOTA_EXHAUSTED_ERRORS or attempt >= self.max_retries:
|
|
1247
|
+
raise exc_cls(
|
|
1238
1248
|
body,
|
|
1239
1249
|
status_code=429,
|
|
1240
1250
|
response_body=body if isinstance(body, dict) else None,
|
|
@@ -40,7 +40,7 @@ class NewsdataAPIError(NewsdataException):
|
|
|
40
40
|
message: Any,
|
|
41
41
|
*,
|
|
42
42
|
status_code: int | None = None,
|
|
43
|
-
response_body: dict[str, Any] | None = None
|
|
43
|
+
response_body: dict[str, Any] | None = None
|
|
44
44
|
) -> None:
|
|
45
45
|
super().__init__(message)
|
|
46
46
|
self.status_code = status_code
|
|
@@ -76,6 +76,21 @@ class NewsdataRateLimitError(NewsdataAPIError):
|
|
|
76
76
|
self.retry_after = retry_after
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
class NewsdataApiLimitExceededError(NewsdataRateLimitError):
|
|
80
|
+
"""Raised on 429 when the account's overall credit pool is exhausted.
|
|
81
|
+
|
|
82
|
+
Blocks every API key on the account, regardless of any per-key cap.
|
|
83
|
+
Never retried.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class NewsdataApiKeyLimitExceededError(NewsdataRateLimitError):
|
|
88
|
+
"""Raised on 429 when this specific API key's own credit cap
|
|
89
|
+
(``max_credit_limit``) is exhausted, even though the account's overall
|
|
90
|
+
pool may still have credits left. Never retried.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
|
|
79
94
|
class NewsdataServerError(NewsdataAPIError):
|
|
80
95
|
"""Raised on 5xx responses when retries are exhausted."""
|
|
81
96
|
|
|
@@ -30,7 +30,7 @@ _WEBSOCKET_SUPPORTED = sys.version_info >= (3, 10)
|
|
|
30
30
|
|
|
31
31
|
if _WEBSOCKET_SUPPORTED:
|
|
32
32
|
from websockets.asyncio import client as _asyncio_client
|
|
33
|
-
from websockets.exceptions import ConnectionClosedError, InvalidStatus
|
|
33
|
+
from websockets.exceptions import ConnectionClosedError, InvalidMessage, InvalidStatus
|
|
34
34
|
from websockets.sync import client as _sync_client
|
|
35
35
|
|
|
36
36
|
logger = logging.getLogger(__name__)
|
|
@@ -44,13 +44,16 @@ def _check_registration_id(registration_id: str) -> None:
|
|
|
44
44
|
)
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
def _permanent_auth_error(
|
|
47
|
+
def _permanent_auth_error(
|
|
48
|
+
exc: Exception, error_response: dict[str, Any] | None
|
|
49
|
+
) -> NewsdataWebSocketAuthError | None:
|
|
48
50
|
"""Return the auth error to raise if ``exc`` is a permanent rejection.
|
|
49
51
|
|
|
50
52
|
Returns ``None`` for transient failures (which the caller reconnects on):
|
|
51
53
|
handshake 401/403 and policy-violation close 1008 are permanent; anything
|
|
52
54
|
else (other handshake status, other close codes, network ``OSError``) is
|
|
53
|
-
transient.
|
|
55
|
+
transient. ``error_response`` is the error response the server sent just
|
|
56
|
+
before closing, if any; it becomes the exception's argument.
|
|
54
57
|
"""
|
|
55
58
|
if isinstance(exc, InvalidStatus):
|
|
56
59
|
if exc.response.status_code in (401, 403):
|
|
@@ -59,14 +62,20 @@ def _permanent_auth_error(exc: Exception) -> NewsdataWebSocketAuthError | None:
|
|
|
59
62
|
if isinstance(exc, ConnectionClosedError):
|
|
60
63
|
close = exc.rcvd
|
|
61
64
|
if close is not None and close.code == constants.WS_POLICY_VIOLATION:
|
|
65
|
+
if error_response is not None:
|
|
66
|
+
return NewsdataWebSocketAuthError(error_response)
|
|
62
67
|
return NewsdataWebSocketAuthError(close.reason or "connection rejected")
|
|
63
68
|
return None
|
|
64
69
|
return None
|
|
65
70
|
|
|
66
71
|
|
|
67
|
-
def _transient_error(
|
|
72
|
+
def _transient_error(
|
|
73
|
+
exc: Exception, error_response: dict[str, Any] | None
|
|
74
|
+
) -> NewsdataWebSocketError:
|
|
68
75
|
"""Wrap a transient failure as a :class:`NewsdataWebSocketError` (used only
|
|
69
76
|
when ``reconnect=False`` so the caller stops instead of retrying)."""
|
|
77
|
+
if error_response is not None:
|
|
78
|
+
return NewsdataWebSocketError(error_response)
|
|
70
79
|
if isinstance(exc, InvalidStatus):
|
|
71
80
|
return NewsdataWebSocketError(f"handshake failed (HTTP {exc.response.status_code})")
|
|
72
81
|
if isinstance(exc, ConnectionClosedError):
|
|
@@ -295,24 +304,38 @@ class NewsDataApiWebSocket:
|
|
|
295
304
|
url = self._url(registration_id)
|
|
296
305
|
delay = self._reconnect_delay
|
|
297
306
|
while True:
|
|
307
|
+
error_response: dict[str, Any] | None = None
|
|
298
308
|
try:
|
|
299
309
|
with _sync_client.connect(url, **self._connect_kwargs()) as websocket:
|
|
300
310
|
self._ws_sync = websocket
|
|
301
311
|
logger.info("connected to %s", _redact_url(url))
|
|
302
|
-
|
|
312
|
+
# reset after a successful connect, unless the server refuses it
|
|
313
|
+
refused_delay, delay = delay, self._reconnect_delay
|
|
303
314
|
for message in websocket:
|
|
304
315
|
try:
|
|
305
316
|
response = json.loads(message)
|
|
306
317
|
except ValueError:
|
|
307
318
|
continue # skip malformed frames
|
|
308
|
-
if isinstance(response, dict):
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
319
|
+
if not isinstance(response, dict):
|
|
320
|
+
continue
|
|
321
|
+
if response.get("status") == "error":
|
|
322
|
+
# sent right before the server closes the connection:
|
|
323
|
+
# raise it with that close, and keep backing off
|
|
324
|
+
error_response, delay = response, refused_delay
|
|
325
|
+
continue
|
|
326
|
+
yield response
|
|
327
|
+
except (
|
|
328
|
+
InvalidStatus,
|
|
329
|
+
InvalidMessage, # connection lost during the handshake
|
|
330
|
+
ConnectionClosedError,
|
|
331
|
+
OSError,
|
|
332
|
+
asyncio.TimeoutError, # not an OSError on Python 3.10
|
|
333
|
+
) as exc:
|
|
334
|
+
auth = _permanent_auth_error(exc, error_response)
|
|
312
335
|
if auth is not None:
|
|
313
336
|
raise auth from exc
|
|
314
337
|
if not self._reconnect:
|
|
315
|
-
raise _transient_error(exc) from exc
|
|
338
|
+
raise _transient_error(exc, error_response) from exc
|
|
316
339
|
logger.warning(
|
|
317
340
|
"connection failed (%s); reconnecting in %.1fs", exc, delay
|
|
318
341
|
)
|
|
@@ -330,26 +353,40 @@ class NewsDataApiWebSocket:
|
|
|
330
353
|
url = self._url(registration_id)
|
|
331
354
|
delay = self._reconnect_delay
|
|
332
355
|
while True:
|
|
356
|
+
error_response: dict[str, Any] | None = None
|
|
333
357
|
try:
|
|
334
358
|
async with _asyncio_client.connect(
|
|
335
359
|
url, **self._connect_kwargs()
|
|
336
360
|
) as websocket:
|
|
337
361
|
self._ws_async = websocket
|
|
338
362
|
logger.info("connected to %s", _redact_url(url))
|
|
339
|
-
|
|
363
|
+
# reset after a successful connect, unless the server refuses it
|
|
364
|
+
refused_delay, delay = delay, self._reconnect_delay
|
|
340
365
|
async for message in websocket:
|
|
341
366
|
try:
|
|
342
367
|
response = json.loads(message)
|
|
343
368
|
except ValueError:
|
|
344
369
|
continue # skip malformed frames
|
|
345
|
-
if isinstance(response, dict):
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
370
|
+
if not isinstance(response, dict):
|
|
371
|
+
continue
|
|
372
|
+
if response.get("status") == "error":
|
|
373
|
+
# sent right before the server closes the connection:
|
|
374
|
+
# raise it with that close, and keep backing off
|
|
375
|
+
error_response, delay = response, refused_delay
|
|
376
|
+
continue
|
|
377
|
+
yield response
|
|
378
|
+
except (
|
|
379
|
+
InvalidStatus,
|
|
380
|
+
InvalidMessage, # connection lost during the handshake
|
|
381
|
+
ConnectionClosedError,
|
|
382
|
+
OSError,
|
|
383
|
+
asyncio.TimeoutError, # not an OSError on Python 3.10
|
|
384
|
+
) as exc:
|
|
385
|
+
auth = _permanent_auth_error(exc, error_response)
|
|
349
386
|
if auth is not None:
|
|
350
387
|
raise auth from exc
|
|
351
388
|
if not self._reconnect:
|
|
352
|
-
raise _transient_error(exc) from exc
|
|
389
|
+
raise _transient_error(exc, error_response) from exc
|
|
353
390
|
logger.warning(
|
|
354
391
|
"connection failed (%s); reconnecting in %.1fs", exc, delay
|
|
355
392
|
)
|
|
@@ -19,6 +19,8 @@ import responses
|
|
|
19
19
|
from newsdataapi import (
|
|
20
20
|
NewsDataApiClient,
|
|
21
21
|
NewsdataAPIError,
|
|
22
|
+
NewsdataApiKeyLimitExceededError,
|
|
23
|
+
NewsdataApiLimitExceededError,
|
|
22
24
|
NewsDataApiWebSocket,
|
|
23
25
|
NewsdataAuthError,
|
|
24
26
|
NewsdataException,
|
|
@@ -728,6 +730,26 @@ def test_include_headers_merges_into_response(
|
|
|
728
730
|
assert response["response_headers"]["X-API-Limit-Remaining"] == "100"
|
|
729
731
|
|
|
730
732
|
|
|
733
|
+
def test_include_headers_merges_apikey_limit_header(
|
|
734
|
+
mocked_responses: responses.RequestsMock,
|
|
735
|
+
) -> None:
|
|
736
|
+
"""``X-APIKEY-Limit-Remaining`` is only sent for API keys with a credit
|
|
737
|
+
cap, but the client passes it through like any other header."""
|
|
738
|
+
with NewsDataApiClient("k", include_headers=True, max_retries=1) as c:
|
|
739
|
+
mocked_responses.get(
|
|
740
|
+
LATEST_URL,
|
|
741
|
+
json={"status": "success", "results": []},
|
|
742
|
+
status=200,
|
|
743
|
+
headers={
|
|
744
|
+
"X-API-Limit-Remaining": "100",
|
|
745
|
+
"X-APIKEY-Limit-Remaining": "25",
|
|
746
|
+
},
|
|
747
|
+
)
|
|
748
|
+
response = c.latest_api()
|
|
749
|
+
assert isinstance(response, dict)
|
|
750
|
+
assert response["response_headers"]["X-APIKEY-Limit-Remaining"] == "25"
|
|
751
|
+
|
|
752
|
+
|
|
731
753
|
def test_status_error_with_200_raises_api_error(
|
|
732
754
|
client: NewsDataApiClient,
|
|
733
755
|
mocked_responses: responses.RequestsMock,
|
|
@@ -835,18 +857,27 @@ def test_429_exhausted_raises_rate_limit_with_retry_after(
|
|
|
835
857
|
assert exc_info.value.retry_after == 5
|
|
836
858
|
|
|
837
859
|
|
|
838
|
-
@pytest.mark.parametrize(
|
|
860
|
+
@pytest.mark.parametrize(
|
|
861
|
+
("code", "expected_exc"),
|
|
862
|
+
[
|
|
863
|
+
("ApiLimitExceeded", NewsdataApiLimitExceededError),
|
|
864
|
+
("ApiKeyLimitExceeded", NewsdataApiKeyLimitExceededError),
|
|
865
|
+
],
|
|
866
|
+
)
|
|
839
867
|
def test_429_quota_exhausted_raises_without_retry(
|
|
840
868
|
code: str,
|
|
869
|
+
expected_exc: type[NewsdataRateLimitError],
|
|
841
870
|
client: NewsDataApiClient,
|
|
842
871
|
mocked_responses: responses.RequestsMock,
|
|
843
872
|
no_sleep: None,
|
|
844
873
|
) -> None:
|
|
845
|
-
"""A 429 whose error code means exhausted credits is never retried
|
|
874
|
+
"""A 429 whose error code means exhausted credits is never retried,
|
|
875
|
+
and raises the specific subclass for that code."""
|
|
846
876
|
body = {"status": "error", "results": {"message": "limit", "code": code}}
|
|
847
877
|
mocked_responses.get(LATEST_URL, json=body, status=429)
|
|
848
|
-
with pytest.raises(
|
|
878
|
+
with pytest.raises(expected_exc) as exc_info:
|
|
849
879
|
client.latest_api()
|
|
880
|
+
assert isinstance(exc_info.value, NewsdataRateLimitError)
|
|
850
881
|
assert len(mocked_responses.calls) == 1
|
|
851
882
|
assert exc_info.value.response_body == body
|
|
852
883
|
|
|
@@ -12,7 +12,7 @@ import json
|
|
|
12
12
|
|
|
13
13
|
import pytest
|
|
14
14
|
from websockets.datastructures import Headers
|
|
15
|
-
from websockets.exceptions import ConnectionClosedError, InvalidStatus
|
|
15
|
+
from websockets.exceptions import ConnectionClosedError, InvalidMessage, InvalidStatus
|
|
16
16
|
from websockets.frames import Close
|
|
17
17
|
from websockets.http11 import Response
|
|
18
18
|
|
|
@@ -39,6 +39,11 @@ def _news(*titles: str) -> str:
|
|
|
39
39
|
)
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
def _error(code: str, message: str = "rejected") -> str:
|
|
43
|
+
"""An error frame, shaped like the server's error response."""
|
|
44
|
+
return json.dumps({"status": "error", "results": {"message": message, "code": code}})
|
|
45
|
+
|
|
46
|
+
|
|
42
47
|
def _closed(code: int, reason: str = "") -> ConnectionClosedError:
|
|
43
48
|
return ConnectionClosedError(Close(code, reason), None)
|
|
44
49
|
|
|
@@ -47,6 +52,11 @@ def _invalid_status(status: int) -> InvalidStatus:
|
|
|
47
52
|
return InvalidStatus(Response(status, "x", Headers(), b""))
|
|
48
53
|
|
|
49
54
|
|
|
55
|
+
# Terminals the fakes raise from connect() itself (handshake / network
|
|
56
|
+
# failures) rather than during iteration.
|
|
57
|
+
_CONNECT_ERRORS = (InvalidStatus, InvalidMessage, OSError, asyncio.TimeoutError)
|
|
58
|
+
|
|
59
|
+
|
|
50
60
|
class _FakeWS:
|
|
51
61
|
"""A scripted sync connection: yields ``messages`` then raises ``terminal``
|
|
52
62
|
(a ConnectionClosedError) or ends normally if ``terminal`` is None."""
|
|
@@ -75,9 +85,10 @@ class _FakeWS:
|
|
|
75
85
|
class _FakeConnect:
|
|
76
86
|
"""Scripted ``connect``: each attempt is ``(messages, terminal)``.
|
|
77
87
|
|
|
78
|
-
``terminal`` raised from ``connect()`` itself if it is
|
|
79
|
-
|
|
80
|
-
Once the script is exhausted it closes with 1008 to stop any
|
|
88
|
+
``terminal`` raised from ``connect()`` itself if it is one of
|
|
89
|
+
``_CONNECT_ERRORS`` (handshake / network failure), otherwise during
|
|
90
|
+
iteration. Once the script is exhausted it closes with 1008 to stop any
|
|
91
|
+
reconnect loop.
|
|
81
92
|
"""
|
|
82
93
|
|
|
83
94
|
def __init__(self, *attempts: tuple[list, BaseException | None]) -> None:
|
|
@@ -95,7 +106,7 @@ class _FakeConnect:
|
|
|
95
106
|
messages, terminal = self._attempts.pop(0)
|
|
96
107
|
else:
|
|
97
108
|
messages, terminal = [], _closed(1008, "exhausted")
|
|
98
|
-
if isinstance(terminal,
|
|
109
|
+
if isinstance(terminal, _CONNECT_ERRORS):
|
|
99
110
|
raise terminal
|
|
100
111
|
conn = _FakeWS(messages, terminal)
|
|
101
112
|
self.conns.append(conn)
|
|
@@ -239,6 +250,18 @@ def test_close_1008_raises_auth_with_reason(monkeypatch: pytest.MonkeyPatch) ->
|
|
|
239
250
|
assert fake.calls == 1
|
|
240
251
|
|
|
241
252
|
|
|
253
|
+
def test_error_response_raises_auth_with_it(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
254
|
+
error = _error("ApiLimitExceeded", "out of credits")
|
|
255
|
+
fake = _FakeConnect(([_news("A"), error], _closed(1008, "ApiLimitExceeded")))
|
|
256
|
+
monkeypatch.setattr(CONNECT, fake)
|
|
257
|
+
got: list[str] = []
|
|
258
|
+
with pytest.raises(NewsdataWebSocketAuthError) as exc_info:
|
|
259
|
+
_collect(_ws(), got) # reconnect=True default
|
|
260
|
+
assert got == ["A"] # the error response is raised, never yielded
|
|
261
|
+
assert exc_info.value.args[0] == json.loads(error)
|
|
262
|
+
assert fake.calls == 1
|
|
263
|
+
|
|
264
|
+
|
|
242
265
|
# --- reconnect=False: stop on first drop ------------------------------------
|
|
243
266
|
|
|
244
267
|
|
|
@@ -269,6 +292,17 @@ def test_no_reconnect_handshake_5xx_raises(monkeypatch: pytest.MonkeyPatch) -> N
|
|
|
269
292
|
assert not isinstance(exc_info.value, NewsdataWebSocketAuthError)
|
|
270
293
|
|
|
271
294
|
|
|
295
|
+
def test_no_reconnect_server_error_response_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
296
|
+
error = _error("ServerError", "try again later")
|
|
297
|
+
fake = _FakeConnect(([error], _closed(1011, "ServerError")))
|
|
298
|
+
monkeypatch.setattr(CONNECT, fake)
|
|
299
|
+
with pytest.raises(NewsdataWebSocketError) as exc_info:
|
|
300
|
+
list(_ws(reconnect=False).stream("r"))
|
|
301
|
+
assert not isinstance(exc_info.value, NewsdataWebSocketAuthError)
|
|
302
|
+
assert exc_info.value.args[0] == json.loads(error)
|
|
303
|
+
assert fake.calls == 1
|
|
304
|
+
|
|
305
|
+
|
|
272
306
|
# --- reconnect=True: retry transient drops ----------------------------------
|
|
273
307
|
|
|
274
308
|
|
|
@@ -324,6 +358,40 @@ def test_reconnects_on_network_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
|
324
358
|
assert fake.calls == 2
|
|
325
359
|
|
|
326
360
|
|
|
361
|
+
def test_reconnects_on_handshake_connection_lost(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
362
|
+
fake = _FakeConnect(
|
|
363
|
+
([], InvalidMessage("did not receive a valid HTTP response")), # lost mid-handshake
|
|
364
|
+
([_news("A")], _closed(1008, "stop")),
|
|
365
|
+
)
|
|
366
|
+
monkeypatch.setattr(CONNECT, fake)
|
|
367
|
+
got: list[str] = []
|
|
368
|
+
with pytest.raises(NewsdataWebSocketAuthError):
|
|
369
|
+
_collect(_ws(), got)
|
|
370
|
+
assert got == ["A"]
|
|
371
|
+
assert fake.calls == 2
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def test_server_error_responses_keep_backing_off(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
375
|
+
slept: list[float] = []
|
|
376
|
+
monkeypatch.setattr(
|
|
377
|
+
"newsdataapi.websocket.time.sleep", lambda d: slept.append(d)
|
|
378
|
+
)
|
|
379
|
+
refused = ([_error("ServerError")], _closed(1011, "ServerError"))
|
|
380
|
+
fake = _FakeConnect(
|
|
381
|
+
refused,
|
|
382
|
+
refused,
|
|
383
|
+
refused,
|
|
384
|
+
([_news("A")], _closed(1013, "SendTimeout")), # accepted -> backoff starts over
|
|
385
|
+
([_news("B")], _closed(1008, "stop")),
|
|
386
|
+
)
|
|
387
|
+
monkeypatch.setattr(CONNECT, fake)
|
|
388
|
+
got: list[str] = []
|
|
389
|
+
with pytest.raises(NewsdataWebSocketAuthError):
|
|
390
|
+
_collect(_ws(), got)
|
|
391
|
+
assert got == ["A", "B"] # error responses are never yielded
|
|
392
|
+
assert slept == [1.0, 2.0, 4.0, 1.0]
|
|
393
|
+
|
|
394
|
+
|
|
327
395
|
# ===========================================================================
|
|
328
396
|
# Async streaming (NewsDataApiWebSocket.stream_async)
|
|
329
397
|
# ===========================================================================
|
|
@@ -373,8 +441,8 @@ class _FakeAsyncCM:
|
|
|
373
441
|
class _FakeAsyncConnect:
|
|
374
442
|
"""Scripted async ``connect``: each attempt is ``(messages, terminal)``.
|
|
375
443
|
|
|
376
|
-
|
|
377
|
-
|
|
444
|
+
A ``_CONNECT_ERRORS`` terminal is raised from ``__aenter__`` (handshake
|
|
445
|
+
failure); otherwise it is raised during async iteration.
|
|
378
446
|
"""
|
|
379
447
|
|
|
380
448
|
def __init__(self, *attempts: tuple[list, BaseException | None]) -> None:
|
|
@@ -392,7 +460,7 @@ class _FakeAsyncConnect:
|
|
|
392
460
|
messages, terminal = self._attempts.pop(0)
|
|
393
461
|
else:
|
|
394
462
|
messages, terminal = [], _closed(1008, "exhausted")
|
|
395
|
-
if isinstance(terminal,
|
|
463
|
+
if isinstance(terminal, _CONNECT_ERRORS):
|
|
396
464
|
return _FakeAsyncCM(None, terminal)
|
|
397
465
|
conn = _FakeAsyncWS(messages, terminal)
|
|
398
466
|
self.conns.append(conn)
|
|
@@ -502,6 +570,18 @@ def test_async_close_1008_raises_auth_with_reason(monkeypatch: pytest.MonkeyPatc
|
|
|
502
570
|
assert fake.calls == 1
|
|
503
571
|
|
|
504
572
|
|
|
573
|
+
def test_async_error_response_raises_auth_with_it(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
574
|
+
error = _error("ApiLimitExceeded", "out of credits")
|
|
575
|
+
fake = _FakeAsyncConnect(([_news("A"), error], _closed(1008, "ApiLimitExceeded")))
|
|
576
|
+
monkeypatch.setattr(CONNECT_ASYNC, fake)
|
|
577
|
+
got: list[str] = []
|
|
578
|
+
with pytest.raises(NewsdataWebSocketAuthError) as exc_info:
|
|
579
|
+
_acollect(_ws(), got) # reconnect=True default
|
|
580
|
+
assert got == ["A"] # the error response is raised, never yielded
|
|
581
|
+
assert exc_info.value.args[0] == json.loads(error)
|
|
582
|
+
assert fake.calls == 1
|
|
583
|
+
|
|
584
|
+
|
|
505
585
|
# --- reconnect=False --------------------------------------------------------
|
|
506
586
|
|
|
507
587
|
|
|
@@ -542,3 +622,52 @@ def test_async_reconnects_on_network_error(monkeypatch: pytest.MonkeyPatch) -> N
|
|
|
542
622
|
_acollect(_ws(), got)
|
|
543
623
|
assert got == ["A"]
|
|
544
624
|
assert fake.calls == 2
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def test_async_reconnects_on_handshake_connection_lost(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
628
|
+
fake = _FakeAsyncConnect(
|
|
629
|
+
([], InvalidMessage("did not receive a valid HTTP response")), # lost mid-handshake
|
|
630
|
+
([_news("A")], _closed(1008, "stop")),
|
|
631
|
+
)
|
|
632
|
+
monkeypatch.setattr(CONNECT_ASYNC, fake)
|
|
633
|
+
got: list[str] = []
|
|
634
|
+
with pytest.raises(NewsdataWebSocketAuthError):
|
|
635
|
+
_acollect(_ws(), got)
|
|
636
|
+
assert got == ["A"]
|
|
637
|
+
assert fake.calls == 2
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def test_async_reconnects_on_open_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
641
|
+
fake = _FakeAsyncConnect(
|
|
642
|
+
([], asyncio.TimeoutError()), # handshake timeout; not an OSError on Python 3.10
|
|
643
|
+
([_news("A")], _closed(1008, "stop")),
|
|
644
|
+
)
|
|
645
|
+
monkeypatch.setattr(CONNECT_ASYNC, fake)
|
|
646
|
+
got: list[str] = []
|
|
647
|
+
with pytest.raises(NewsdataWebSocketAuthError):
|
|
648
|
+
_acollect(_ws(), got)
|
|
649
|
+
assert got == ["A"]
|
|
650
|
+
assert fake.calls == 2
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def test_async_server_error_responses_keep_backing_off(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
654
|
+
slept: list[float] = []
|
|
655
|
+
|
|
656
|
+
async def record(delay: float) -> None:
|
|
657
|
+
slept.append(delay)
|
|
658
|
+
|
|
659
|
+
monkeypatch.setattr("newsdataapi.websocket.asyncio.sleep", record)
|
|
660
|
+
refused = ([_error("ServerError")], _closed(1011, "ServerError"))
|
|
661
|
+
fake = _FakeAsyncConnect(
|
|
662
|
+
refused,
|
|
663
|
+
refused,
|
|
664
|
+
refused,
|
|
665
|
+
([_news("A")], _closed(1013, "SendTimeout")), # accepted -> backoff starts over
|
|
666
|
+
([_news("B")], _closed(1008, "stop")),
|
|
667
|
+
)
|
|
668
|
+
monkeypatch.setattr(CONNECT_ASYNC, fake)
|
|
669
|
+
got: list[str] = []
|
|
670
|
+
with pytest.raises(NewsdataWebSocketAuthError):
|
|
671
|
+
_acollect(_ws(), got)
|
|
672
|
+
assert got == ["A", "B"] # error responses are never yielded
|
|
673
|
+
assert slept == [1.0, 2.0, 4.0, 1.0]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|