taba-client 0.3.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taba/__init__.py +54 -0
- taba/_calls.py +232 -0
- taba/_json.py +165 -0
- taba/_transport.py +310 -0
- taba/_version.py +5 -0
- taba/aio.py +236 -0
- taba/cancel.py +62 -0
- taba/client.py +387 -0
- taba/errors.py +49 -0
- taba/models.py +409 -0
- taba/py.typed +0 -0
- taba_client-0.3.19.dist-info/METADATA +106 -0
- taba_client-0.3.19.dist-info/RECORD +15 -0
- taba_client-0.3.19.dist-info/WHEEL +5 -0
- taba_client-0.3.19.dist-info/top_level.txt +1 -0
taba/_transport.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The client's HTTP transport: builds the request, sends it, reads the body, turns the API error into a
|
|
3
|
+
`TabaError` and does the call retries of the common contract (doc 21).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import errno
|
|
7
|
+
import http.client
|
|
8
|
+
import socket
|
|
9
|
+
import ssl
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Callable, Dict, List, Optional, Tuple
|
|
14
|
+
from urllib.parse import quote, urlsplit
|
|
15
|
+
|
|
16
|
+
from ._json import JsonError, JsonFields
|
|
17
|
+
from .cancel import Cancelled, CancelToken
|
|
18
|
+
from .errors import TabaError
|
|
19
|
+
from .models import Config
|
|
20
|
+
|
|
21
|
+
#: The signature of the wait between attempts (tests replace it): `sleep(seconds, token)` raises `Cancelled`.
|
|
22
|
+
Sleep = Callable[[float, CancelToken], None]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def default_sleep(seconds: float, token: CancelToken) -> None:
|
|
26
|
+
if token.wait(seconds):
|
|
27
|
+
raise Cancelled()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def segment(value: str) -> str:
|
|
31
|
+
return quote(str(value), safe="")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class ApiCall:
|
|
36
|
+
"""
|
|
37
|
+
The description of a request to the API: method, path, query, body, headers and the rules of the call
|
|
38
|
+
(timeout, retry of 502/504, responses that count as success after a 503).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
method: str
|
|
42
|
+
#: Path with the segments already encoded (`/topics/orders`).
|
|
43
|
+
path: str
|
|
44
|
+
query: List[Tuple[str, str]] = field(default_factory=list)
|
|
45
|
+
body: Optional[bytes] = None
|
|
46
|
+
content_type: Optional[str] = None
|
|
47
|
+
request_id: Optional[str] = None
|
|
48
|
+
traceparent: Optional[str] = None
|
|
49
|
+
#: Overrides the total time of the call in seconds (`consume` uses `wait_time + 10`).
|
|
50
|
+
timeout: Optional[float] = None
|
|
51
|
+
#: Retries 502 and 504: only publishing to a FIFO queue, where the primary deduplicates.
|
|
52
|
+
retry_gateway: bool = False
|
|
53
|
+
#: After a 503, these responses count as success (the operation had already been applied).
|
|
54
|
+
recover: List[Tuple[int, str]] = field(default_factory=list)
|
|
55
|
+
|
|
56
|
+
def with_query(self, key: str, value: object) -> "ApiCall":
|
|
57
|
+
if isinstance(value, bool):
|
|
58
|
+
value = "true" if value else "false"
|
|
59
|
+
self.query.append((key, str(value)))
|
|
60
|
+
return self
|
|
61
|
+
|
|
62
|
+
def with_json(self, text: str) -> "ApiCall":
|
|
63
|
+
self.body = text.encode("utf-8")
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class ApiReply:
|
|
69
|
+
status: int
|
|
70
|
+
body: bytes
|
|
71
|
+
recovered: bool = False
|
|
72
|
+
#: The response carries `Taba-Topic-Recreated: true` (`PUT /topics/{topic}`).
|
|
73
|
+
recreated: bool = False
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def text(self) -> str:
|
|
77
|
+
return self.body.decode("utf-8", errors="replace")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class _Retry(Exception):
|
|
81
|
+
"""A failure before the body was sent (refused, DNS): the call can be repeated like a 503."""
|
|
82
|
+
|
|
83
|
+
def __init__(self, error: TabaError) -> None:
|
|
84
|
+
super().__init__(str(error))
|
|
85
|
+
self.error = error
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _is_connect_failure(e: BaseException) -> bool:
|
|
89
|
+
if isinstance(e, (ConnectionRefusedError, socket.gaierror)):
|
|
90
|
+
return True
|
|
91
|
+
return isinstance(e, OSError) and e.errno in (errno.ENETUNREACH, errno.EHOSTUNREACH)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _describe(e: BaseException) -> str:
|
|
95
|
+
return "%s: %s" % (type(e).__name__, e) if str(e) else type(e).__name__
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _error_code_of(body: bytes) -> Optional[str]:
|
|
99
|
+
try:
|
|
100
|
+
return JsonFields.parse(body.decode("utf-8", errors="replace")).string("code")
|
|
101
|
+
except JsonError:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _error_from(status: int, headers: Dict[str, str], body: bytes) -> TabaError:
|
|
106
|
+
text = body.decode("utf-8", errors="replace")
|
|
107
|
+
code = message = request_id = details = None
|
|
108
|
+
try:
|
|
109
|
+
fields = JsonFields.parse(text)
|
|
110
|
+
code = fields.string("code")
|
|
111
|
+
if code is not None:
|
|
112
|
+
message = fields.string("message") or ""
|
|
113
|
+
request_id = fields.string("requestId")
|
|
114
|
+
details = fields.raw("details")
|
|
115
|
+
except JsonError:
|
|
116
|
+
pass # Not an ErrorResponse: falls to the generic code below.
|
|
117
|
+
if code is None:
|
|
118
|
+
code = "http.%d" % status
|
|
119
|
+
trimmed = text.strip()
|
|
120
|
+
message = "response %d with no body" % status if not trimmed else trimmed[:300]
|
|
121
|
+
retry_header = headers.get("retry-after")
|
|
122
|
+
retry_after = None
|
|
123
|
+
if retry_header is not None and retry_header.strip().isdigit():
|
|
124
|
+
retry_after = int(retry_header.strip())
|
|
125
|
+
return TabaError(
|
|
126
|
+
status,
|
|
127
|
+
code,
|
|
128
|
+
message or "",
|
|
129
|
+
request_id=request_id or headers.get("x-request-id"),
|
|
130
|
+
retry_after=retry_after,
|
|
131
|
+
details=details,
|
|
132
|
+
body=body,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class Transport:
|
|
137
|
+
def __init__(self, config: Config, sleep: Optional[Sleep] = None) -> None:
|
|
138
|
+
self._config = config
|
|
139
|
+
self._scheme, self._host, self._port, self._prefix = self._validate(config.base_url)
|
|
140
|
+
self._context = config.ssl_context
|
|
141
|
+
if self._scheme == "https" and self._context is None:
|
|
142
|
+
self._context = ssl.create_default_context()
|
|
143
|
+
self._context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
144
|
+
self._local = threading.local()
|
|
145
|
+
self.sleep: Sleep = sleep or default_sleep
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _validate(base_url: object) -> Tuple[str, str, int, str]:
|
|
149
|
+
parts = None
|
|
150
|
+
if isinstance(base_url, str) and base_url.strip():
|
|
151
|
+
try:
|
|
152
|
+
parts = urlsplit(base_url.strip())
|
|
153
|
+
port = parts.port
|
|
154
|
+
except ValueError:
|
|
155
|
+
parts = None
|
|
156
|
+
if parts is None or parts.scheme not in ("http", "https") or not parts.hostname:
|
|
157
|
+
shown = base_url if isinstance(base_url, str) and base_url.strip() else "empty"
|
|
158
|
+
raise TabaError.client(
|
|
159
|
+
"client.invalid-config",
|
|
160
|
+
"base_url must be an http:// or https:// address (received: %s)" % shown,
|
|
161
|
+
)
|
|
162
|
+
default_port = 443 if parts.scheme == "https" else 80
|
|
163
|
+
return parts.scheme, parts.hostname, port or default_port, parts.path.rstrip("/")
|
|
164
|
+
|
|
165
|
+
# -- connection (one per thread: `http.client` connections are not thread-safe) --------------------
|
|
166
|
+
|
|
167
|
+
def _new_connection(self) -> http.client.HTTPConnection:
|
|
168
|
+
timeout = self._config.connect_timeout
|
|
169
|
+
if self._scheme == "https":
|
|
170
|
+
return http.client.HTTPSConnection(self._host, self._port, timeout=timeout, context=self._context)
|
|
171
|
+
return http.client.HTTPConnection(self._host, self._port, timeout=timeout)
|
|
172
|
+
|
|
173
|
+
def _drop(self) -> None:
|
|
174
|
+
conn = getattr(self._local, "conn", None)
|
|
175
|
+
self._local.conn = None
|
|
176
|
+
if conn is not None:
|
|
177
|
+
try:
|
|
178
|
+
conn.close()
|
|
179
|
+
except OSError:
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
def close(self) -> None:
|
|
183
|
+
"""Closes the calling thread's connection."""
|
|
184
|
+
self._drop()
|
|
185
|
+
|
|
186
|
+
def _url(self, call: ApiCall) -> str:
|
|
187
|
+
url = self._prefix + call.path
|
|
188
|
+
if call.query:
|
|
189
|
+
url += "?" + "&".join("%s=%s" % (quote(k, safe=""), quote(v, safe="")) for k, v in call.query)
|
|
190
|
+
return url
|
|
191
|
+
|
|
192
|
+
def _headers(self, call: ApiCall) -> Dict[str, str]:
|
|
193
|
+
headers = {"Accept": "application/json", "User-Agent": self._config.user_agent}
|
|
194
|
+
if self._config.token:
|
|
195
|
+
headers["Authorization"] = "Bearer " + self._config.token
|
|
196
|
+
if call.request_id:
|
|
197
|
+
headers["X-Request-Id"] = call.request_id
|
|
198
|
+
if call.traceparent:
|
|
199
|
+
headers["traceparent"] = call.traceparent
|
|
200
|
+
if call.body is not None:
|
|
201
|
+
headers["Content-Type"] = call.content_type or "application/json"
|
|
202
|
+
return headers
|
|
203
|
+
|
|
204
|
+
def _once(self, call: ApiCall, timeout: float, cancel: CancelToken) -> Tuple[int, Dict[str, str], bytes]:
|
|
205
|
+
deadline = time.monotonic() + timeout
|
|
206
|
+
for stale_retry in (True, False):
|
|
207
|
+
cancel.raise_if_cancelled()
|
|
208
|
+
conn = getattr(self._local, "conn", None)
|
|
209
|
+
reused = conn is not None and conn.sock is not None
|
|
210
|
+
if conn is None:
|
|
211
|
+
conn = self._new_connection()
|
|
212
|
+
self._local.conn = conn
|
|
213
|
+
|
|
214
|
+
def abort(c: http.client.HTTPConnection = conn) -> None:
|
|
215
|
+
sock = c.sock
|
|
216
|
+
if sock is not None:
|
|
217
|
+
try:
|
|
218
|
+
sock.shutdown(socket.SHUT_RDWR)
|
|
219
|
+
except OSError:
|
|
220
|
+
pass
|
|
221
|
+
# `close()` only marks the socket closed while the response still holds a `makefile`
|
|
222
|
+
# reference, and on Windows `shutdown` does not wake a thread blocked in `recv`: closing
|
|
223
|
+
# the descriptor for real does.
|
|
224
|
+
getattr(socket.socket, "_real_close", socket.socket.close)(sock)
|
|
225
|
+
|
|
226
|
+
cancel._attach(abort)
|
|
227
|
+
sent = False
|
|
228
|
+
try:
|
|
229
|
+
if conn.sock is None:
|
|
230
|
+
conn.timeout = self._config.connect_timeout
|
|
231
|
+
conn.connect()
|
|
232
|
+
remaining = deadline - time.monotonic()
|
|
233
|
+
if remaining <= 0:
|
|
234
|
+
raise socket.timeout()
|
|
235
|
+
conn.sock.settimeout(remaining)
|
|
236
|
+
conn.request(call.method, self._url(call), body=call.body, headers=self._headers(call))
|
|
237
|
+
sent = True
|
|
238
|
+
response = conn.getresponse()
|
|
239
|
+
chunks = []
|
|
240
|
+
while True:
|
|
241
|
+
remaining = deadline - time.monotonic()
|
|
242
|
+
if remaining <= 0:
|
|
243
|
+
raise socket.timeout()
|
|
244
|
+
if conn.sock is not None:
|
|
245
|
+
conn.sock.settimeout(remaining)
|
|
246
|
+
chunk = response.read(65536)
|
|
247
|
+
if not chunk:
|
|
248
|
+
break
|
|
249
|
+
chunks.append(chunk)
|
|
250
|
+
headers = {k.lower(): v for k, v in response.getheaders()}
|
|
251
|
+
if response.will_close:
|
|
252
|
+
self._drop()
|
|
253
|
+
return response.status, headers, b"".join(chunks)
|
|
254
|
+
except Exception as e: # noqa: BLE001 - classified below
|
|
255
|
+
self._drop()
|
|
256
|
+
if cancel.is_set():
|
|
257
|
+
raise Cancelled()
|
|
258
|
+
if isinstance(e, (socket.timeout, TimeoutError)):
|
|
259
|
+
if conn.sock is None and not sent:
|
|
260
|
+
raise TabaError.client("client.network", "timed out connecting to %s" % self._host)
|
|
261
|
+
raise TabaError.client("client.timeout", "timed out waiting for the response")
|
|
262
|
+
error = TabaError.client("client.network", _describe(e))
|
|
263
|
+
if not sent and not reused and _is_connect_failure(e):
|
|
264
|
+
raise _Retry(error)
|
|
265
|
+
# A pooled connection the server closed while idle fails before anything is read.
|
|
266
|
+
stale = reused and stale_retry and isinstance(
|
|
267
|
+
e, (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError)
|
|
268
|
+
)
|
|
269
|
+
if stale:
|
|
270
|
+
continue
|
|
271
|
+
raise error
|
|
272
|
+
finally:
|
|
273
|
+
cancel._detach(abort)
|
|
274
|
+
raise AssertionError("unreachable") # pragma: no cover
|
|
275
|
+
|
|
276
|
+
def send(self, call: ApiCall, cancel: Optional[CancelToken] = None) -> ApiReply:
|
|
277
|
+
"""
|
|
278
|
+
Makes the call, retrying only what the contract says to retry: 429 and 503 (waiting for the
|
|
279
|
+
`Retry-After`, or 1 s, 2 s, 4 s), 502 and 504 only with `retry_gateway`, and a failure to connect
|
|
280
|
+
(refused, DNS). Never a 4xx (except 429), nor a failure after the body was sent, nor a timeout.
|
|
281
|
+
"""
|
|
282
|
+
cancel = cancel or CancelToken()
|
|
283
|
+
timeout = call.timeout if call.timeout is not None else self._config.request_timeout
|
|
284
|
+
maximum = max(1, self._config.max_attempts)
|
|
285
|
+
after_retryable = False
|
|
286
|
+
attempt = 0
|
|
287
|
+
while True:
|
|
288
|
+
attempt += 1
|
|
289
|
+
backoff = float(1 << min(attempt - 1, 6))
|
|
290
|
+
try:
|
|
291
|
+
status, headers, body = self._once(call, timeout, cancel)
|
|
292
|
+
if status < 400:
|
|
293
|
+
recreated = (headers.get("taba-topic-recreated") or "").strip().lower() == "true"
|
|
294
|
+
return ApiReply(status, body, recreated=recreated)
|
|
295
|
+
if after_retryable and any(
|
|
296
|
+
s == status and c == _error_code_of(body) for s, c in call.recover
|
|
297
|
+
):
|
|
298
|
+
return ApiReply(status, body, recovered=True)
|
|
299
|
+
error = _error_from(status, headers, body)
|
|
300
|
+
retry = status in (429, 503) or (status in (502, 504) and call.retry_gateway)
|
|
301
|
+
if not retry or attempt >= maximum:
|
|
302
|
+
raise error
|
|
303
|
+
after_retryable = after_retryable or status == 503
|
|
304
|
+
wait = float(error.retry_after) if error.retry_after is not None else backoff
|
|
305
|
+
except _Retry as r:
|
|
306
|
+
if attempt >= maximum:
|
|
307
|
+
raise r.error
|
|
308
|
+
after_retryable = True
|
|
309
|
+
wait = backoff
|
|
310
|
+
self.sleep(wait, cancel)
|
taba/_version.py
ADDED
taba/aio.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The `asyncio` Taba client.
|
|
3
|
+
|
|
4
|
+
Python's standard library has no asynchronous HTTP client, and the connector has no dependencies, so each call
|
|
5
|
+
runs the synchronous transport in a worker thread (`asyncio.to_thread`). Cancelling the task closes the
|
|
6
|
+
connection of the call in flight, so a long polling does not keep a thread for 30 s.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import json
|
|
11
|
+
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
|
|
12
|
+
|
|
13
|
+
from . import _calls
|
|
14
|
+
from .cancel import CancelToken, Cancelled
|
|
15
|
+
from .client import Client, _lost_or_gone, make_config
|
|
16
|
+
from .errors import TabaError
|
|
17
|
+
from .models import (
|
|
18
|
+
BatchMessage,
|
|
19
|
+
BatchResult,
|
|
20
|
+
Config,
|
|
21
|
+
ConsumeOptions,
|
|
22
|
+
Message,
|
|
23
|
+
MessageContext,
|
|
24
|
+
Page,
|
|
25
|
+
PublishResponse,
|
|
26
|
+
Topic,
|
|
27
|
+
TopicPut,
|
|
28
|
+
TopicRequest,
|
|
29
|
+
TopicUpdate,
|
|
30
|
+
VisibilityResponse,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
Sleep = Callable[[float], Awaitable[None]]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class AsyncMessageContext(MessageContext):
|
|
37
|
+
"""The `MessageContext` of the asynchronous client: `change_visibility` is a coroutine."""
|
|
38
|
+
|
|
39
|
+
async def change_visibility(self, seconds: int) -> None: # type: ignore[override]
|
|
40
|
+
await self._change(seconds)
|
|
41
|
+
self._lease_seconds = seconds
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class AsyncClient:
|
|
45
|
+
"""
|
|
46
|
+
The Taba client API client for `asyncio`: the same nine operations, `subscribe` and the consume with a
|
|
47
|
+
handler, as coroutines. Stop a `subscribe` by cancelling its task (`task.cancel()`, `asyncio.timeout`...).
|
|
48
|
+
|
|
49
|
+
>>> async with AsyncClient("http://localhost:8080", token="...") as client:
|
|
50
|
+
... await client.publish_json("orders", {"order": 9001})
|
|
51
|
+
... for message in await client.consume("orders", wait_time=20):
|
|
52
|
+
... await client.ack("orders", message)
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(self, config: Union[Config, str, None] = None, *, _sleep: Optional[Sleep] = None, **options: Any) -> None:
|
|
56
|
+
self._client = Client(make_config(config, options))
|
|
57
|
+
self._sleep: Sleep = _sleep or asyncio.sleep
|
|
58
|
+
self._serializer = self._client._serializer
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def config(self) -> Config:
|
|
62
|
+
return self._client.config
|
|
63
|
+
|
|
64
|
+
async def close(self) -> None:
|
|
65
|
+
"""Closes the worker-thread connections the client opened (they are closed with the threads)."""
|
|
66
|
+
self._client.close()
|
|
67
|
+
|
|
68
|
+
async def __aenter__(self) -> "AsyncClient":
|
|
69
|
+
return self
|
|
70
|
+
|
|
71
|
+
async def __aexit__(self, *exc_info: Any) -> None:
|
|
72
|
+
await self.close()
|
|
73
|
+
|
|
74
|
+
async def _run(self, function: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
|
75
|
+
token = CancelToken()
|
|
76
|
+
try:
|
|
77
|
+
return await asyncio.to_thread(function, *args, cancel=token, **kwargs)
|
|
78
|
+
except asyncio.CancelledError:
|
|
79
|
+
token.cancel() # closes the connection of the call in flight
|
|
80
|
+
raise
|
|
81
|
+
|
|
82
|
+
# -- topics --------------------------------------------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
async def create_topic(self, request: TopicRequest) -> Topic:
|
|
85
|
+
return await self._run(self._client.create_topic, request)
|
|
86
|
+
|
|
87
|
+
async def get_topic(self, name: str) -> Topic:
|
|
88
|
+
return await self._run(self._client.get_topic, name)
|
|
89
|
+
|
|
90
|
+
async def list_topics(
|
|
91
|
+
self, limit: Optional[int] = None, cursor: Optional[str] = None, name: Optional[str] = None
|
|
92
|
+
) -> Page:
|
|
93
|
+
return await self._run(self._client.list_topics, limit, cursor, name)
|
|
94
|
+
|
|
95
|
+
async def list_all_topics(self, name: Optional[str] = None) -> List[Topic]:
|
|
96
|
+
return await self._run(self._client.list_all_topics, name)
|
|
97
|
+
|
|
98
|
+
async def update_topic(self, name: str, update: TopicUpdate) -> Topic:
|
|
99
|
+
return await self._run(self._client.update_topic, name, update)
|
|
100
|
+
|
|
101
|
+
async def create_or_update_topic(self, request: TopicRequest, *, recreate: Optional[str] = None) -> TopicPut:
|
|
102
|
+
return await self._run(self._client.create_or_update_topic, request, recreate=recreate)
|
|
103
|
+
|
|
104
|
+
async def delete_topic(self, name: str, *, if_empty: bool = False) -> None:
|
|
105
|
+
await self._run(self._client.delete_topic, name, if_empty=if_empty)
|
|
106
|
+
|
|
107
|
+
# -- publishing ----------------------------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
async def publish(self, topic: str, body: Union[bytes, bytearray, memoryview, str, None], **options: Any) -> PublishResponse:
|
|
110
|
+
return await self._run(self._client.publish, topic, body, **options)
|
|
111
|
+
|
|
112
|
+
async def publish_json(self, topic: str, value: Any, **options: Any) -> PublishResponse:
|
|
113
|
+
return await self._run(self._client.publish_json, topic, value, **options)
|
|
114
|
+
|
|
115
|
+
async def publish_raw_json(self, topic: str, text: str, **options: Any) -> PublishResponse:
|
|
116
|
+
return await self._run(self._client.publish_raw_json, topic, text, **options)
|
|
117
|
+
|
|
118
|
+
async def publish_text(self, topic: str, text: str, **options: Any) -> PublishResponse:
|
|
119
|
+
return await self._run(self._client.publish_text, topic, text, **options)
|
|
120
|
+
|
|
121
|
+
async def publish_batch(self, topic: str, messages: List[BatchMessage]) -> List[BatchResult]:
|
|
122
|
+
return await self._run(self._client.publish_batch, topic, messages)
|
|
123
|
+
|
|
124
|
+
# -- consuming -----------------------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
async def consume(self, topic: str, **options: Any) -> List[Message]:
|
|
127
|
+
return await self._run(self._client.consume, topic, **options)
|
|
128
|
+
|
|
129
|
+
async def ack(self, topic: str, message: Union[Message, str], receipt_token: Optional[str] = None) -> None:
|
|
130
|
+
await self._run(self._client.ack, topic, message, receipt_token)
|
|
131
|
+
|
|
132
|
+
async def change_visibility(
|
|
133
|
+
self, topic: str, message: Union[Message, str], seconds: int, receipt_token: Optional[str] = None
|
|
134
|
+
) -> VisibilityResponse:
|
|
135
|
+
return await self._run(self._client.change_visibility, topic, message, seconds, receipt_token)
|
|
136
|
+
|
|
137
|
+
async def ack_batch(self, topic: str, messages: List[Message]) -> List[BatchResult]:
|
|
138
|
+
return await self._run(self._client.ack_batch, topic, messages)
|
|
139
|
+
|
|
140
|
+
async def change_visibility_batch(self, topic: str, messages: List[Message], seconds: int) -> List[BatchResult]:
|
|
141
|
+
return await self._run(self._client.change_visibility_batch, topic, messages, seconds)
|
|
142
|
+
|
|
143
|
+
async def consume_with(self, topic: str, handler: Callable[[MessageContext], Awaitable[None]], **options: Any) -> int:
|
|
144
|
+
"""
|
|
145
|
+
The `Client.consume_with` for `asyncio`: `handler` is a coroutine function. The invisibility is renewed
|
|
146
|
+
by an `asyncio` task while it runs.
|
|
147
|
+
"""
|
|
148
|
+
opts = _calls.handler_defaults(ConsumeOptions(**options))
|
|
149
|
+
handled = 0
|
|
150
|
+
for message in await self._run(self._client._consume, topic, opts):
|
|
151
|
+
context = self._context(topic, opts, message)
|
|
152
|
+
await self._handle(topic, context, handler)
|
|
153
|
+
handled += 1
|
|
154
|
+
return handled
|
|
155
|
+
|
|
156
|
+
async def consume_json(self, topic: str, handler: Callable[[MessageContext], Awaitable[None]], **options: Any) -> int:
|
|
157
|
+
"""`consume_with` with the body already parsed in `context.body` (see `Client.consume_json`)."""
|
|
158
|
+
opts = _calls.handler_defaults(ConsumeOptions(**options))
|
|
159
|
+
handled = 0
|
|
160
|
+
for message in await self._run(self._client._consume, topic, opts):
|
|
161
|
+
try:
|
|
162
|
+
body = self._serializer.loads(message.body_text)
|
|
163
|
+
except ValueError as e:
|
|
164
|
+
raise TabaError.client(
|
|
165
|
+
"client.invalid-response", "body of message %s is not valid JSON: %s" % (message.id, e)
|
|
166
|
+
)
|
|
167
|
+
context = self._context(topic, opts, message)
|
|
168
|
+
context.body = body
|
|
169
|
+
await self._handle(topic, context, handler)
|
|
170
|
+
handled += 1
|
|
171
|
+
return handled
|
|
172
|
+
|
|
173
|
+
def _context(self, topic: str, options: ConsumeOptions, message: Message) -> AsyncMessageContext:
|
|
174
|
+
return AsyncMessageContext(
|
|
175
|
+
message,
|
|
176
|
+
options.visibility_timeout or _calls.HANDLER_VISIBILITY_TIMEOUT,
|
|
177
|
+
lambda seconds: self.change_visibility(topic, message, seconds),
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
async def _handle(self, topic: str, context: AsyncMessageContext, handler: Callable[[Any], Awaitable[None]]) -> None:
|
|
181
|
+
renewal = asyncio.ensure_future(self._keep_alive(topic, context))
|
|
182
|
+
try:
|
|
183
|
+
await handler(context)
|
|
184
|
+
finally:
|
|
185
|
+
renewal.cancel()
|
|
186
|
+
await asyncio.gather(renewal, return_exceptions=True)
|
|
187
|
+
if context.delete:
|
|
188
|
+
try:
|
|
189
|
+
# Shielded: the message was already handled, and acknowledging it must not be cut short.
|
|
190
|
+
await asyncio.shield(self.ack(topic, context.message))
|
|
191
|
+
except TabaError as e:
|
|
192
|
+
if not _lost_or_gone(e):
|
|
193
|
+
raise
|
|
194
|
+
|
|
195
|
+
async def _keep_alive(self, topic: str, context: AsyncMessageContext) -> None:
|
|
196
|
+
"""The second task: renews the invisibility until the handler finishes (it is cancelled). Never raises."""
|
|
197
|
+
while True:
|
|
198
|
+
await self._sleep(_calls.renewal_interval(context.lease_seconds))
|
|
199
|
+
try:
|
|
200
|
+
await self.change_visibility(topic, context.message, context.lease_seconds)
|
|
201
|
+
except TabaError as e:
|
|
202
|
+
if _lost_or_gone(e):
|
|
203
|
+
context._lost = True
|
|
204
|
+
return
|
|
205
|
+
# Transient failure (network, 5xx): tries again on the next round, before the deadline expires.
|
|
206
|
+
|
|
207
|
+
async def subscribe(
|
|
208
|
+
self,
|
|
209
|
+
topic: str,
|
|
210
|
+
handler: Callable[[Message], Awaitable[None]],
|
|
211
|
+
*,
|
|
212
|
+
on_error: Optional[Callable[[Message, Exception], None]] = None,
|
|
213
|
+
**options: Any,
|
|
214
|
+
) -> None:
|
|
215
|
+
"""
|
|
216
|
+
The consume loop (see `Client.subscribe`). It never returns by itself: stop it by cancelling the task,
|
|
217
|
+
which raises `asyncio.CancelledError` as any `asyncio` coroutine does. A message already handled is
|
|
218
|
+
acknowledged even if the task is cancelled right after.
|
|
219
|
+
"""
|
|
220
|
+
poll = _calls.subscribe_defaults(ConsumeOptions(**options))
|
|
221
|
+
while True:
|
|
222
|
+
messages = await self._run(self._client._consume, topic, poll)
|
|
223
|
+
for message in messages:
|
|
224
|
+
handled = False
|
|
225
|
+
try:
|
|
226
|
+
await handler(message)
|
|
227
|
+
handled = True
|
|
228
|
+
except Exception as e: # noqa: BLE001 - the handler is the application's code
|
|
229
|
+
if on_error is not None:
|
|
230
|
+
on_error(message, e)
|
|
231
|
+
if handled:
|
|
232
|
+
try:
|
|
233
|
+
await asyncio.shield(self.ack(topic, message))
|
|
234
|
+
except TabaError as e:
|
|
235
|
+
if not _lost_or_gone(e):
|
|
236
|
+
raise
|
taba/cancel.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Cancellation of a call or of a loop in the synchronous client (the `CancellationToken` of the C# connector)."""
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
from typing import Callable, List
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Cancelled(Exception):
|
|
8
|
+
"""The call was cancelled through a `CancelToken`. It is not a `TabaError`: it is not a failure of the call."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CancelToken:
|
|
12
|
+
"""
|
|
13
|
+
Cancel a `Client` call or a `subscribe` loop from another thread.
|
|
14
|
+
|
|
15
|
+
>>> token = CancelToken()
|
|
16
|
+
>>> threading.Timer(60, token.cancel).start()
|
|
17
|
+
>>> client.subscribe("orders", handler, cancel=token) # returns when cancelled
|
|
18
|
+
|
|
19
|
+
Cancelling closes the connection of the call in flight, so a long polling does not wait for its timeout.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self) -> None:
|
|
23
|
+
self._event = threading.Event()
|
|
24
|
+
self._lock = threading.Lock()
|
|
25
|
+
self._callbacks: List[Callable[[], None]] = []
|
|
26
|
+
|
|
27
|
+
def cancel(self) -> None:
|
|
28
|
+
with self._lock:
|
|
29
|
+
self._event.set()
|
|
30
|
+
callbacks = list(self._callbacks)
|
|
31
|
+
for callback in callbacks:
|
|
32
|
+
try:
|
|
33
|
+
callback()
|
|
34
|
+
except Exception: # closing a connection that is already closed
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
def is_set(self) -> bool:
|
|
38
|
+
return self._event.is_set()
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def cancelled(self) -> bool:
|
|
42
|
+
return self._event.is_set()
|
|
43
|
+
|
|
44
|
+
def wait(self, timeout: float) -> bool:
|
|
45
|
+
"""Waits up to `timeout` seconds; `True` when cancelled."""
|
|
46
|
+
return self._event.wait(timeout)
|
|
47
|
+
|
|
48
|
+
def raise_if_cancelled(self) -> None:
|
|
49
|
+
if self._event.is_set():
|
|
50
|
+
raise Cancelled()
|
|
51
|
+
|
|
52
|
+
def _attach(self, callback: Callable[[], None]) -> None:
|
|
53
|
+
with self._lock:
|
|
54
|
+
self._callbacks.append(callback)
|
|
55
|
+
already = self._event.is_set()
|
|
56
|
+
if already:
|
|
57
|
+
callback()
|
|
58
|
+
|
|
59
|
+
def _detach(self, callback: Callable[[], None]) -> None:
|
|
60
|
+
with self._lock:
|
|
61
|
+
if callback in self._callbacks:
|
|
62
|
+
self._callbacks.remove(callback)
|