camera-ui-rpc 1.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- camera_ui_rpc/__init__.py +56 -0
- camera_ui_rpc/channel.py +285 -0
- camera_ui_rpc/chunking.py +351 -0
- camera_ui_rpc/client.py +3202 -0
- camera_ui_rpc/codec.py +299 -0
- camera_ui_rpc/decorators.py +261 -0
- camera_ui_rpc/errors.py +65 -0
- camera_ui_rpc/executor.py +17 -0
- camera_ui_rpc/handler.py +972 -0
- camera_ui_rpc/patches.py +220 -0
- camera_ui_rpc/py.typed +0 -0
- camera_ui_rpc/service.py +545 -0
- camera_ui_rpc/svcm.py +11 -0
- camera_ui_rpc/types.py +379 -0
- camera_ui_rpc/utils.py +387 -0
- camera_ui_rpc-1.1.2.dist-info/METADATA +36 -0
- camera_ui_rpc-1.1.2.dist-info/RECORD +20 -0
- camera_ui_rpc-1.1.2.dist-info/WHEEL +5 -0
- camera_ui_rpc-1.1.2.dist-info/licenses/LICENSE.md +22 -0
- camera_ui_rpc-1.1.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
from nats.micro.service import ServiceConfig, ServiceInfo, ServiceStats
|
|
4
|
+
|
|
5
|
+
from .channel import Channel, PrivateChannel
|
|
6
|
+
from .client import create_rpc_client
|
|
7
|
+
from .decorators import RPCClass, RPCMethod, RPCNested, RPCProperty
|
|
8
|
+
from .errors import RPCException, RPCTimeoutError
|
|
9
|
+
from .service import RPCService, Service
|
|
10
|
+
from .types import (
|
|
11
|
+
CallbackInvocation,
|
|
12
|
+
CloseHandler,
|
|
13
|
+
ErrorCode,
|
|
14
|
+
NoResponderRetryOptions,
|
|
15
|
+
OrphanProbeOptions,
|
|
16
|
+
ProxyWithClose,
|
|
17
|
+
PullCallbackParams,
|
|
18
|
+
RPCClient,
|
|
19
|
+
RPCClientOptions,
|
|
20
|
+
RPCError,
|
|
21
|
+
)
|
|
22
|
+
from .utils import RPCCallbacks, rpc_callbacks
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"create_rpc_client",
|
|
26
|
+
"Channel",
|
|
27
|
+
"PrivateChannel",
|
|
28
|
+
"RPCException",
|
|
29
|
+
"RPCTimeoutError",
|
|
30
|
+
"ErrorCode",
|
|
31
|
+
"RPCService",
|
|
32
|
+
"Service",
|
|
33
|
+
"RPCClass",
|
|
34
|
+
"RPCMethod",
|
|
35
|
+
"RPCNested",
|
|
36
|
+
"RPCProperty",
|
|
37
|
+
"rpc_callbacks",
|
|
38
|
+
"RPCCallbacks",
|
|
39
|
+
"RPCClient",
|
|
40
|
+
"RPCClientOptions",
|
|
41
|
+
"NoResponderRetryOptions",
|
|
42
|
+
"OrphanProbeOptions",
|
|
43
|
+
"RPCError",
|
|
44
|
+
"ProxyWithClose",
|
|
45
|
+
"CloseHandler",
|
|
46
|
+
"PullCallbackParams",
|
|
47
|
+
"CallbackInvocation",
|
|
48
|
+
"ServiceConfig",
|
|
49
|
+
"ServiceInfo",
|
|
50
|
+
"ServiceStats",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
__version__ = version("camera-ui-rpc")
|
|
55
|
+
except PackageNotFoundError:
|
|
56
|
+
__version__ = "0.0.0"
|
camera_ui_rpc/channel.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import contextlib
|
|
3
|
+
from collections.abc import Awaitable, Callable, Coroutine
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Literal, TypedDict, overload
|
|
5
|
+
|
|
6
|
+
from .utils import is_async_function
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from .client import RPCClient
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ChannelMessage(TypedDict):
|
|
13
|
+
type: Literal["message", "close", "error"]
|
|
14
|
+
data: Any | None
|
|
15
|
+
error: str | None
|
|
16
|
+
sender: str | None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Channel:
|
|
20
|
+
def __init__(self, client: "RPCClient", channel_id: str):
|
|
21
|
+
self._client: RPCClient = client
|
|
22
|
+
self._channel_id: str = channel_id
|
|
23
|
+
self._closed: bool = False
|
|
24
|
+
self._initialized: bool = False
|
|
25
|
+
self._unsubscribe: Callable[[], Coroutine[Any, Any, None]] | None = None
|
|
26
|
+
# handler -> is_async, checked once at registration instead of per message
|
|
27
|
+
self.__handlers: dict[str, dict[Callable[[Any], None | Awaitable[None]], bool]] = {}
|
|
28
|
+
self.__close_handlers: dict[Callable[[], None | Awaitable[None]], bool] = {}
|
|
29
|
+
self.__error_handlers: dict[Callable[[Exception], None | Awaitable[None]], bool] = {}
|
|
30
|
+
# single underscore: RPCClient sets the two clients, PrivateChannel adds subscriptions
|
|
31
|
+
self._subscriptions: list[Callable[[], Coroutine[Any, Any, None]]] = []
|
|
32
|
+
self._isolated_client: RPCClient | None = None
|
|
33
|
+
self._parent_client: RPCClient | None = None
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def is_closed(self) -> bool:
|
|
37
|
+
return self._closed
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def id(self) -> str:
|
|
41
|
+
return self._channel_id
|
|
42
|
+
|
|
43
|
+
async def init(self) -> None:
|
|
44
|
+
if self._initialized:
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
self._initialized = True
|
|
48
|
+
subject = f"channel.{self._channel_id}"
|
|
49
|
+
|
|
50
|
+
async def message_handler(msg: ChannelMessage) -> None:
|
|
51
|
+
if msg["type"] == "message":
|
|
52
|
+
await self._emit("message", msg.get("data"))
|
|
53
|
+
elif msg["type"] == "close":
|
|
54
|
+
await self._handle_close()
|
|
55
|
+
elif msg["type"] == "error":
|
|
56
|
+
await self._handle_error(Exception(msg.get("error", "Channel error")))
|
|
57
|
+
|
|
58
|
+
self._unsubscribe = await self._client.subscribe(subject, message_handler)
|
|
59
|
+
|
|
60
|
+
async def send(self, data: Any) -> None:
|
|
61
|
+
if self._closed:
|
|
62
|
+
raise RuntimeError("Channel is closed")
|
|
63
|
+
|
|
64
|
+
msg: ChannelMessage = {"type": "message", "data": data, "error": None, "sender": None}
|
|
65
|
+
|
|
66
|
+
await self._client.publish(f"channel.{self._channel_id}", msg)
|
|
67
|
+
|
|
68
|
+
async def request(self, data: Any, timeout: int = 5000) -> Any:
|
|
69
|
+
if self._closed:
|
|
70
|
+
raise RuntimeError("Channel is closed")
|
|
71
|
+
|
|
72
|
+
subject = f"channel.{self._channel_id}.request"
|
|
73
|
+
return await self._client.request(subject, data, timeout=timeout)
|
|
74
|
+
|
|
75
|
+
async def on_request(self, handler: Callable[[Any], Any]) -> Callable[[], Coroutine[Any, None, None]]:
|
|
76
|
+
subject = f"channel.{self._channel_id}.request"
|
|
77
|
+
unsubscribe = await self._client.on_request(subject, handler)
|
|
78
|
+
self._subscriptions.append(unsubscribe)
|
|
79
|
+
return unsubscribe
|
|
80
|
+
|
|
81
|
+
@overload
|
|
82
|
+
def on(self, event: Literal["message"], handler: Callable[[Any], None | Awaitable[None]]) -> None: ...
|
|
83
|
+
@overload
|
|
84
|
+
def on(self, event: Literal["close"], handler: Callable[[], None | Awaitable[None]]) -> None: ...
|
|
85
|
+
@overload
|
|
86
|
+
def on(self, event: Literal["error"], handler: Callable[[Exception], None | Awaitable[None]]) -> None: ...
|
|
87
|
+
def on(self, event: str, handler: Callable[..., None | Awaitable[None]]) -> None:
|
|
88
|
+
handler_is_async = is_async_function(handler)
|
|
89
|
+
if event == "close":
|
|
90
|
+
self.__close_handlers[handler] = handler_is_async
|
|
91
|
+
elif event == "error":
|
|
92
|
+
self.__error_handlers[handler] = handler_is_async
|
|
93
|
+
else:
|
|
94
|
+
if event not in self.__handlers:
|
|
95
|
+
self.__handlers[event] = {}
|
|
96
|
+
self.__handlers[event][handler] = handler_is_async
|
|
97
|
+
|
|
98
|
+
@overload
|
|
99
|
+
def off(self, event: Literal["message"], handler: Callable[[Any], None | Awaitable[None]]) -> None: ...
|
|
100
|
+
@overload
|
|
101
|
+
def off(self, event: Literal["close"], handler: Callable[[], None | Awaitable[None]]) -> None: ...
|
|
102
|
+
@overload
|
|
103
|
+
def off(
|
|
104
|
+
self, event: Literal["error"], handler: Callable[[Exception], None | Awaitable[None]]
|
|
105
|
+
) -> None: ...
|
|
106
|
+
def off(self, event: str, handler: Callable[..., None | Awaitable[None]]) -> None:
|
|
107
|
+
if event == "close":
|
|
108
|
+
self.__close_handlers.pop(handler, None)
|
|
109
|
+
elif event == "error":
|
|
110
|
+
self.__error_handlers.pop(handler, None)
|
|
111
|
+
else:
|
|
112
|
+
if event in self.__handlers:
|
|
113
|
+
self.__handlers[event].pop(handler, None)
|
|
114
|
+
|
|
115
|
+
async def close(self) -> None:
|
|
116
|
+
if self._closed:
|
|
117
|
+
return
|
|
118
|
+
|
|
119
|
+
self._closed = True
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
msg: ChannelMessage = {"type": "close", "data": None, "error": None, "sender": None}
|
|
123
|
+
await self._client.publish(f"channel.{self._channel_id}", msg)
|
|
124
|
+
except Exception:
|
|
125
|
+
pass
|
|
126
|
+
|
|
127
|
+
await self._cleanup()
|
|
128
|
+
|
|
129
|
+
async def _emit(self, event: str, data: Any = None) -> None:
|
|
130
|
+
if event in self.__handlers:
|
|
131
|
+
# snapshot, an inline handler may add or remove listeners re-entrantly
|
|
132
|
+
for handler, handler_is_async in list(self.__handlers[event].items()):
|
|
133
|
+
try:
|
|
134
|
+
if handler_is_async:
|
|
135
|
+
await handler(data) # type: ignore[misc]
|
|
136
|
+
else:
|
|
137
|
+
handler(data)
|
|
138
|
+
except Exception as e:
|
|
139
|
+
print(f"Error in channel handler: {e}")
|
|
140
|
+
|
|
141
|
+
async def _handle_close(self) -> None:
|
|
142
|
+
if self._closed:
|
|
143
|
+
return
|
|
144
|
+
|
|
145
|
+
self._closed = True
|
|
146
|
+
|
|
147
|
+
for handler, handler_is_async in list(self.__close_handlers.items()):
|
|
148
|
+
with contextlib.suppress(Exception):
|
|
149
|
+
if handler_is_async:
|
|
150
|
+
await handler() # type: ignore[misc]
|
|
151
|
+
else:
|
|
152
|
+
handler()
|
|
153
|
+
|
|
154
|
+
asyncio.create_task(self._cleanup())
|
|
155
|
+
|
|
156
|
+
async def _handle_error(self, error: Exception) -> None:
|
|
157
|
+
for handler, handler_is_async in list(self.__error_handlers.items()):
|
|
158
|
+
with contextlib.suppress(Exception):
|
|
159
|
+
if handler_is_async:
|
|
160
|
+
await handler(error) # type: ignore[misc]
|
|
161
|
+
else:
|
|
162
|
+
handler(error)
|
|
163
|
+
|
|
164
|
+
async def _cleanup(self) -> None:
|
|
165
|
+
self.__handlers.clear()
|
|
166
|
+
self.__close_handlers.clear()
|
|
167
|
+
self.__error_handlers.clear()
|
|
168
|
+
|
|
169
|
+
subscriptions = list(self._subscriptions)
|
|
170
|
+
self._subscriptions.clear()
|
|
171
|
+
await asyncio.gather(*[unsub() for unsub in subscriptions], return_exceptions=True)
|
|
172
|
+
|
|
173
|
+
if self._unsubscribe:
|
|
174
|
+
with contextlib.suppress(Exception):
|
|
175
|
+
await self._unsubscribe()
|
|
176
|
+
self._unsubscribe = None
|
|
177
|
+
|
|
178
|
+
isolated = self._isolated_client
|
|
179
|
+
if isolated is not None:
|
|
180
|
+
self._isolated_client = None
|
|
181
|
+
parent = self._parent_client
|
|
182
|
+
if parent is not None and isolated in parent.isolated_clients:
|
|
183
|
+
parent.isolated_clients.remove(isolated)
|
|
184
|
+
# shielded: a close from the channel's own message handler cancels the task it runs in
|
|
185
|
+
with contextlib.suppress(Exception):
|
|
186
|
+
await asyncio.shield(isolated.disconnect())
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class PrivateChannel(Channel):
|
|
190
|
+
def __init__(self, client: "RPCClient", channel_id: str, target_client_id: str):
|
|
191
|
+
super().__init__(client, channel_id)
|
|
192
|
+
self.__target_client_id: str = target_client_id
|
|
193
|
+
|
|
194
|
+
# the base client name, without the suffixes isolated connections add
|
|
195
|
+
full_name = client.options.get("name", f"client-{int(asyncio.get_event_loop().time() * 1000)}")
|
|
196
|
+
self.__client_id: str = full_name.split("-channel-")[0].split("-private-")[0]
|
|
197
|
+
self.__remote_client_id: str | None = None
|
|
198
|
+
self._unsubscribe: Callable[[], Coroutine[Any, Any, None]] | None = None
|
|
199
|
+
|
|
200
|
+
@property
|
|
201
|
+
def remote_id(self) -> str | None:
|
|
202
|
+
return self.__remote_client_id
|
|
203
|
+
|
|
204
|
+
async def init(self) -> None:
|
|
205
|
+
if self._initialized:
|
|
206
|
+
return
|
|
207
|
+
|
|
208
|
+
self._initialized = True
|
|
209
|
+
|
|
210
|
+
sorted_ids = sorted([self.__client_id, self.__target_client_id])
|
|
211
|
+
subject = f"channel.private.{self._channel_id}.{'.'.join(sorted_ids)}"
|
|
212
|
+
|
|
213
|
+
async def message_handler(msg: ChannelMessage) -> None:
|
|
214
|
+
if msg.get("sender") == self.__client_id:
|
|
215
|
+
return
|
|
216
|
+
|
|
217
|
+
if msg.get("sender") != self.__target_client_id:
|
|
218
|
+
return
|
|
219
|
+
|
|
220
|
+
if not self.__remote_client_id:
|
|
221
|
+
self.__remote_client_id = msg["sender"]
|
|
222
|
+
|
|
223
|
+
if msg["type"] == "message":
|
|
224
|
+
if isinstance(msg.get("data"), dict):
|
|
225
|
+
data: Any = msg["data"]
|
|
226
|
+
if data.get("__handshake"):
|
|
227
|
+
return
|
|
228
|
+
await self._emit("message", msg.get("data"))
|
|
229
|
+
elif msg["type"] == "close":
|
|
230
|
+
await self._handle_close()
|
|
231
|
+
elif msg["type"] == "error":
|
|
232
|
+
await self._handle_error(Exception(msg.get("error", "Channel error")))
|
|
233
|
+
|
|
234
|
+
self._unsubscribe = await self._client.subscribe(subject, message_handler)
|
|
235
|
+
|
|
236
|
+
with contextlib.suppress(Exception):
|
|
237
|
+
await self._send_raw(
|
|
238
|
+
{
|
|
239
|
+
"type": "message",
|
|
240
|
+
"data": {"__handshake": True},
|
|
241
|
+
"sender": self.__client_id,
|
|
242
|
+
"error": None,
|
|
243
|
+
}
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
async def send(self, data: Any) -> None:
|
|
247
|
+
if self._closed:
|
|
248
|
+
raise RuntimeError("Channel is closed")
|
|
249
|
+
|
|
250
|
+
await self._send_raw({"type": "message", "data": data, "sender": self.__client_id, "error": None})
|
|
251
|
+
|
|
252
|
+
async def request(self, data: Any, timeout: int = 5000) -> Any:
|
|
253
|
+
if self._closed:
|
|
254
|
+
raise RuntimeError("Channel is closed")
|
|
255
|
+
|
|
256
|
+
sorted_ids = sorted([self.__client_id, self.__target_client_id])
|
|
257
|
+
subject = f"channel.private.{self._channel_id}.{'.'.join(sorted_ids)}.request"
|
|
258
|
+
return await self._client.request(subject, data, timeout=timeout)
|
|
259
|
+
|
|
260
|
+
async def on_request(self, handler: Callable[[Any], Any]) -> Callable[[], Coroutine[Any, None, None]]:
|
|
261
|
+
sorted_ids = sorted([self.__client_id, self.__target_client_id])
|
|
262
|
+
subject = f"channel.private.{self._channel_id}.{'.'.join(sorted_ids)}.request"
|
|
263
|
+
unsubscribe = await self._client.on_request(subject, handler)
|
|
264
|
+
self._subscriptions.append(unsubscribe)
|
|
265
|
+
return unsubscribe
|
|
266
|
+
|
|
267
|
+
async def close(self) -> None:
|
|
268
|
+
if self._closed:
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
self._closed = True
|
|
272
|
+
|
|
273
|
+
with contextlib.suppress(Exception):
|
|
274
|
+
await self._send_raw({"type": "close", "sender": self.__client_id, "data": None, "error": None})
|
|
275
|
+
|
|
276
|
+
await self._cleanup()
|
|
277
|
+
|
|
278
|
+
def is_connected_to(self, client_id: str) -> bool:
|
|
279
|
+
return self.__remote_client_id == client_id
|
|
280
|
+
|
|
281
|
+
async def _send_raw(self, msg: ChannelMessage) -> None:
|
|
282
|
+
# must match the subject init() subscribes
|
|
283
|
+
sorted_ids = sorted([self.__client_id, self.__target_client_id])
|
|
284
|
+
subject = f"channel.private.{self._channel_id}.{'.'.join(sorted_ids)}"
|
|
285
|
+
await self._client.publish(subject, msg)
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import time
|
|
3
|
+
from collections.abc import Callable, Generator
|
|
4
|
+
from typing import Any, cast
|
|
5
|
+
|
|
6
|
+
from .codec import decode_message
|
|
7
|
+
from .types import ChunkData
|
|
8
|
+
|
|
9
|
+
DEFAULT_MAX_TRANSFER_SIZE = 1024 * 1024 * 1024
|
|
10
|
+
MAX_CHUNK_SIZE = 64 * 1024 * 1024 # the NATS max_payload ceiling
|
|
11
|
+
MAX_CONCURRENT_TRANSFERS = 32
|
|
12
|
+
CHUNK_RESERVE = 8192 # kept free in every message for headers and the envelope
|
|
13
|
+
MIN_PAYLOAD_LIMIT = 16384 # leaves room for a chunk next to CHUNK_RESERVE
|
|
14
|
+
_MAX_INDEX_DIGITS = 15
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_chunk_index(value: str | None) -> int | None:
|
|
18
|
+
if not value or len(value) > _MAX_INDEX_DIGITS or not value.isascii() or not value.isdigit():
|
|
19
|
+
return None
|
|
20
|
+
return int(value)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _as_int(value: Any) -> int | None:
|
|
24
|
+
# msgpackr encodes large numbers as float64, an integral float is still a valid size
|
|
25
|
+
if type(value) is int:
|
|
26
|
+
return value
|
|
27
|
+
if type(value) is float and value.is_integer():
|
|
28
|
+
return int(value)
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _header_problem(
|
|
33
|
+
total_chunks: Any, total_size: Any, chunk_size: Any, max_transfer_size: int
|
|
34
|
+
) -> str | None:
|
|
35
|
+
chunks = _as_int(total_chunks)
|
|
36
|
+
size = _as_int(total_size)
|
|
37
|
+
step = _as_int(chunk_size)
|
|
38
|
+
if step is None or not 0 < step <= MAX_CHUNK_SIZE:
|
|
39
|
+
return f"invalid chunkSize {chunk_size!r}"
|
|
40
|
+
if size is None or not 0 < size <= max_transfer_size:
|
|
41
|
+
return f"invalid totalSize {total_size!r} (limit {max_transfer_size})"
|
|
42
|
+
if chunks is None or chunks != -(-size // step):
|
|
43
|
+
return f"totalChunks {total_chunks!r} does not match totalSize {size} and chunkSize {step}"
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TransferBudget:
|
|
48
|
+
def __init__(self, limit: int | None = None) -> None:
|
|
49
|
+
self.limit: int = limit or DEFAULT_MAX_TRANSFER_SIZE
|
|
50
|
+
self.used: int = 0
|
|
51
|
+
|
|
52
|
+
def reserve(self, size: int) -> bool:
|
|
53
|
+
if self.used + size > self.limit:
|
|
54
|
+
return False
|
|
55
|
+
self.used += size
|
|
56
|
+
return True
|
|
57
|
+
|
|
58
|
+
def release(self, size: int) -> None:
|
|
59
|
+
self.used = max(0, self.used - size)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def create_chunks(encoded: bytes, chunk_id: str, max_chunk_size: int) -> Generator[ChunkData, None, None]:
|
|
63
|
+
total_size = len(encoded)
|
|
64
|
+
if total_size == 0:
|
|
65
|
+
return
|
|
66
|
+
|
|
67
|
+
total_chunks = (total_size + max_chunk_size - 1) // max_chunk_size
|
|
68
|
+
|
|
69
|
+
mv = memoryview(encoded)
|
|
70
|
+
|
|
71
|
+
for i in range(total_chunks):
|
|
72
|
+
start = i * max_chunk_size
|
|
73
|
+
end = min(start + max_chunk_size, total_size)
|
|
74
|
+
|
|
75
|
+
chunk_view = mv[start:end]
|
|
76
|
+
|
|
77
|
+
# the only copy, nats-py needs bytes
|
|
78
|
+
yield {"transferId": chunk_id, "index": i, "data": bytes(chunk_view)}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class ChunkAssembler:
|
|
82
|
+
def __init__(self, chunk_id: str):
|
|
83
|
+
self.id: str = chunk_id
|
|
84
|
+
self.chunks_received: set[int] = set()
|
|
85
|
+
self.total_chunks: int | None = None
|
|
86
|
+
self.total_size: int = 0
|
|
87
|
+
self.buffer: bytearray | None = None
|
|
88
|
+
self.temp_chunks: dict[int, bytes] = {}
|
|
89
|
+
self.chunk_size: int | None = None
|
|
90
|
+
# set by ChunkingManager, the buffer is only allocated within it
|
|
91
|
+
self.budget: TransferBudget | None = None
|
|
92
|
+
self.reserved: int = 0
|
|
93
|
+
self._preallocate = False
|
|
94
|
+
|
|
95
|
+
def set_expected_chunks(
|
|
96
|
+
self, count: int, total_size: int | None = None, chunk_size: int | None = None
|
|
97
|
+
) -> None:
|
|
98
|
+
self.total_chunks = count
|
|
99
|
+
self.chunk_size = chunk_size
|
|
100
|
+
if total_size is not None and total_size > 0:
|
|
101
|
+
self.total_size = total_size
|
|
102
|
+
self._preallocate = True
|
|
103
|
+
|
|
104
|
+
def add_chunk(self, chunk: dict[str, Any]) -> bool:
|
|
105
|
+
if chunk["transferId"] != self.id:
|
|
106
|
+
raise ValueError(f"Chunk ID mismatch: expected {self.id}, got {chunk['transferId']}")
|
|
107
|
+
|
|
108
|
+
chunk_index = chunk["index"]
|
|
109
|
+
chunk_data = chunk["data"]
|
|
110
|
+
chunk_size = len(chunk_data)
|
|
111
|
+
|
|
112
|
+
if (
|
|
113
|
+
type(chunk_index) is not int
|
|
114
|
+
or chunk_index < 0
|
|
115
|
+
or (self.total_chunks is not None and chunk_index >= self.total_chunks)
|
|
116
|
+
):
|
|
117
|
+
raise ValueError(f"Invalid chunk index {chunk_index!r} for {self.total_chunks} chunks")
|
|
118
|
+
|
|
119
|
+
if self.chunk_size is not None and self.total_chunks is not None and self.total_size > 0:
|
|
120
|
+
last = self.total_chunks - 1
|
|
121
|
+
expected = self.chunk_size if chunk_index < last else self.total_size - last * self.chunk_size
|
|
122
|
+
if chunk_size != expected:
|
|
123
|
+
raise ValueError(f"Chunk {chunk_index} has {chunk_size} bytes, expected {expected}")
|
|
124
|
+
|
|
125
|
+
if chunk_index in self.chunks_received:
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
# chunk 0 sets the chunk size, senders differ in it
|
|
129
|
+
if self.chunk_size is None and chunk_index == 0 and self.total_chunks and self.total_chunks > 1:
|
|
130
|
+
self.chunk_size = chunk_size
|
|
131
|
+
|
|
132
|
+
effective_chunk_size = self.chunk_size if self.chunk_size else chunk_size
|
|
133
|
+
|
|
134
|
+
if self.buffer is None and self._preallocate:
|
|
135
|
+
# not on the header: bytearray zero-fills, a bare header would commit totalSize
|
|
136
|
+
if self.budget is not None:
|
|
137
|
+
if not self.budget.reserve(self.total_size):
|
|
138
|
+
raise ValueError(f"Transfers in progress would hold more than {self.budget.limit} bytes")
|
|
139
|
+
self.reserved = self.total_size
|
|
140
|
+
self._preallocate = False
|
|
141
|
+
try:
|
|
142
|
+
self.buffer = bytearray(self.total_size)
|
|
143
|
+
except MemoryError:
|
|
144
|
+
print(f"MemoryError: Failed to allocate {self.total_size} bytes for chunk assembly.")
|
|
145
|
+
|
|
146
|
+
if self.buffer is not None:
|
|
147
|
+
offset = chunk_index * effective_chunk_size
|
|
148
|
+
if offset + chunk_size <= self.total_size:
|
|
149
|
+
self.buffer[offset : offset + chunk_size] = chunk_data
|
|
150
|
+
else:
|
|
151
|
+
# only the last chunk may be smaller
|
|
152
|
+
remaining = self.total_size - offset
|
|
153
|
+
if remaining > 0:
|
|
154
|
+
self.buffer[offset:] = chunk_data[:remaining]
|
|
155
|
+
else:
|
|
156
|
+
self.temp_chunks[chunk_index] = chunk_data
|
|
157
|
+
|
|
158
|
+
self.chunks_received.add(chunk_index)
|
|
159
|
+
|
|
160
|
+
if chunk.get("isLast", False):
|
|
161
|
+
self.total_chunks = chunk_index + 1
|
|
162
|
+
|
|
163
|
+
return self.is_complete()
|
|
164
|
+
|
|
165
|
+
def is_complete(self) -> bool:
|
|
166
|
+
if self.total_chunks is None:
|
|
167
|
+
return False
|
|
168
|
+
return len(self.chunks_received) == self.total_chunks
|
|
169
|
+
|
|
170
|
+
def get_data(self) -> Any:
|
|
171
|
+
if not self.is_complete():
|
|
172
|
+
raise RuntimeError("Cannot get data: Not all chunks have been received.")
|
|
173
|
+
|
|
174
|
+
if self.buffer is not None:
|
|
175
|
+
# chunked payloads are full wire messages, a CUIB frame yields memoryview slices that keep the buffer alive
|
|
176
|
+
data_view = memoryview(self.buffer)
|
|
177
|
+
return decode_message(data_view)
|
|
178
|
+
else:
|
|
179
|
+
total_size = sum(len(self.temp_chunks[i]) for i in range(self.total_chunks or 0))
|
|
180
|
+
combined = bytearray(total_size)
|
|
181
|
+
offset = 0
|
|
182
|
+
for i in range(self.total_chunks or 0):
|
|
183
|
+
chunk = self.temp_chunks[i]
|
|
184
|
+
chunk_len = len(chunk)
|
|
185
|
+
combined[offset : offset + chunk_len] = chunk
|
|
186
|
+
offset += chunk_len
|
|
187
|
+
return decode_message(combined)
|
|
188
|
+
|
|
189
|
+
def release(self) -> None:
|
|
190
|
+
if self.budget is not None and self.reserved:
|
|
191
|
+
self.budget.release(self.reserved)
|
|
192
|
+
self.reserved = 0
|
|
193
|
+
|
|
194
|
+
def get_progress(self) -> dict[str, int | float]:
|
|
195
|
+
received = len(self.chunks_received)
|
|
196
|
+
total = self.total_chunks or 0
|
|
197
|
+
percentage = (received / total * 100) if total > 0 else 0
|
|
198
|
+
return {"received": received, "total": total, "percentage": percentage}
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class ChunkingManager:
|
|
202
|
+
# an incomplete transfer holds its full buffer until this long after its last chunk
|
|
203
|
+
_STALE_TRANSFER_TTL: float = 30.0
|
|
204
|
+
# sweeping on every chunk of a large transfer is pure overhead
|
|
205
|
+
_SWEEP_INTERVAL: float = 1.0
|
|
206
|
+
|
|
207
|
+
def __init__(
|
|
208
|
+
self,
|
|
209
|
+
max_transfer_size: int | None = None,
|
|
210
|
+
max_transfers: float = MAX_CONCURRENT_TRANSFERS,
|
|
211
|
+
budget: TransferBudget | None = None,
|
|
212
|
+
) -> None:
|
|
213
|
+
self.assemblers: dict[str, ChunkAssembler] = {}
|
|
214
|
+
self.completed_callbacks: dict[str, Callable[[Any], None]] = {}
|
|
215
|
+
self.error_callbacks: dict[str, Callable[[Exception], None]] = {}
|
|
216
|
+
self.max_transfer_size: int = max_transfer_size or DEFAULT_MAX_TRANSFER_SIZE
|
|
217
|
+
self.max_transfers: float = max_transfers
|
|
218
|
+
self.budget: TransferBudget = budget if budget is not None else TransferBudget(self.max_transfer_size)
|
|
219
|
+
self._last_activity: dict[str, float] = {}
|
|
220
|
+
self._last_sweep: float = 0.0
|
|
221
|
+
self._timer: asyncio.TimerHandle | None = None
|
|
222
|
+
|
|
223
|
+
def start_receiving(
|
|
224
|
+
self,
|
|
225
|
+
transfer_id: str,
|
|
226
|
+
total_chunks: Any,
|
|
227
|
+
on_complete: Callable[[Any], None],
|
|
228
|
+
on_error: Callable[[Exception], None],
|
|
229
|
+
total_size: Any = None,
|
|
230
|
+
chunk_size: Any = None,
|
|
231
|
+
) -> None:
|
|
232
|
+
self._sweep_stale()
|
|
233
|
+
problem = _header_problem(total_chunks, total_size, chunk_size, self.max_transfer_size)
|
|
234
|
+
if problem is None and transfer_id in self.assemblers:
|
|
235
|
+
self._drop(transfer_id)
|
|
236
|
+
if problem is None and len(self.assemblers) >= self.max_transfers:
|
|
237
|
+
problem = f"{self.max_transfers} transfers already open"
|
|
238
|
+
if problem is not None:
|
|
239
|
+
on_error(ValueError(f"Rejected chunked transfer {transfer_id}: {problem}"))
|
|
240
|
+
return
|
|
241
|
+
|
|
242
|
+
assembler = ChunkAssembler(transfer_id)
|
|
243
|
+
assembler.set_expected_chunks(
|
|
244
|
+
cast(int, _as_int(total_chunks)), _as_int(total_size), _as_int(chunk_size)
|
|
245
|
+
)
|
|
246
|
+
assembler.budget = self.budget
|
|
247
|
+
self.assemblers[transfer_id] = assembler
|
|
248
|
+
self.completed_callbacks[transfer_id] = on_complete
|
|
249
|
+
self.error_callbacks[transfer_id] = on_error
|
|
250
|
+
self._last_activity[transfer_id] = time.monotonic()
|
|
251
|
+
self._arm()
|
|
252
|
+
|
|
253
|
+
def process_chunk(self, chunk: dict[str, Any]) -> None:
|
|
254
|
+
self._sweep_stale()
|
|
255
|
+
|
|
256
|
+
transfer_id = chunk.get("id") or chunk.get("transferId")
|
|
257
|
+
if not transfer_id:
|
|
258
|
+
return
|
|
259
|
+
|
|
260
|
+
assembler = self.assemblers.get(transfer_id)
|
|
261
|
+
if not assembler:
|
|
262
|
+
return
|
|
263
|
+
|
|
264
|
+
self._last_activity[transfer_id] = time.monotonic()
|
|
265
|
+
|
|
266
|
+
try:
|
|
267
|
+
# senders name the keys differently, normalized in place
|
|
268
|
+
if "transferId" not in chunk:
|
|
269
|
+
chunk["transferId"] = transfer_id
|
|
270
|
+
if "index" not in chunk and "chunkIndex" in chunk:
|
|
271
|
+
chunk["index"] = chunk["chunkIndex"]
|
|
272
|
+
|
|
273
|
+
if assembler.add_chunk(chunk):
|
|
274
|
+
data = assembler.get_data()
|
|
275
|
+
callback = self.completed_callbacks.get(transfer_id)
|
|
276
|
+
self._drop(transfer_id)
|
|
277
|
+
|
|
278
|
+
if callback:
|
|
279
|
+
callback(data)
|
|
280
|
+
|
|
281
|
+
except Exception as e:
|
|
282
|
+
self._handle_error(transfer_id, e)
|
|
283
|
+
|
|
284
|
+
def cancel(self, transfer_id: str) -> None:
|
|
285
|
+
self._handle_error(transfer_id, RuntimeError("Transfer cancelled"))
|
|
286
|
+
|
|
287
|
+
def discard(self, transfer_id: str) -> None:
|
|
288
|
+
self._drop(transfer_id)
|
|
289
|
+
|
|
290
|
+
def get_progress(self, transfer_id: str) -> dict[str, int | float] | None:
|
|
291
|
+
assembler = self.assemblers.get(transfer_id)
|
|
292
|
+
return assembler.get_progress() if assembler else None
|
|
293
|
+
|
|
294
|
+
def close(self) -> None:
|
|
295
|
+
for assembler in self.assemblers.values():
|
|
296
|
+
assembler.release()
|
|
297
|
+
self.assemblers.clear()
|
|
298
|
+
self.completed_callbacks.clear()
|
|
299
|
+
self.error_callbacks.clear()
|
|
300
|
+
self._last_activity.clear()
|
|
301
|
+
self._cancel_timer()
|
|
302
|
+
|
|
303
|
+
def _cancel_timer(self) -> None:
|
|
304
|
+
if self._timer is not None:
|
|
305
|
+
self._timer.cancel()
|
|
306
|
+
self._timer = None
|
|
307
|
+
|
|
308
|
+
def _arm(self) -> None:
|
|
309
|
+
if self._timer is not None or not self._last_activity:
|
|
310
|
+
return
|
|
311
|
+
try:
|
|
312
|
+
loop = asyncio.get_running_loop()
|
|
313
|
+
except RuntimeError:
|
|
314
|
+
return
|
|
315
|
+
due = min(self._last_activity.values()) + self._STALE_TRANSFER_TTL - time.monotonic()
|
|
316
|
+
self._timer = loop.call_later(max(0.0, due) + 0.1, self._on_timer)
|
|
317
|
+
|
|
318
|
+
def _on_timer(self) -> None:
|
|
319
|
+
self._timer = None
|
|
320
|
+
self._last_sweep = 0.0
|
|
321
|
+
self._sweep_stale()
|
|
322
|
+
self._arm()
|
|
323
|
+
|
|
324
|
+
def _sweep_stale(self) -> None:
|
|
325
|
+
if not self._last_activity:
|
|
326
|
+
return
|
|
327
|
+
now = time.monotonic()
|
|
328
|
+
if now - self._last_sweep < self._SWEEP_INTERVAL:
|
|
329
|
+
return
|
|
330
|
+
self._last_sweep = now
|
|
331
|
+
cutoff = now - self._STALE_TRANSFER_TTL
|
|
332
|
+
stale = [tid for tid, ts in self._last_activity.items() if ts < cutoff]
|
|
333
|
+
for transfer_id in stale:
|
|
334
|
+
self._handle_error(transfer_id, RuntimeError("Transfer timed out"))
|
|
335
|
+
|
|
336
|
+
def _handle_error(self, transfer_id: str, error: Exception) -> None:
|
|
337
|
+
error_callback = self.error_callbacks.get(transfer_id)
|
|
338
|
+
self._drop(transfer_id)
|
|
339
|
+
|
|
340
|
+
if error_callback:
|
|
341
|
+
error_callback(error)
|
|
342
|
+
|
|
343
|
+
def _drop(self, transfer_id: str) -> None:
|
|
344
|
+
assembler = self.assemblers.pop(transfer_id, None)
|
|
345
|
+
if assembler is not None:
|
|
346
|
+
assembler.release()
|
|
347
|
+
self.completed_callbacks.pop(transfer_id, None)
|
|
348
|
+
self.error_callbacks.pop(transfer_id, None)
|
|
349
|
+
self._last_activity.pop(transfer_id, None)
|
|
350
|
+
if not self._last_activity:
|
|
351
|
+
self._cancel_timer()
|