loop-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loop_sdk/__init__.py +84 -0
- loop_sdk/gen/__init__.py +0 -0
- loop_sdk/gen/__init__.pyi +1 -0
- loop_sdk/gen/v1/__init__.py +0 -0
- loop_sdk/gen/v1/__init__.pyi +1 -0
- loop_sdk/gen/v1/source_ingest_pb2.py +132 -0
- loop_sdk/gen/v1/source_ingest_pb2.pyi +466 -0
- loop_sdk/gen/v1/source_ingest_pb2_grpc.py +288 -0
- loop_sdk/source/__init__.py +0 -0
- loop_sdk/source/client.py +123 -0
- loop_sdk/source/consumer.py +54 -0
- loop_sdk/source/domain/__init__.py +0 -0
- loop_sdk/source/domain/config.py +61 -0
- loop_sdk/source/domain/frame.py +108 -0
- loop_sdk/source/domain/schema.py +193 -0
- loop_sdk/source/domain/source_exception.py +48 -0
- loop_sdk/source/domain/source_kind.py +18 -0
- loop_sdk/source/domain/stats.py +22 -0
- loop_sdk/source/outbound/__init__.py +0 -0
- loop_sdk/source/outbound/frame_queue.py +85 -0
- loop_sdk/source/outbound/grpc_source_client.py +360 -0
- loop_sdk/source/outbound/grpc_source_reader.py +82 -0
- loop_sdk/source/outbound/proto_mapping.py +239 -0
- loop_sdk/source/port/__init__.py +0 -0
- loop_sdk/source/port/source_client.py +49 -0
- loop_sdk/source/port/source_reader.py +34 -0
- loop_sdk/source/robot_step_sender.py +410 -0
- loop_sdk/source/service/__init__.py +0 -0
- loop_sdk/source/service/producer_session.py +81 -0
- loop_sdk/source/setup/__init__.py +0 -0
- loop_sdk/source/setup/config.py +30 -0
- loop_sdk-0.1.0.dist-info/METADATA +9 -0
- loop_sdk-0.1.0.dist-info/RECORD +34 -0
- loop_sdk-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""gRPC implementation of the source client (synchronous + background threads).
|
|
2
|
+
|
|
3
|
+
Threading model (see ``docs/spec_discrepancies.md`` items 3, 12, 14):
|
|
4
|
+
|
|
5
|
+
* A **supervisor** thread owns the connect/reconnect loop. Each iteration runs
|
|
6
|
+
one session and, on transport failure, backs off and reconnects (re-sending
|
|
7
|
+
``ClientReady`` and re-answering ``Describe``).
|
|
8
|
+
* Within a session, the **control** plane runs on the supervisor thread itself:
|
|
9
|
+
it sends ``ClientEvent``s from an outbound event queue and reads
|
|
10
|
+
``RecorderCommand``s, answering ``Describe`` from the cached schema and acting
|
|
11
|
+
on ``Open`` / ``Close`` / ``Shutdown``. The first ``Open`` records the session
|
|
12
|
+
id and unblocks the sender; ``Close`` stops the sender for that session while
|
|
13
|
+
leaving the control connection up; ``Shutdown`` tears the connection down.
|
|
14
|
+
* A **sender** thread runs ``StreamSamples``, draining the frame queue and
|
|
15
|
+
mapping each frame to a one-sample ``SampleBatch``. It starts once the Source Bus
|
|
16
|
+
opens the session and retires on ``Close`` or disconnect.
|
|
17
|
+
|
|
18
|
+
``submit()`` only touches the lock-free-ish bounded queue, so the customer's
|
|
19
|
+
control loop never blocks on the network. Transport failures are translated into
|
|
20
|
+
domain ``SourceError`` subclasses at this boundary.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import queue
|
|
26
|
+
import threading
|
|
27
|
+
from collections.abc import Iterator
|
|
28
|
+
|
|
29
|
+
import grpc
|
|
30
|
+
|
|
31
|
+
from loop_sdk.gen.v1 import source_ingest_pb2 as pb2
|
|
32
|
+
from loop_sdk.gen.v1 import source_ingest_pb2_grpc as pb2_grpc
|
|
33
|
+
from loop_sdk.source.domain.config import OnSelectRobotConfig, RobotConfig
|
|
34
|
+
from loop_sdk.source.domain.frame import FrameSample
|
|
35
|
+
from loop_sdk.source.domain.schema import SourceSchema
|
|
36
|
+
from loop_sdk.source.domain.source_exception import NotConnectedError
|
|
37
|
+
from loop_sdk.source.domain.stats import SendStats
|
|
38
|
+
from loop_sdk.source.outbound import proto_mapping
|
|
39
|
+
from loop_sdk.source.outbound.frame_queue import FrameQueue
|
|
40
|
+
from loop_sdk.source.port.source_client import SourceClient
|
|
41
|
+
|
|
42
|
+
# gRPC keepalive so a silent half-open connection is detected and recycled.
|
|
43
|
+
_CHANNEL_OPTIONS = (
|
|
44
|
+
("grpc.keepalive_time_ms", 10_000),
|
|
45
|
+
("grpc.keepalive_timeout_ms", 5_000),
|
|
46
|
+
("grpc.keepalive_permit_without_calls", 1),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class _Stats:
|
|
51
|
+
"""Mutable, lock-guarded counters projected into the immutable ``SendStats``."""
|
|
52
|
+
|
|
53
|
+
def __init__(self) -> None:
|
|
54
|
+
self._lock = threading.Lock()
|
|
55
|
+
self.enqueued = 0
|
|
56
|
+
self.sent = 0
|
|
57
|
+
self.failed = 0
|
|
58
|
+
self.reconnects = 0
|
|
59
|
+
self.connected = False
|
|
60
|
+
|
|
61
|
+
def snapshot(self, dropped: int) -> SendStats:
|
|
62
|
+
with self._lock:
|
|
63
|
+
return SendStats(
|
|
64
|
+
enqueued=self.enqueued,
|
|
65
|
+
sent=self.sent,
|
|
66
|
+
failed=self.failed,
|
|
67
|
+
dropped=dropped,
|
|
68
|
+
reconnects=self.reconnects,
|
|
69
|
+
connected=self.connected,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def incr(self, field: str, by: int = 1) -> None:
|
|
73
|
+
with self._lock:
|
|
74
|
+
setattr(self, field, getattr(self, field) + by)
|
|
75
|
+
|
|
76
|
+
def set_connected(self, value: bool) -> None:
|
|
77
|
+
with self._lock:
|
|
78
|
+
self.connected = value
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class GrpcSourceClient(SourceClient):
|
|
82
|
+
"""Drive one Source Bus connection over synchronous gRPC with background threads."""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
loop_addr: str,
|
|
87
|
+
client_id: str,
|
|
88
|
+
version: str,
|
|
89
|
+
reconnect_backoff_seconds: float,
|
|
90
|
+
reconnect_backoff_max_seconds: float,
|
|
91
|
+
on_select: OnSelectRobotConfig | None,
|
|
92
|
+
) -> None:
|
|
93
|
+
self._loop_addr = loop_addr
|
|
94
|
+
self._client_id = client_id
|
|
95
|
+
self._version = version
|
|
96
|
+
self._reconnect_backoff_seconds = reconnect_backoff_seconds
|
|
97
|
+
self._reconnect_backoff_max_seconds = reconnect_backoff_max_seconds
|
|
98
|
+
self._on_select = on_select
|
|
99
|
+
|
|
100
|
+
self._schema: SourceSchema | None = None
|
|
101
|
+
self._frames = FrameQueue()
|
|
102
|
+
self._stats = _Stats()
|
|
103
|
+
self._closed = threading.Event()
|
|
104
|
+
self._supervisor: threading.Thread | None = None
|
|
105
|
+
# The in-flight Connect call, cancelled on shutdown/close to unblock the
|
|
106
|
+
# response iterator (closing only the request side does not end it).
|
|
107
|
+
self._active_call: grpc.Call | None = None
|
|
108
|
+
|
|
109
|
+
# Per-session state, reset on each (re)connect.
|
|
110
|
+
self._events: queue.SimpleQueue[pb2.ClientEvent | None] = queue.SimpleQueue()
|
|
111
|
+
self._opened = False
|
|
112
|
+
self._session_id = ""
|
|
113
|
+
self._session_ready = threading.Event()
|
|
114
|
+
self._session_ended = threading.Event()
|
|
115
|
+
|
|
116
|
+
# -- SourceClient contract -------------------------------------------------
|
|
117
|
+
|
|
118
|
+
def start(self, schema: SourceSchema) -> None:
|
|
119
|
+
self._schema = schema
|
|
120
|
+
self._supervisor = threading.Thread(target=self._supervise, name="source-supervisor", daemon=True)
|
|
121
|
+
self._supervisor.start()
|
|
122
|
+
|
|
123
|
+
def submit(self, frame: FrameSample) -> None:
|
|
124
|
+
if self._schema is None:
|
|
125
|
+
raise NotConnectedError("start() must be called before submit()")
|
|
126
|
+
self._stats.incr("enqueued")
|
|
127
|
+
self._frames.put(frame)
|
|
128
|
+
|
|
129
|
+
def report_error(self, source_id: str, code: str, message: str = "") -> None:
|
|
130
|
+
# Queue a control-plane ClientError; the server turns it into a per-source
|
|
131
|
+
# SourceEvent(FAILED). Best-effort: if no session is live the event rides
|
|
132
|
+
# the next connection or is dropped on reset. Never blocks the caller.
|
|
133
|
+
if self._schema is None:
|
|
134
|
+
raise NotConnectedError("start() must be called before report_error()")
|
|
135
|
+
self._events.put(_error_event(self._client_id, source_id, code, message))
|
|
136
|
+
|
|
137
|
+
def stats(self) -> SendStats:
|
|
138
|
+
return self._stats.snapshot(self._frames.dropped)
|
|
139
|
+
|
|
140
|
+
def close(self) -> None:
|
|
141
|
+
self._closed.set()
|
|
142
|
+
self._frames.close()
|
|
143
|
+
self._events.put(None)
|
|
144
|
+
self._cancel_active_call()
|
|
145
|
+
supervisor = self._supervisor
|
|
146
|
+
if supervisor is not None:
|
|
147
|
+
supervisor.join(timeout=5.0)
|
|
148
|
+
|
|
149
|
+
# -- supervisor / reconnect loop --------------------------------------------
|
|
150
|
+
|
|
151
|
+
def _supervise(self) -> None:
|
|
152
|
+
backoff = self._reconnect_backoff_seconds
|
|
153
|
+
first_attempt = True
|
|
154
|
+
while not self._closed.is_set():
|
|
155
|
+
if not first_attempt:
|
|
156
|
+
self._stats.incr("reconnects")
|
|
157
|
+
first_attempt = False
|
|
158
|
+
reached_live = self._run_session()
|
|
159
|
+
if self._closed.is_set():
|
|
160
|
+
return
|
|
161
|
+
if reached_live:
|
|
162
|
+
backoff = self._reconnect_backoff_seconds
|
|
163
|
+
self._sleep_backoff(backoff)
|
|
164
|
+
backoff = min(backoff * 2, self._reconnect_backoff_max_seconds)
|
|
165
|
+
|
|
166
|
+
def _run_session(self) -> bool:
|
|
167
|
+
"""Run one connection. Return whether it reached a live (post-ready) state.
|
|
168
|
+
|
|
169
|
+
"Live" means the server actually answered (the first ``RecorderCommand``
|
|
170
|
+
arrived), not merely that the channel opened — a connection that opens
|
|
171
|
+
and instantly drops must NOT reset the reconnect backoff.
|
|
172
|
+
"""
|
|
173
|
+
self._reset_session()
|
|
174
|
+
sender: threading.Thread | None = None
|
|
175
|
+
reached_live = False
|
|
176
|
+
try:
|
|
177
|
+
channel = grpc.insecure_channel(self._loop_addr, options=_CHANNEL_OPTIONS)
|
|
178
|
+
with channel:
|
|
179
|
+
stub = pb2_grpc.SourceIngestServiceStub(channel)
|
|
180
|
+
self._events.put(_ready_event(self._client_id, self._version))
|
|
181
|
+
sender = threading.Thread(target=self._run_sender, args=(stub,), name="source-sender", daemon=True)
|
|
182
|
+
sender.start()
|
|
183
|
+
call = stub.Connect(self._drain_events())
|
|
184
|
+
self._active_call = call
|
|
185
|
+
self._stats.set_connected(True)
|
|
186
|
+
for command in call:
|
|
187
|
+
reached_live = True
|
|
188
|
+
self._handle_command(command)
|
|
189
|
+
except grpc.RpcError:
|
|
190
|
+
pass
|
|
191
|
+
finally:
|
|
192
|
+
self._active_call = None
|
|
193
|
+
self._stats.set_connected(False)
|
|
194
|
+
self._end_session(sender)
|
|
195
|
+
return reached_live
|
|
196
|
+
|
|
197
|
+
def _reset_session(self) -> None:
|
|
198
|
+
self._events = queue.SimpleQueue()
|
|
199
|
+
self._opened = False
|
|
200
|
+
self._session_id = ""
|
|
201
|
+
self._session_ready.clear()
|
|
202
|
+
self._session_ended.clear()
|
|
203
|
+
self._frames.reset_wake()
|
|
204
|
+
|
|
205
|
+
def _end_session(self, sender: threading.Thread | None) -> None:
|
|
206
|
+
# Unblock the sender's StreamSamples iterator so the thread can exit
|
|
207
|
+
# without closing the shared frame queue (which survives reconnects).
|
|
208
|
+
self._session_ended.set()
|
|
209
|
+
self._session_ready.set()
|
|
210
|
+
self._events.put(None)
|
|
211
|
+
self._frames.wake()
|
|
212
|
+
if sender is not None:
|
|
213
|
+
sender.join(timeout=2.0)
|
|
214
|
+
|
|
215
|
+
def _sleep_backoff(self, seconds: float) -> None:
|
|
216
|
+
# Wait, but wake immediately if close() is called.
|
|
217
|
+
self._closed.wait(timeout=seconds)
|
|
218
|
+
|
|
219
|
+
def _cancel_active_call(self) -> None:
|
|
220
|
+
# Cancel the in-flight Connect so the response iterator unblocks. Safe to
|
|
221
|
+
# call when there is no active call or it has already finished.
|
|
222
|
+
call = self._active_call
|
|
223
|
+
if call is None:
|
|
224
|
+
return
|
|
225
|
+
call.cancel()
|
|
226
|
+
|
|
227
|
+
# -- control plane ----------------------------------------------------------
|
|
228
|
+
|
|
229
|
+
def _drain_events(self) -> Iterator[pb2.ClientEvent]:
|
|
230
|
+
while True:
|
|
231
|
+
event = self._events.get()
|
|
232
|
+
if event is None:
|
|
233
|
+
return
|
|
234
|
+
yield event
|
|
235
|
+
|
|
236
|
+
def _handle_command(self, command: pb2.RecorderCommand) -> None:
|
|
237
|
+
which = command.WhichOneof("command")
|
|
238
|
+
if which == "describe":
|
|
239
|
+
self._answer_describe(command.command_id)
|
|
240
|
+
return
|
|
241
|
+
if which == "open":
|
|
242
|
+
self._handle_open(command.open)
|
|
243
|
+
return
|
|
244
|
+
if which == "close":
|
|
245
|
+
self._handle_close(command.close)
|
|
246
|
+
return
|
|
247
|
+
if which == "shutdown":
|
|
248
|
+
self._events.put(None)
|
|
249
|
+
self._cancel_active_call()
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
def _answer_describe(self, command_id: str) -> None:
|
|
253
|
+
assert self._schema is not None
|
|
254
|
+
discovered = proto_mapping.discovered_events(self._schema)
|
|
255
|
+
for message in discovered:
|
|
256
|
+
event = pb2.ClientEvent(client_id=self._client_id, command_id=command_id)
|
|
257
|
+
event.source_discovered.CopyFrom(message)
|
|
258
|
+
self._events.put(event)
|
|
259
|
+
completed = pb2.ClientEvent(client_id=self._client_id, command_id=command_id)
|
|
260
|
+
completed.scan_completed.CopyFrom(pb2.ScanCompleted(discovered_count=len(discovered)))
|
|
261
|
+
self._events.put(completed)
|
|
262
|
+
|
|
263
|
+
def _handle_open(self, open_command: pb2.OpenCommand) -> None:
|
|
264
|
+
# A client may receive several open commands (one per source), each
|
|
265
|
+
# possibly carrying a negotiated robot config to apply — so negotiate on
|
|
266
|
+
# every open, not just the first.
|
|
267
|
+
self._apply_robot_config(open_command)
|
|
268
|
+
# Open unblocks the data-plane sender. The Source Bus identifies the
|
|
269
|
+
# opened source(s) via ``source_ids``; ``session_id`` is an optional tag
|
|
270
|
+
# the bus may leave empty (it routes batches by source_id), so it must
|
|
271
|
+
# NOT gate streaming. The operator drives recording start/stop.
|
|
272
|
+
if self._opened:
|
|
273
|
+
return
|
|
274
|
+
self._opened = True
|
|
275
|
+
self._session_id = open_command.session_id
|
|
276
|
+
self._session_ready.set()
|
|
277
|
+
|
|
278
|
+
def _apply_robot_config(self, open_command: pb2.OpenCommand) -> None:
|
|
279
|
+
if open_command.WhichOneof("params") != "robot":
|
|
280
|
+
return
|
|
281
|
+
if not open_command.robot.HasField("requested_config"):
|
|
282
|
+
return
|
|
283
|
+
requested = proto_mapping.robot_config_from_pb(open_command.robot.requested_config)
|
|
284
|
+
for source_id in open_command.source_ids:
|
|
285
|
+
self._negotiate_source(open_command.session_id, source_id, requested)
|
|
286
|
+
|
|
287
|
+
def _negotiate_source(self, session_id: str, source_id: str, requested: RobotConfig) -> None:
|
|
288
|
+
if self._on_select is None:
|
|
289
|
+
# The server selected a config but the client gave no apply callback;
|
|
290
|
+
# it cannot honor the request. Report FAILED and let the server decide.
|
|
291
|
+
self._events.put(
|
|
292
|
+
_failed_event(self._client_id, session_id, source_id, "no on_select callback to apply robot config")
|
|
293
|
+
)
|
|
294
|
+
return
|
|
295
|
+
try:
|
|
296
|
+
applied = self._on_select(source_id, requested)
|
|
297
|
+
except Exception as error: # user callback: any failure means the source cannot open
|
|
298
|
+
self._events.put(_failed_event(self._client_id, session_id, source_id, str(error)))
|
|
299
|
+
return
|
|
300
|
+
self._events.put(_applied_event(self._client_id, session_id, source_id, applied or requested))
|
|
301
|
+
|
|
302
|
+
def _handle_close(self, close_command: pb2.Close) -> None:
|
|
303
|
+
# The Source Bus ended the session's data stream. Stop the sender (end the
|
|
304
|
+
# StreamSamples iterator) but keep the Connect control stream up so the
|
|
305
|
+
# server can re-scan or shut down cleanly. The frame queue survives.
|
|
306
|
+
if close_command.session_id and close_command.session_id != self._session_id:
|
|
307
|
+
return
|
|
308
|
+
self._session_ended.set()
|
|
309
|
+
self._session_ready.set()
|
|
310
|
+
self._frames.wake()
|
|
311
|
+
|
|
312
|
+
# -- data plane -------------------------------------------------------------
|
|
313
|
+
|
|
314
|
+
def _run_sender(self, stub: pb2_grpc.SourceIngestServiceStub) -> None:
|
|
315
|
+
# Wait until the Source Bus has opened the source(s) (or the session ended).
|
|
316
|
+
self._session_ready.wait()
|
|
317
|
+
if not self._opened or self._closed.is_set():
|
|
318
|
+
return
|
|
319
|
+
try:
|
|
320
|
+
stub.StreamSamples(self._drain_batches(self._session_id))
|
|
321
|
+
except grpc.RpcError:
|
|
322
|
+
self._stats.incr("failed")
|
|
323
|
+
|
|
324
|
+
def _drain_batches(self, session_id: str) -> Iterator[pb2.SampleBatch]:
|
|
325
|
+
while not self._session_ended.is_set():
|
|
326
|
+
frame = self._frames.get()
|
|
327
|
+
if frame is None:
|
|
328
|
+
return
|
|
329
|
+
yield proto_mapping.sample_batch(frame, session_id)
|
|
330
|
+
self._stats.incr("sent")
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _ready_event(client_id: str, version: str) -> pb2.ClientEvent:
|
|
334
|
+
event = pb2.ClientEvent(client_id=client_id)
|
|
335
|
+
event.ready.CopyFrom(pb2.ClientReady(version=version))
|
|
336
|
+
return event
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _error_event(client_id: str, source_id: str, code: str, message: str) -> pb2.ClientEvent:
|
|
340
|
+
event = pb2.ClientEvent(client_id=client_id)
|
|
341
|
+
event.error.CopyFrom(pb2.ClientError(code=code, message=message, source_id=source_id))
|
|
342
|
+
return event
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _applied_event(client_id: str, session_id: str, source_id: str, applied: RobotConfig) -> pb2.ClientEvent:
|
|
346
|
+
event = pb2.ClientEvent(client_id=client_id)
|
|
347
|
+
changed = pb2.StreamStateChanged(session_id=session_id, source_id=source_id, state=pb2.STREAM_STATE_STARTED)
|
|
348
|
+
changed.robot.CopyFrom(proto_mapping.robot_config_to_pb(applied))
|
|
349
|
+
event.stream_state_changed.CopyFrom(changed)
|
|
350
|
+
return event
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _failed_event(client_id: str, session_id: str, source_id: str, detail: str) -> pb2.ClientEvent:
|
|
354
|
+
event = pb2.ClientEvent(client_id=client_id)
|
|
355
|
+
event.stream_state_changed.CopyFrom(
|
|
356
|
+
pb2.StreamStateChanged(
|
|
357
|
+
session_id=session_id, source_id=source_id, state=pb2.STREAM_STATE_FAILED, detail=detail
|
|
358
|
+
)
|
|
359
|
+
)
|
|
360
|
+
return event
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""gRPC implementation of the source reader (synchronous, blocking iterator).
|
|
2
|
+
|
|
3
|
+
The mirror of ``GrpcSourceClient``: where the producer drives ``StreamSamples``
|
|
4
|
+
(client -> server), the consumer drives ``SourceReadService.Subscribe`` (server
|
|
5
|
+
-> client). A subscription is a single server-streaming call, so it maps cleanly
|
|
6
|
+
to a blocking Python iterator — no background threads are needed.
|
|
7
|
+
|
|
8
|
+
``close()`` cancels the in-flight call from another thread so a consumer blocked
|
|
9
|
+
in the ``for frame in ...:`` loop unblocks promptly. Transport failures and
|
|
10
|
+
per-source faults are translated into domain ``SourceError`` subclasses at this
|
|
11
|
+
boundary.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import threading
|
|
17
|
+
from collections.abc import Iterator
|
|
18
|
+
|
|
19
|
+
import grpc
|
|
20
|
+
|
|
21
|
+
from loop_sdk.gen.v1 import source_ingest_pb2 as pb2
|
|
22
|
+
from loop_sdk.gen.v1 import source_ingest_pb2_grpc as pb2_grpc
|
|
23
|
+
from loop_sdk.source.domain.frame import FrameSample
|
|
24
|
+
from loop_sdk.source.domain.source_exception import SubscriptionFailedError
|
|
25
|
+
from loop_sdk.source.outbound import proto_mapping
|
|
26
|
+
from loop_sdk.source.port.source_reader import SourceReader
|
|
27
|
+
|
|
28
|
+
# gRPC keepalive so a silent half-open connection is detected and recycled.
|
|
29
|
+
_CHANNEL_OPTIONS = (
|
|
30
|
+
("grpc.keepalive_time_ms", 10_000),
|
|
31
|
+
("grpc.keepalive_timeout_ms", 5_000),
|
|
32
|
+
("grpc.keepalive_permit_without_calls", 1),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class GrpcSourceReader(SourceReader):
|
|
37
|
+
"""Read one source's samples over synchronous gRPC server-streaming."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, loop_addr: str) -> None:
|
|
40
|
+
self._channel = grpc.insecure_channel(loop_addr, options=_CHANNEL_OPTIONS)
|
|
41
|
+
self._stub = pb2_grpc.SourceReadServiceStub(self._channel)
|
|
42
|
+
self._closed = threading.Event()
|
|
43
|
+
self._active_call: grpc.Call | None = None
|
|
44
|
+
|
|
45
|
+
def subscribe(self, source_id: str, session_id: str = "") -> Iterator[FrameSample]:
|
|
46
|
+
request = pb2.SubscribeRequest(source_id=source_id, session_id=session_id)
|
|
47
|
+
call = self._stub.Subscribe(request)
|
|
48
|
+
self._active_call = call
|
|
49
|
+
try:
|
|
50
|
+
yield from self._drain(source_id, call)
|
|
51
|
+
except grpc.RpcError as error:
|
|
52
|
+
if self._closed.is_set() or error.code() == grpc.StatusCode.CANCELLED:
|
|
53
|
+
return
|
|
54
|
+
raise SubscriptionFailedError(source_id, _rpc_detail(error)) from error
|
|
55
|
+
finally:
|
|
56
|
+
self._active_call = None
|
|
57
|
+
|
|
58
|
+
def _drain(self, source_id: str, call: Iterator[pb2.SubscribeEvent]) -> Iterator[FrameSample]:
|
|
59
|
+
for event in call:
|
|
60
|
+
which = event.WhichOneof("event")
|
|
61
|
+
if which == "sample":
|
|
62
|
+
yield proto_mapping.frame_from_sample(source_id, event.sample)
|
|
63
|
+
continue
|
|
64
|
+
if which == "state":
|
|
65
|
+
# STARTED/UNSPECIFIED are informational; STOPPED ends the stream
|
|
66
|
+
# cleanly; FAILED surfaces the per-source fault to the caller.
|
|
67
|
+
if event.state.state == pb2.STREAM_STATE_STOPPED:
|
|
68
|
+
return
|
|
69
|
+
if event.state.state == pb2.STREAM_STATE_FAILED:
|
|
70
|
+
raise SubscriptionFailedError(source_id, event.state.detail)
|
|
71
|
+
|
|
72
|
+
def close(self) -> None:
|
|
73
|
+
self._closed.set()
|
|
74
|
+
call = self._active_call
|
|
75
|
+
if call is not None:
|
|
76
|
+
call.cancel()
|
|
77
|
+
self._channel.close()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _rpc_detail(error: grpc.RpcError) -> str:
|
|
81
|
+
code = error.code()
|
|
82
|
+
return code.name if code is not None else "transport error"
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Protobuf <-> domain mapping for the Source Bus contract.
|
|
2
|
+
|
|
3
|
+
Lives in ``outbound/`` because it touches generated stubs. The rest of the SDK
|
|
4
|
+
(domain, service, port) never imports ``pb2``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from loop_sdk.gen.v1 import source_ingest_pb2 as pb2
|
|
10
|
+
from loop_sdk.source.domain.config import RobotConfig, RobotConfigOptions
|
|
11
|
+
from loop_sdk.source.domain.frame import (
|
|
12
|
+
FrameSample,
|
|
13
|
+
MarkerFrame,
|
|
14
|
+
PolicyActionFrame,
|
|
15
|
+
RobotFrame,
|
|
16
|
+
RobotStateValue,
|
|
17
|
+
TactileChannelValue,
|
|
18
|
+
TactileFrame,
|
|
19
|
+
)
|
|
20
|
+
from loop_sdk.source.domain.schema import (
|
|
21
|
+
ChannelRole,
|
|
22
|
+
ChannelSpec,
|
|
23
|
+
MarkerStreamSchema,
|
|
24
|
+
RobotStreamSchema,
|
|
25
|
+
RotationType,
|
|
26
|
+
SourceSchema,
|
|
27
|
+
TactileStreamSchema,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Domain closed-enum -> proto enum. None/omitted maps to the proto's UNSPECIFIED.
|
|
31
|
+
_ROLE_TO_PB = {
|
|
32
|
+
ChannelRole.CORE: pb2.CHANNEL_ROLE_CORE,
|
|
33
|
+
ChannelRole.AUX: pb2.CHANNEL_ROLE_AUX,
|
|
34
|
+
}
|
|
35
|
+
_ROT_TO_PB = {
|
|
36
|
+
RotationType.QUATERNION: pb2.ROBOT_ROTATION_TYPE_QUATERNION,
|
|
37
|
+
RotationType.EULER: pb2.ROBOT_ROTATION_TYPE_EULER,
|
|
38
|
+
RotationType.ROTATION_6D: pb2.ROBOT_ROTATION_TYPE_ROTATION_6D,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def discovered_events(schema: SourceSchema) -> list[pb2.SourceDiscovered]:
|
|
43
|
+
"""Build the ``SourceDiscovered`` messages that answer a ``Describe``."""
|
|
44
|
+
events: list[pb2.SourceDiscovered] = []
|
|
45
|
+
events.extend(_robot_discovered(stream) for stream in schema.robot)
|
|
46
|
+
events.extend(_tactile_discovered(stream) for stream in schema.tactile)
|
|
47
|
+
events.extend(_marker_discovered(stream) for stream in schema.marker)
|
|
48
|
+
return events
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _robot_channel(channel: ChannelSpec) -> pb2.RobotChannelDescriptor:
|
|
52
|
+
descriptor = pb2.RobotChannelDescriptor(
|
|
53
|
+
key=channel.key, label=channel.label, unit=channel.unit, group=channel.group
|
|
54
|
+
)
|
|
55
|
+
if channel.role is not None:
|
|
56
|
+
descriptor.role = _ROLE_TO_PB[channel.role]
|
|
57
|
+
if channel.rot_type is not None:
|
|
58
|
+
descriptor.rot_type = _ROT_TO_PB[channel.rot_type]
|
|
59
|
+
if channel.range is not None:
|
|
60
|
+
descriptor.range.extend(channel.range)
|
|
61
|
+
return descriptor
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _robot_discovered(stream: RobotStreamSchema) -> pb2.SourceDiscovered:
|
|
65
|
+
message = pb2.SourceDiscovered(source_id=stream.source_id, name=stream.name)
|
|
66
|
+
message.robot.channels.extend(_robot_channel(channel) for channel in stream.channels)
|
|
67
|
+
message.robot.available_options.CopyFrom(_robot_config_options(stream.available_options))
|
|
68
|
+
return message
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _robot_config_options(options: RobotConfigOptions) -> pb2.RobotConfigOptions:
|
|
72
|
+
return pb2.RobotConfigOptions(
|
|
73
|
+
control_hz=options.control_hz,
|
|
74
|
+
action_space=options.action_space,
|
|
75
|
+
gripper_type=options.gripper_type,
|
|
76
|
+
finger_type=options.finger_type,
|
|
77
|
+
robot_type=options.robot_type,
|
|
78
|
+
robot_firmware_version=options.robot_firmware_version,
|
|
79
|
+
teleoperation_version=options.teleoperation_version,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def robot_config_to_pb(config: RobotConfig) -> pb2.RobotConfig:
|
|
84
|
+
# control_hz/action_space are always set; the optional identity fields are
|
|
85
|
+
# set only when non-empty so an unset proto `optional` stays absent.
|
|
86
|
+
message = pb2.RobotConfig(control_hz=config.control_hz, action_space=config.action_space)
|
|
87
|
+
if config.gripper_type:
|
|
88
|
+
message.gripper_type = config.gripper_type
|
|
89
|
+
if config.finger_type:
|
|
90
|
+
message.finger_type = config.finger_type
|
|
91
|
+
if config.robot_type:
|
|
92
|
+
message.robot_type = config.robot_type
|
|
93
|
+
if config.robot_firmware_version:
|
|
94
|
+
message.robot_firmware_version = config.robot_firmware_version
|
|
95
|
+
if config.teleoperation_version:
|
|
96
|
+
message.teleoperation_version = config.teleoperation_version
|
|
97
|
+
return message
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def robot_config_from_pb(message: pb2.RobotConfig) -> RobotConfig:
|
|
101
|
+
# Unset proto `optional string` reads back as "" — the domain default.
|
|
102
|
+
return RobotConfig(
|
|
103
|
+
control_hz=message.control_hz,
|
|
104
|
+
action_space=message.action_space,
|
|
105
|
+
gripper_type=message.gripper_type,
|
|
106
|
+
finger_type=message.finger_type,
|
|
107
|
+
robot_type=message.robot_type,
|
|
108
|
+
robot_firmware_version=message.robot_firmware_version,
|
|
109
|
+
teleoperation_version=message.teleoperation_version,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _tactile_discovered(stream: TactileStreamSchema) -> pb2.SourceDiscovered:
|
|
114
|
+
message = pb2.SourceDiscovered(source_id=stream.source_id, name=stream.name)
|
|
115
|
+
message.tactile.channels.extend(
|
|
116
|
+
pb2.TactileChannelDescriptor(
|
|
117
|
+
key=channel.key,
|
|
118
|
+
label=channel.label,
|
|
119
|
+
sample_format=channel.sample_format,
|
|
120
|
+
sample_count=channel.sample_count,
|
|
121
|
+
)
|
|
122
|
+
for channel in stream.channels
|
|
123
|
+
)
|
|
124
|
+
return message
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _marker_discovered(stream: MarkerStreamSchema) -> pb2.SourceDiscovered:
|
|
128
|
+
message = pb2.SourceDiscovered(source_id=stream.source_id, name=stream.name)
|
|
129
|
+
message.marker.CopyFrom(pb2.MarkerSourceDescriptor())
|
|
130
|
+
return message
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def sample_batch(frame: FrameSample, session_id: str) -> pb2.SampleBatch:
|
|
134
|
+
"""Map one domain frame to a single-sample ``SampleBatch``."""
|
|
135
|
+
return pb2.SampleBatch(
|
|
136
|
+
session_id=session_id,
|
|
137
|
+
source_id=frame.source_id,
|
|
138
|
+
samples=[_sample(frame)],
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _to_pb_robot_value(value: RobotStateValue) -> pb2.RobotValue:
|
|
143
|
+
# bool is a subclass of int, so it must be checked first. None / unknown
|
|
144
|
+
# types yield an empty RobotValue (no arm set) and are filtered before this.
|
|
145
|
+
if isinstance(value, bool):
|
|
146
|
+
return pb2.RobotValue(bool_value=value)
|
|
147
|
+
if isinstance(value, int):
|
|
148
|
+
return pb2.RobotValue(int_value=value)
|
|
149
|
+
if isinstance(value, float):
|
|
150
|
+
return pb2.RobotValue(scalar=value)
|
|
151
|
+
if isinstance(value, str):
|
|
152
|
+
return pb2.RobotValue(string_value=value)
|
|
153
|
+
if isinstance(value, list):
|
|
154
|
+
return pb2.RobotValue(array=pb2.DoubleArray(values=[float(item) for item in value]))
|
|
155
|
+
return pb2.RobotValue()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _robot_state_value(value: pb2.RobotValue) -> RobotStateValue:
|
|
159
|
+
arm = value.WhichOneof("value")
|
|
160
|
+
if arm == "scalar":
|
|
161
|
+
return value.scalar
|
|
162
|
+
if arm == "array":
|
|
163
|
+
return list(value.array.values)
|
|
164
|
+
if arm == "int_value":
|
|
165
|
+
return value.int_value
|
|
166
|
+
if arm == "bool_value":
|
|
167
|
+
return value.bool_value
|
|
168
|
+
if arm == "string_value":
|
|
169
|
+
return value.string_value
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _sample(frame: FrameSample) -> pb2.Sample:
|
|
174
|
+
sample = pb2.Sample(timestamp_us=frame.timestamp_us, sequence=frame.sequence)
|
|
175
|
+
if isinstance(frame, RobotFrame):
|
|
176
|
+
for key, value in frame.state.items():
|
|
177
|
+
if value is None:
|
|
178
|
+
continue
|
|
179
|
+
sample.robot.state[key].CopyFrom(_to_pb_robot_value(value))
|
|
180
|
+
return sample
|
|
181
|
+
if isinstance(frame, TactileFrame):
|
|
182
|
+
sample.tactile.channels.extend(
|
|
183
|
+
pb2.TactileChannelSample(key=channel.key, raw=channel.raw, values_i16=channel.values_i16)
|
|
184
|
+
for channel in frame.channels
|
|
185
|
+
)
|
|
186
|
+
return sample
|
|
187
|
+
if isinstance(frame, MarkerFrame):
|
|
188
|
+
sample.marker.CopyFrom(pb2.MarkerPayload(label=frame.label, category=frame.category))
|
|
189
|
+
return sample
|
|
190
|
+
raise TypeError(f"unsupported frame type: {type(frame).__name__}")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def frame_from_sample(source_id: str, sample: pb2.Sample) -> FrameSample:
|
|
194
|
+
"""Decode one read-plane ``Sample`` into a domain frame (the inverse of ``_sample``).
|
|
195
|
+
|
|
196
|
+
``Sample`` carries no ``source_id`` (it is per-source on the wire); the
|
|
197
|
+
subscription supplies it.
|
|
198
|
+
"""
|
|
199
|
+
payload = sample.WhichOneof("payload")
|
|
200
|
+
if payload == "robot":
|
|
201
|
+
return RobotFrame(
|
|
202
|
+
source_id=source_id,
|
|
203
|
+
timestamp_us=sample.timestamp_us,
|
|
204
|
+
sequence=sample.sequence,
|
|
205
|
+
state={key: _robot_state_value(value) for key, value in sample.robot.state.items()},
|
|
206
|
+
)
|
|
207
|
+
if payload == "tactile":
|
|
208
|
+
return TactileFrame(
|
|
209
|
+
source_id=source_id,
|
|
210
|
+
timestamp_us=sample.timestamp_us,
|
|
211
|
+
sequence=sample.sequence,
|
|
212
|
+
channels=tuple(
|
|
213
|
+
TactileChannelValue(key=channel.key, raw=channel.raw, values_i16=tuple(channel.values_i16))
|
|
214
|
+
for channel in sample.tactile.channels
|
|
215
|
+
),
|
|
216
|
+
)
|
|
217
|
+
if payload == "marker":
|
|
218
|
+
return MarkerFrame(
|
|
219
|
+
source_id=source_id,
|
|
220
|
+
timestamp_us=sample.timestamp_us,
|
|
221
|
+
sequence=sample.sequence,
|
|
222
|
+
label=sample.marker.label,
|
|
223
|
+
category=sample.marker.category,
|
|
224
|
+
)
|
|
225
|
+
if payload == "policy_action":
|
|
226
|
+
pa = sample.policy_action
|
|
227
|
+
return PolicyActionFrame(
|
|
228
|
+
source_id=source_id,
|
|
229
|
+
timestamp_us=sample.timestamp_us,
|
|
230
|
+
sequence=sample.sequence,
|
|
231
|
+
actions=tuple(tuple(vector.values) for vector in pa.actions),
|
|
232
|
+
input_sample_set_source_id=(
|
|
233
|
+
pa.input_sample_set_source_id if pa.HasField("input_sample_set_source_id") else None
|
|
234
|
+
),
|
|
235
|
+
input_sample_set_sequence=(
|
|
236
|
+
pa.input_sample_set_sequence if pa.HasField("input_sample_set_sequence") else None
|
|
237
|
+
),
|
|
238
|
+
)
|
|
239
|
+
raise TypeError(f"sample carries no known payload (oneof={payload!r})")
|
|
File without changes
|