livekit 1.2.1__py3-none-win_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. livekit/rtc/__init__.py +264 -0
  2. livekit/rtc/_ffi_client.py +321 -0
  3. livekit/rtc/_proto/__init__.py +0 -0
  4. livekit/rtc/_proto/audio_frame_pb2.py +159 -0
  5. livekit/rtc/_proto/audio_frame_pb2.pyi +1560 -0
  6. livekit/rtc/_proto/capture_pb2.py +70 -0
  7. livekit/rtc/_proto/capture_pb2.pyi +559 -0
  8. livekit/rtc/_proto/data_stream_pb2.py +171 -0
  9. livekit/rtc/_proto/data_stream_pb2.pyi +1498 -0
  10. livekit/rtc/_proto/data_track_pb2.py +120 -0
  11. livekit/rtc/_proto/data_track_pb2.pyi +1073 -0
  12. livekit/rtc/_proto/e2ee_pb2.py +81 -0
  13. livekit/rtc/_proto/e2ee_pb2.pyi +591 -0
  14. livekit/rtc/_proto/ffi_pb2.py +55 -0
  15. livekit/rtc/_proto/ffi_pb2.pyi +1246 -0
  16. livekit/rtc/_proto/handle_pb2.py +27 -0
  17. livekit/rtc/_proto/handle_pb2.pyi +51 -0
  18. livekit/rtc/_proto/participant_pb2.py +45 -0
  19. livekit/rtc/_proto/participant_pb2.pyi +313 -0
  20. livekit/rtc/_proto/room_pb2.py +311 -0
  21. livekit/rtc/_proto/room_pb2.pyi +3136 -0
  22. livekit/rtc/_proto/rpc_pb2.py +47 -0
  23. livekit/rtc/_proto/rpc_pb2.pyi +261 -0
  24. livekit/rtc/_proto/stats_pb2.py +123 -0
  25. livekit/rtc/_proto/stats_pb2.pyi +1514 -0
  26. livekit/rtc/_proto/track_pb2.py +76 -0
  27. livekit/rtc/_proto/track_pb2.pyi +544 -0
  28. livekit/rtc/_proto/track_publication_pb2.py +39 -0
  29. livekit/rtc/_proto/track_publication_pb2.pyi +145 -0
  30. livekit/rtc/_proto/video_frame_pb2.py +97 -0
  31. livekit/rtc/_proto/video_frame_pb2.pyi +838 -0
  32. livekit/rtc/_utils.py +187 -0
  33. livekit/rtc/apm.py +119 -0
  34. livekit/rtc/audio_filter.py +23 -0
  35. livekit/rtc/audio_frame.py +242 -0
  36. livekit/rtc/audio_mixer.py +247 -0
  37. livekit/rtc/audio_resampler.py +174 -0
  38. livekit/rtc/audio_source.py +191 -0
  39. livekit/rtc/audio_stream.py +354 -0
  40. livekit/rtc/data_stream.py +640 -0
  41. livekit/rtc/data_track.py +296 -0
  42. livekit/rtc/e2ee.py +320 -0
  43. livekit/rtc/encoded_video.py +127 -0
  44. livekit/rtc/event_emitter.py +215 -0
  45. livekit/rtc/frame_processor.py +37 -0
  46. livekit/rtc/jupyter.py +68 -0
  47. livekit/rtc/log.py +3 -0
  48. livekit/rtc/media_devices.py +596 -0
  49. livekit/rtc/participant.py +1171 -0
  50. livekit/rtc/platform_audio.py +426 -0
  51. livekit/rtc/py.typed +0 -0
  52. livekit/rtc/resources/LICENSE.md +4055 -0
  53. livekit/rtc/resources/__init__.py +1 -0
  54. livekit/rtc/resources/jupyter-html/index.html +101 -0
  55. livekit/rtc/resources/livekit_ffi.dll +0 -0
  56. livekit/rtc/resources/livekit_ffi.h +42 -0
  57. livekit/rtc/room.py +1304 -0
  58. livekit/rtc/rpc.py +244 -0
  59. livekit/rtc/synchronizer.py +202 -0
  60. livekit/rtc/track.py +264 -0
  61. livekit/rtc/track_publication.py +145 -0
  62. livekit/rtc/transcription.py +19 -0
  63. livekit/rtc/utils.py +138 -0
  64. livekit/rtc/version.py +1 -0
  65. livekit/rtc/video_frame.py +347 -0
  66. livekit/rtc/video_source.py +74 -0
  67. livekit/rtc/video_stream.py +191 -0
  68. livekit-1.2.1.dist-info/METADATA +80 -0
  69. livekit-1.2.1.dist-info/RECORD +71 -0
  70. livekit-1.2.1.dist-info/WHEEL +5 -0
  71. livekit-1.2.1.dist-info/top_level.txt +1 -0
livekit/rtc/rpc.py ADDED
@@ -0,0 +1,244 @@
1
+ # Copyright 2023 LiveKit, Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Awaitable, Callable, ClassVar, Dict, Optional, Sequence, Union
18
+ from enum import IntEnum
19
+ from ._proto import rpc_pb2 as proto_rpc
20
+ from dataclasses import dataclass
21
+
22
+
23
+ @dataclass
24
+ class RpcInvocationData:
25
+ """Data passed to method handler for incoming RPC invocations
26
+
27
+ Attributes:
28
+ request_id (str): The unique request ID. Will match at both sides of the call, useful for debugging or logging.
29
+ caller_identity (str): The unique participant identity of the caller.
30
+ payload (str): The payload of the request. User-definable format, typically JSON.
31
+ response_timeout (float): The maximum time the caller will wait for a response.
32
+ method (str): The name of the invoked RPC method.
33
+ cancel_reason (Optional[RpcError.ErrorCode]): Why the SDK cancelled the handler chain,
34
+ set on this object just before it does: ``RESPONSE_TIMEOUT`` when the caller's
35
+ deadline passed, ``RECIPIENT_DISCONNECTED`` when the room disconnected. ``None``
36
+ while the chain runs, and for a ``CancelledError`` raised inside the chain (which
37
+ the caller receives as ``APPLICATION_ERROR``). Lets an interceptor unwinding from
38
+ the cancellation record the outcome the caller gets.
39
+ """
40
+
41
+ request_id: str
42
+ caller_identity: str
43
+ payload: str
44
+ response_timeout: float
45
+ method: str = ""
46
+ cancel_reason: Optional[RpcError.ErrorCode] = None
47
+
48
+
49
+ @dataclass
50
+ class RpcCallInfo:
51
+ """An outgoing RPC call, as passed to :meth:`RpcInterceptor.intercept_outgoing`.
52
+
53
+ Mirrors the arguments of :meth:`LocalParticipant.perform_rpc`. An interceptor may pass a
54
+ modified copy to ``next`` (for example to add a header to a JSON payload).
55
+
56
+ Attributes:
57
+ destination_identity (str): The identity of the participant being called.
58
+ method (str): The method name.
59
+ payload (str): The request payload.
60
+ response_timeout (Optional[float]): Seconds to wait for a response, or ``None`` for the default.
61
+ max_round_trip_latency (Optional[float]): See :meth:`LocalParticipant.perform_rpc`.
62
+ """
63
+
64
+ destination_identity: str
65
+ method: str
66
+ payload: str
67
+ response_timeout: Optional[float] = None
68
+ max_round_trip_latency: Optional[float] = None
69
+
70
+
71
+ OutgoingRpcNext = Callable[["RpcCallInfo"], Awaitable[str]]
72
+ """Continuation handed to :meth:`RpcInterceptor.intercept_outgoing`: performs the call."""
73
+ IncomingRpcNext = Callable[["RpcInvocationData"], Awaitable[Optional[str]]]
74
+ """Continuation handed to :meth:`RpcInterceptor.intercept_incoming`: runs the handler."""
75
+
76
+
77
+ class RpcInterceptor:
78
+ """Observe or wrap RPC calls made and handled by a :class:`LocalParticipant`.
79
+
80
+ Register with :meth:`LocalParticipant.add_rpc_interceptor`. Each method receives the call
81
+ and a ``next`` continuation and must return (or raise) what ``next`` returns (or raises),
82
+ unless it deliberately short-circuits the call. Interceptors run in registration order:
83
+ the first one added is the outermost. Both methods default to pass-through, so override
84
+ only the direction you care about.
85
+
86
+ Errors flow through the chain unchanged: a :class:`RpcError` raised by the remote side
87
+ (outgoing) or by the handler (incoming) is visible to every interceptor before it
88
+ reaches the caller. On the incoming side, any other exception raised by the handler is
89
+ also visible; the SDK converts it to ``APPLICATION_ERROR`` only after the chain returns,
90
+ and a call for an unregistered method reaches the chain with ``next`` raising
91
+ ``UNSUPPORTED_METHOD``. The caller's ``response_timeout`` covers the whole incoming
92
+ chain: when it passes, the chain is cancelled (interceptors see ``CancelledError``) and
93
+ the caller receives ``RESPONSE_TIMEOUT``.
94
+
95
+ Example:
96
+ Time every RPC in both directions::
97
+
98
+ class TimingInterceptor(rtc.RpcInterceptor):
99
+ async def intercept_outgoing(self, call, next):
100
+ start = time.perf_counter()
101
+ try:
102
+ return await next(call)
103
+ finally:
104
+ log("rpc call", call.method, time.perf_counter() - start)
105
+
106
+ async def intercept_incoming(self, invocation, next):
107
+ start = time.perf_counter()
108
+ try:
109
+ return await next(invocation)
110
+ finally:
111
+ log("rpc handled", invocation.method, time.perf_counter() - start)
112
+
113
+ room.local_participant.add_rpc_interceptor(TimingInterceptor())
114
+ """
115
+
116
+ async def intercept_outgoing(self, call: RpcCallInfo, next: OutgoingRpcNext) -> str:
117
+ """Wrap an outgoing :meth:`LocalParticipant.perform_rpc`. Return the response payload."""
118
+ return await next(call)
119
+
120
+ async def intercept_incoming(
121
+ self, invocation: RpcInvocationData, next: IncomingRpcNext
122
+ ) -> Optional[str]:
123
+ """Wrap the handling of an incoming invocation. Return the response payload."""
124
+ return await next(invocation)
125
+
126
+
127
+ def _chain_outgoing(
128
+ interceptors: Sequence[RpcInterceptor], terminal: OutgoingRpcNext
129
+ ) -> OutgoingRpcNext:
130
+ """Compose ``interceptors`` around ``terminal``; the first interceptor is outermost."""
131
+ call_next = terminal
132
+ for interceptor in reversed(interceptors):
133
+ call_next = _bind_outgoing(interceptor, call_next)
134
+ return call_next
135
+
136
+
137
+ def _chain_incoming(
138
+ interceptors: Sequence[RpcInterceptor], terminal: IncomingRpcNext
139
+ ) -> IncomingRpcNext:
140
+ call_next = terminal
141
+ for interceptor in reversed(interceptors):
142
+ call_next = _bind_incoming(interceptor, call_next)
143
+ return call_next
144
+
145
+
146
+ def _bind_outgoing(interceptor: RpcInterceptor, call_next: OutgoingRpcNext) -> OutgoingRpcNext:
147
+ async def bound(call: RpcCallInfo) -> str:
148
+ return await interceptor.intercept_outgoing(call, call_next)
149
+
150
+ return bound
151
+
152
+
153
+ def _bind_incoming(interceptor: RpcInterceptor, call_next: IncomingRpcNext) -> IncomingRpcNext:
154
+ async def bound(invocation: RpcInvocationData) -> Optional[str]:
155
+ return await interceptor.intercept_incoming(invocation, call_next)
156
+
157
+ return bound
158
+
159
+
160
+ class RpcError(Exception):
161
+ """
162
+ Specialized error handling for RPC methods.
163
+
164
+ Instances of this type, when thrown in a method handler, will have their `message`
165
+ serialized and sent across the wire. The caller will receive an equivalent error on the other side.
166
+
167
+ Built-in errors are included (codes 1001-1999) but developers may use the code, message, and data fields to create their own errors.
168
+ """
169
+
170
+ class ErrorCode(IntEnum):
171
+ APPLICATION_ERROR = 1500
172
+ CONNECTION_TIMEOUT = 1501
173
+ RESPONSE_TIMEOUT = 1502
174
+ RECIPIENT_DISCONNECTED = 1503
175
+ RESPONSE_PAYLOAD_TOO_LARGE = 1504
176
+ SEND_FAILED = 1505
177
+
178
+ UNSUPPORTED_METHOD = 1400
179
+ RECIPIENT_NOT_FOUND = 1401
180
+ REQUEST_PAYLOAD_TOO_LARGE = 1402
181
+ UNSUPPORTED_SERVER = 1403
182
+ UNSUPPORTED_VERSION = 1404
183
+
184
+ ErrorMessage: ClassVar[Dict[ErrorCode, str]] = {
185
+ ErrorCode.APPLICATION_ERROR: "Application error in method handler",
186
+ ErrorCode.CONNECTION_TIMEOUT: "Connection timeout",
187
+ ErrorCode.RESPONSE_TIMEOUT: "Response timeout",
188
+ ErrorCode.RECIPIENT_DISCONNECTED: "Recipient disconnected",
189
+ ErrorCode.RESPONSE_PAYLOAD_TOO_LARGE: "Response payload too large",
190
+ ErrorCode.SEND_FAILED: "Failed to send",
191
+ ErrorCode.UNSUPPORTED_METHOD: "Method not supported at destination",
192
+ ErrorCode.RECIPIENT_NOT_FOUND: "Recipient not found",
193
+ ErrorCode.REQUEST_PAYLOAD_TOO_LARGE: "Request payload too large",
194
+ ErrorCode.UNSUPPORTED_SERVER: "RPC not supported by server",
195
+ ErrorCode.UNSUPPORTED_VERSION: "Unsupported RPC version",
196
+ }
197
+
198
+ def __init__(
199
+ self,
200
+ code: Union[int, "RpcError.ErrorCode"],
201
+ message: str,
202
+ data: Optional[str] = None,
203
+ ):
204
+ """
205
+ Creates an error object with the given code and message, plus an optional data payload.
206
+
207
+ If thrown in an RPC method handler, the error will be sent back to the caller.
208
+
209
+ Args:
210
+ code (int): Your error code (Error codes 1001-1999 are reserved for built-in errors)
211
+ message (str): A readable error message.
212
+ data (Optional[str]): Optional additional data associated with the error (JSON recommended)
213
+ """
214
+ super().__init__(message)
215
+ self._code = code
216
+ self._message = message
217
+ self._data = data
218
+
219
+ @property
220
+ def code(self) -> int:
221
+ """Error code value. Codes 1001-1999 are reserved for built-in errors (see RpcError.ErrorCode for their meanings)."""
222
+ return self._code
223
+
224
+ @property
225
+ def message(self) -> str:
226
+ """A readable error message."""
227
+ return self._message
228
+
229
+ @property
230
+ def data(self) -> Optional[str]:
231
+ """Optional additional data associated with the error (JSON recommended)."""
232
+ return self._data
233
+
234
+ @classmethod
235
+ def _from_proto(cls, proto: proto_rpc.RpcError) -> "RpcError":
236
+ return cls(proto.code, proto.message, proto.data)
237
+
238
+ def _to_proto(self) -> proto_rpc.RpcError:
239
+ return proto_rpc.RpcError(code=self.code, message=self.message, data=self.data)
240
+
241
+ @classmethod
242
+ def _built_in(cls, code: "RpcError.ErrorCode", data: Optional[str] = None) -> "RpcError":
243
+ message = cls.ErrorMessage[code]
244
+ return cls(code, message, data)
@@ -0,0 +1,202 @@
1
+ import asyncio
2
+ import logging
3
+ import time
4
+ from collections import deque
5
+ from typing import Any, Optional, Union
6
+
7
+ from .video_frame import VideoFrame
8
+ from .audio_frame import AudioFrame
9
+ from .audio_source import AudioSource
10
+ from .video_source import VideoSource
11
+
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ class AVSynchronizer:
17
+ """Synchronize audio and video capture.
18
+
19
+ Usage:
20
+ av_sync = AVSynchronizer(
21
+ audio_source=audio_source,
22
+ video_source=video_source,
23
+ video_fps=video_fps,
24
+ )
25
+
26
+ async for video_frame, audio_frame in video_generator:
27
+ await av_sync.push(video_frame)
28
+ await av_sync.push(audio_frame)
29
+ """
30
+
31
+ def __init__(
32
+ self,
33
+ *,
34
+ audio_source: AudioSource,
35
+ video_source: VideoSource,
36
+ video_fps: float,
37
+ video_queue_size_ms: float = 100,
38
+ _max_delay_tolerance_ms: float = 300,
39
+ ):
40
+ self._audio_source = audio_source
41
+ self._video_source = video_source
42
+ self._video_fps = video_fps
43
+ self._video_queue_size_ms = video_queue_size_ms
44
+ self._max_delay_tolerance_ms = _max_delay_tolerance_ms
45
+
46
+ self._stopped = False
47
+ # the time of the last video/audio frame captured
48
+ self._last_video_time: float = 0
49
+ self._last_audio_time: float = 0
50
+
51
+ self._video_queue_max_size = int(self._video_fps * self._video_queue_size_ms / 1000)
52
+ if self._video_queue_size_ms > 0:
53
+ # ensure queue is bounded if queue size is specified
54
+ self._video_queue_max_size = max(1, self._video_queue_max_size)
55
+
56
+ self._video_queue = asyncio.Queue[tuple[VideoFrame, Optional[float]]](
57
+ maxsize=self._video_queue_max_size
58
+ )
59
+ self._fps_controller = _FPSController(
60
+ expected_fps=self._video_fps,
61
+ max_delay_tolerance_ms=self._max_delay_tolerance_ms,
62
+ )
63
+ self._capture_video_task = asyncio.create_task(self._capture_video())
64
+
65
+ async def push(
66
+ self, frame: Union[VideoFrame, AudioFrame], timestamp: Optional[float] = None
67
+ ) -> None:
68
+ """Push a frame to the synchronizer
69
+
70
+ Args:
71
+ frame: The video or audio frame to push.
72
+ timestamp: (optional) The timestamp of the frame, for logging purposes for now.
73
+ For AudioFrame, it should be the end time of the frame.
74
+ """
75
+ if isinstance(frame, AudioFrame):
76
+ await self._audio_source.capture_frame(frame)
77
+ if timestamp is not None:
78
+ self._last_audio_time = timestamp
79
+ return
80
+
81
+ await self._video_queue.put((frame, timestamp))
82
+
83
+ async def clear_queue(self) -> None:
84
+ self._audio_source.clear_queue()
85
+ while not self._video_queue.empty():
86
+ await self._video_queue.get()
87
+ self._video_queue.task_done()
88
+
89
+ async def wait_for_playout(self) -> None:
90
+ """Wait until all video and audio frames are played out."""
91
+ await asyncio.gather(
92
+ self._audio_source.wait_for_playout(),
93
+ self._video_queue.join(),
94
+ )
95
+
96
+ def reset(self) -> None:
97
+ self._fps_controller.reset()
98
+
99
+ async def _capture_video(self) -> None:
100
+ while not self._stopped:
101
+ frame, timestamp = await self._video_queue.get()
102
+ async with self._fps_controller:
103
+ self._video_source.capture_frame(frame)
104
+ if timestamp is not None:
105
+ self._last_video_time = timestamp
106
+ self._video_queue.task_done()
107
+
108
+ async def aclose(self) -> None:
109
+ self._stopped = True
110
+ if self._capture_video_task:
111
+ self._capture_video_task.cancel()
112
+
113
+ @property
114
+ def actual_fps(self) -> float:
115
+ return self._fps_controller.actual_fps
116
+
117
+ @property
118
+ def last_video_time(self) -> float:
119
+ """The time of the last video frame captured"""
120
+ return self._last_video_time
121
+
122
+ @property
123
+ def last_audio_time(self) -> float:
124
+ """The time of the last audio frame played out"""
125
+ return self._last_audio_time - self._audio_source.queued_duration
126
+
127
+
128
+ class _FPSController:
129
+ def __init__(self, *, expected_fps: float, max_delay_tolerance_ms: float = 300) -> None:
130
+ """Controls frame rate by adjusting sleep time based on actual FPS.
131
+
132
+ Usage:
133
+ async with _FPSController(expected_fps=30):
134
+ # process frame
135
+ pass
136
+
137
+ Args:
138
+ expected_fps: Target frames per second
139
+ max_delay_tolerance_ms: Maximum delay tolerance in milliseconds
140
+ """
141
+ self._expected_fps = expected_fps
142
+ self._frame_interval = 1.0 / expected_fps
143
+ self._max_delay_tolerance_secs = max_delay_tolerance_ms / 1000
144
+
145
+ self._next_frame_time: Optional[float] = None
146
+ self._fps_calc_winsize = max(2, int(1.0 * expected_fps))
147
+ self._send_timestamps: deque[float] = deque(maxlen=self._fps_calc_winsize)
148
+
149
+ async def __aenter__(self) -> None:
150
+ await self.wait_next_process()
151
+
152
+ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
153
+ self.after_process()
154
+
155
+ def reset(self) -> None:
156
+ self._next_frame_time = None
157
+ self._send_timestamps.clear()
158
+
159
+ async def wait_next_process(self) -> None:
160
+ """Wait until it's time for the next frame.
161
+
162
+ Adjusts sleep time based on actual FPS to maintain target rate.
163
+ """
164
+ current_time = time.perf_counter()
165
+
166
+ # initialize the next frame time
167
+ if self._next_frame_time is None:
168
+ self._next_frame_time = current_time
169
+
170
+ # calculate sleep time
171
+ sleep_time = self._next_frame_time - current_time
172
+ if sleep_time > 0:
173
+ await asyncio.sleep(sleep_time)
174
+ else:
175
+ # check if significantly behind schedule
176
+ if -sleep_time > self._max_delay_tolerance_secs:
177
+ logger.warning(f"Frame capture was behind schedule for {-sleep_time * 1000:.2f} ms")
178
+ self._next_frame_time = time.perf_counter()
179
+
180
+ def after_process(self) -> None:
181
+ """Update timing information after processing a frame."""
182
+ assert self._next_frame_time is not None, "wait_next_process must be called first"
183
+
184
+ # update timing information
185
+ self._send_timestamps.append(time.perf_counter())
186
+
187
+ # calculate next frame time
188
+ self._next_frame_time += self._frame_interval
189
+
190
+ @property
191
+ def expected_fps(self) -> float:
192
+ return self._expected_fps
193
+
194
+ @property
195
+ def actual_fps(self) -> float:
196
+ """Get current average FPS."""
197
+ if len(self._send_timestamps) < 2:
198
+ return 0
199
+
200
+ return (len(self._send_timestamps) - 1) / (
201
+ self._send_timestamps[-1] - self._send_timestamps[0]
202
+ )