livekit 1.2.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit/rtc/__init__.py +264 -0
- livekit/rtc/_ffi_client.py +321 -0
- livekit/rtc/_proto/__init__.py +0 -0
- livekit/rtc/_proto/audio_frame_pb2.py +159 -0
- livekit/rtc/_proto/audio_frame_pb2.pyi +1560 -0
- livekit/rtc/_proto/capture_pb2.py +70 -0
- livekit/rtc/_proto/capture_pb2.pyi +559 -0
- livekit/rtc/_proto/data_stream_pb2.py +171 -0
- livekit/rtc/_proto/data_stream_pb2.pyi +1498 -0
- livekit/rtc/_proto/data_track_pb2.py +120 -0
- livekit/rtc/_proto/data_track_pb2.pyi +1073 -0
- livekit/rtc/_proto/e2ee_pb2.py +81 -0
- livekit/rtc/_proto/e2ee_pb2.pyi +591 -0
- livekit/rtc/_proto/ffi_pb2.py +55 -0
- livekit/rtc/_proto/ffi_pb2.pyi +1246 -0
- livekit/rtc/_proto/handle_pb2.py +27 -0
- livekit/rtc/_proto/handle_pb2.pyi +51 -0
- livekit/rtc/_proto/participant_pb2.py +45 -0
- livekit/rtc/_proto/participant_pb2.pyi +313 -0
- livekit/rtc/_proto/room_pb2.py +311 -0
- livekit/rtc/_proto/room_pb2.pyi +3136 -0
- livekit/rtc/_proto/rpc_pb2.py +47 -0
- livekit/rtc/_proto/rpc_pb2.pyi +261 -0
- livekit/rtc/_proto/stats_pb2.py +123 -0
- livekit/rtc/_proto/stats_pb2.pyi +1514 -0
- livekit/rtc/_proto/track_pb2.py +76 -0
- livekit/rtc/_proto/track_pb2.pyi +544 -0
- livekit/rtc/_proto/track_publication_pb2.py +39 -0
- livekit/rtc/_proto/track_publication_pb2.pyi +145 -0
- livekit/rtc/_proto/video_frame_pb2.py +97 -0
- livekit/rtc/_proto/video_frame_pb2.pyi +838 -0
- livekit/rtc/_utils.py +187 -0
- livekit/rtc/apm.py +119 -0
- livekit/rtc/audio_filter.py +23 -0
- livekit/rtc/audio_frame.py +242 -0
- livekit/rtc/audio_mixer.py +247 -0
- livekit/rtc/audio_resampler.py +174 -0
- livekit/rtc/audio_source.py +191 -0
- livekit/rtc/audio_stream.py +354 -0
- livekit/rtc/data_stream.py +640 -0
- livekit/rtc/data_track.py +296 -0
- livekit/rtc/e2ee.py +320 -0
- livekit/rtc/encoded_video.py +127 -0
- livekit/rtc/event_emitter.py +215 -0
- livekit/rtc/frame_processor.py +37 -0
- livekit/rtc/jupyter.py +68 -0
- livekit/rtc/log.py +3 -0
- livekit/rtc/media_devices.py +596 -0
- livekit/rtc/participant.py +1171 -0
- livekit/rtc/platform_audio.py +426 -0
- livekit/rtc/py.typed +0 -0
- livekit/rtc/resources/LICENSE.md +4055 -0
- livekit/rtc/resources/__init__.py +1 -0
- livekit/rtc/resources/jupyter-html/index.html +101 -0
- livekit/rtc/resources/livekit_ffi.dll +0 -0
- livekit/rtc/resources/livekit_ffi.h +42 -0
- livekit/rtc/room.py +1304 -0
- livekit/rtc/rpc.py +244 -0
- livekit/rtc/synchronizer.py +202 -0
- livekit/rtc/track.py +264 -0
- livekit/rtc/track_publication.py +145 -0
- livekit/rtc/transcription.py +19 -0
- livekit/rtc/utils.py +138 -0
- livekit/rtc/version.py +1 -0
- livekit/rtc/video_frame.py +347 -0
- livekit/rtc/video_source.py +74 -0
- livekit/rtc/video_stream.py +191 -0
- livekit-1.2.1.dist-info/METADATA +80 -0
- livekit-1.2.1.dist-info/RECORD +71 -0
- livekit-1.2.1.dist-info/WHEEL +5 -0
- livekit-1.2.1.dist-info/top_level.txt +1 -0
livekit/rtc/rpc.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Awaitable, Callable, ClassVar, Dict, Optional, Sequence, Union
|
|
18
|
+
from enum import IntEnum
|
|
19
|
+
from ._proto import rpc_pb2 as proto_rpc
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class RpcInvocationData:
|
|
25
|
+
"""Data passed to method handler for incoming RPC invocations
|
|
26
|
+
|
|
27
|
+
Attributes:
|
|
28
|
+
request_id (str): The unique request ID. Will match at both sides of the call, useful for debugging or logging.
|
|
29
|
+
caller_identity (str): The unique participant identity of the caller.
|
|
30
|
+
payload (str): The payload of the request. User-definable format, typically JSON.
|
|
31
|
+
response_timeout (float): The maximum time the caller will wait for a response.
|
|
32
|
+
method (str): The name of the invoked RPC method.
|
|
33
|
+
cancel_reason (Optional[RpcError.ErrorCode]): Why the SDK cancelled the handler chain,
|
|
34
|
+
set on this object just before it does: ``RESPONSE_TIMEOUT`` when the caller's
|
|
35
|
+
deadline passed, ``RECIPIENT_DISCONNECTED`` when the room disconnected. ``None``
|
|
36
|
+
while the chain runs, and for a ``CancelledError`` raised inside the chain (which
|
|
37
|
+
the caller receives as ``APPLICATION_ERROR``). Lets an interceptor unwinding from
|
|
38
|
+
the cancellation record the outcome the caller gets.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
request_id: str
|
|
42
|
+
caller_identity: str
|
|
43
|
+
payload: str
|
|
44
|
+
response_timeout: float
|
|
45
|
+
method: str = ""
|
|
46
|
+
cancel_reason: Optional[RpcError.ErrorCode] = None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class RpcCallInfo:
|
|
51
|
+
"""An outgoing RPC call, as passed to :meth:`RpcInterceptor.intercept_outgoing`.
|
|
52
|
+
|
|
53
|
+
Mirrors the arguments of :meth:`LocalParticipant.perform_rpc`. An interceptor may pass a
|
|
54
|
+
modified copy to ``next`` (for example to add a header to a JSON payload).
|
|
55
|
+
|
|
56
|
+
Attributes:
|
|
57
|
+
destination_identity (str): The identity of the participant being called.
|
|
58
|
+
method (str): The method name.
|
|
59
|
+
payload (str): The request payload.
|
|
60
|
+
response_timeout (Optional[float]): Seconds to wait for a response, or ``None`` for the default.
|
|
61
|
+
max_round_trip_latency (Optional[float]): See :meth:`LocalParticipant.perform_rpc`.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
destination_identity: str
|
|
65
|
+
method: str
|
|
66
|
+
payload: str
|
|
67
|
+
response_timeout: Optional[float] = None
|
|
68
|
+
max_round_trip_latency: Optional[float] = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
OutgoingRpcNext = Callable[["RpcCallInfo"], Awaitable[str]]
|
|
72
|
+
"""Continuation handed to :meth:`RpcInterceptor.intercept_outgoing`: performs the call."""
|
|
73
|
+
IncomingRpcNext = Callable[["RpcInvocationData"], Awaitable[Optional[str]]]
|
|
74
|
+
"""Continuation handed to :meth:`RpcInterceptor.intercept_incoming`: runs the handler."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class RpcInterceptor:
|
|
78
|
+
"""Observe or wrap RPC calls made and handled by a :class:`LocalParticipant`.
|
|
79
|
+
|
|
80
|
+
Register with :meth:`LocalParticipant.add_rpc_interceptor`. Each method receives the call
|
|
81
|
+
and a ``next`` continuation and must return (or raise) what ``next`` returns (or raises),
|
|
82
|
+
unless it deliberately short-circuits the call. Interceptors run in registration order:
|
|
83
|
+
the first one added is the outermost. Both methods default to pass-through, so override
|
|
84
|
+
only the direction you care about.
|
|
85
|
+
|
|
86
|
+
Errors flow through the chain unchanged: a :class:`RpcError` raised by the remote side
|
|
87
|
+
(outgoing) or by the handler (incoming) is visible to every interceptor before it
|
|
88
|
+
reaches the caller. On the incoming side, any other exception raised by the handler is
|
|
89
|
+
also visible; the SDK converts it to ``APPLICATION_ERROR`` only after the chain returns,
|
|
90
|
+
and a call for an unregistered method reaches the chain with ``next`` raising
|
|
91
|
+
``UNSUPPORTED_METHOD``. The caller's ``response_timeout`` covers the whole incoming
|
|
92
|
+
chain: when it passes, the chain is cancelled (interceptors see ``CancelledError``) and
|
|
93
|
+
the caller receives ``RESPONSE_TIMEOUT``.
|
|
94
|
+
|
|
95
|
+
Example:
|
|
96
|
+
Time every RPC in both directions::
|
|
97
|
+
|
|
98
|
+
class TimingInterceptor(rtc.RpcInterceptor):
|
|
99
|
+
async def intercept_outgoing(self, call, next):
|
|
100
|
+
start = time.perf_counter()
|
|
101
|
+
try:
|
|
102
|
+
return await next(call)
|
|
103
|
+
finally:
|
|
104
|
+
log("rpc call", call.method, time.perf_counter() - start)
|
|
105
|
+
|
|
106
|
+
async def intercept_incoming(self, invocation, next):
|
|
107
|
+
start = time.perf_counter()
|
|
108
|
+
try:
|
|
109
|
+
return await next(invocation)
|
|
110
|
+
finally:
|
|
111
|
+
log("rpc handled", invocation.method, time.perf_counter() - start)
|
|
112
|
+
|
|
113
|
+
room.local_participant.add_rpc_interceptor(TimingInterceptor())
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
async def intercept_outgoing(self, call: RpcCallInfo, next: OutgoingRpcNext) -> str:
|
|
117
|
+
"""Wrap an outgoing :meth:`LocalParticipant.perform_rpc`. Return the response payload."""
|
|
118
|
+
return await next(call)
|
|
119
|
+
|
|
120
|
+
async def intercept_incoming(
|
|
121
|
+
self, invocation: RpcInvocationData, next: IncomingRpcNext
|
|
122
|
+
) -> Optional[str]:
|
|
123
|
+
"""Wrap the handling of an incoming invocation. Return the response payload."""
|
|
124
|
+
return await next(invocation)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _chain_outgoing(
|
|
128
|
+
interceptors: Sequence[RpcInterceptor], terminal: OutgoingRpcNext
|
|
129
|
+
) -> OutgoingRpcNext:
|
|
130
|
+
"""Compose ``interceptors`` around ``terminal``; the first interceptor is outermost."""
|
|
131
|
+
call_next = terminal
|
|
132
|
+
for interceptor in reversed(interceptors):
|
|
133
|
+
call_next = _bind_outgoing(interceptor, call_next)
|
|
134
|
+
return call_next
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _chain_incoming(
|
|
138
|
+
interceptors: Sequence[RpcInterceptor], terminal: IncomingRpcNext
|
|
139
|
+
) -> IncomingRpcNext:
|
|
140
|
+
call_next = terminal
|
|
141
|
+
for interceptor in reversed(interceptors):
|
|
142
|
+
call_next = _bind_incoming(interceptor, call_next)
|
|
143
|
+
return call_next
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _bind_outgoing(interceptor: RpcInterceptor, call_next: OutgoingRpcNext) -> OutgoingRpcNext:
|
|
147
|
+
async def bound(call: RpcCallInfo) -> str:
|
|
148
|
+
return await interceptor.intercept_outgoing(call, call_next)
|
|
149
|
+
|
|
150
|
+
return bound
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _bind_incoming(interceptor: RpcInterceptor, call_next: IncomingRpcNext) -> IncomingRpcNext:
|
|
154
|
+
async def bound(invocation: RpcInvocationData) -> Optional[str]:
|
|
155
|
+
return await interceptor.intercept_incoming(invocation, call_next)
|
|
156
|
+
|
|
157
|
+
return bound
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class RpcError(Exception):
|
|
161
|
+
"""
|
|
162
|
+
Specialized error handling for RPC methods.
|
|
163
|
+
|
|
164
|
+
Instances of this type, when thrown in a method handler, will have their `message`
|
|
165
|
+
serialized and sent across the wire. The caller will receive an equivalent error on the other side.
|
|
166
|
+
|
|
167
|
+
Built-in errors are included (codes 1001-1999) but developers may use the code, message, and data fields to create their own errors.
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
class ErrorCode(IntEnum):
|
|
171
|
+
APPLICATION_ERROR = 1500
|
|
172
|
+
CONNECTION_TIMEOUT = 1501
|
|
173
|
+
RESPONSE_TIMEOUT = 1502
|
|
174
|
+
RECIPIENT_DISCONNECTED = 1503
|
|
175
|
+
RESPONSE_PAYLOAD_TOO_LARGE = 1504
|
|
176
|
+
SEND_FAILED = 1505
|
|
177
|
+
|
|
178
|
+
UNSUPPORTED_METHOD = 1400
|
|
179
|
+
RECIPIENT_NOT_FOUND = 1401
|
|
180
|
+
REQUEST_PAYLOAD_TOO_LARGE = 1402
|
|
181
|
+
UNSUPPORTED_SERVER = 1403
|
|
182
|
+
UNSUPPORTED_VERSION = 1404
|
|
183
|
+
|
|
184
|
+
ErrorMessage: ClassVar[Dict[ErrorCode, str]] = {
|
|
185
|
+
ErrorCode.APPLICATION_ERROR: "Application error in method handler",
|
|
186
|
+
ErrorCode.CONNECTION_TIMEOUT: "Connection timeout",
|
|
187
|
+
ErrorCode.RESPONSE_TIMEOUT: "Response timeout",
|
|
188
|
+
ErrorCode.RECIPIENT_DISCONNECTED: "Recipient disconnected",
|
|
189
|
+
ErrorCode.RESPONSE_PAYLOAD_TOO_LARGE: "Response payload too large",
|
|
190
|
+
ErrorCode.SEND_FAILED: "Failed to send",
|
|
191
|
+
ErrorCode.UNSUPPORTED_METHOD: "Method not supported at destination",
|
|
192
|
+
ErrorCode.RECIPIENT_NOT_FOUND: "Recipient not found",
|
|
193
|
+
ErrorCode.REQUEST_PAYLOAD_TOO_LARGE: "Request payload too large",
|
|
194
|
+
ErrorCode.UNSUPPORTED_SERVER: "RPC not supported by server",
|
|
195
|
+
ErrorCode.UNSUPPORTED_VERSION: "Unsupported RPC version",
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
def __init__(
|
|
199
|
+
self,
|
|
200
|
+
code: Union[int, "RpcError.ErrorCode"],
|
|
201
|
+
message: str,
|
|
202
|
+
data: Optional[str] = None,
|
|
203
|
+
):
|
|
204
|
+
"""
|
|
205
|
+
Creates an error object with the given code and message, plus an optional data payload.
|
|
206
|
+
|
|
207
|
+
If thrown in an RPC method handler, the error will be sent back to the caller.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
code (int): Your error code (Error codes 1001-1999 are reserved for built-in errors)
|
|
211
|
+
message (str): A readable error message.
|
|
212
|
+
data (Optional[str]): Optional additional data associated with the error (JSON recommended)
|
|
213
|
+
"""
|
|
214
|
+
super().__init__(message)
|
|
215
|
+
self._code = code
|
|
216
|
+
self._message = message
|
|
217
|
+
self._data = data
|
|
218
|
+
|
|
219
|
+
@property
|
|
220
|
+
def code(self) -> int:
|
|
221
|
+
"""Error code value. Codes 1001-1999 are reserved for built-in errors (see RpcError.ErrorCode for their meanings)."""
|
|
222
|
+
return self._code
|
|
223
|
+
|
|
224
|
+
@property
|
|
225
|
+
def message(self) -> str:
|
|
226
|
+
"""A readable error message."""
|
|
227
|
+
return self._message
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def data(self) -> Optional[str]:
|
|
231
|
+
"""Optional additional data associated with the error (JSON recommended)."""
|
|
232
|
+
return self._data
|
|
233
|
+
|
|
234
|
+
@classmethod
|
|
235
|
+
def _from_proto(cls, proto: proto_rpc.RpcError) -> "RpcError":
|
|
236
|
+
return cls(proto.code, proto.message, proto.data)
|
|
237
|
+
|
|
238
|
+
def _to_proto(self) -> proto_rpc.RpcError:
|
|
239
|
+
return proto_rpc.RpcError(code=self.code, message=self.message, data=self.data)
|
|
240
|
+
|
|
241
|
+
@classmethod
|
|
242
|
+
def _built_in(cls, code: "RpcError.ErrorCode", data: Optional[str] = None) -> "RpcError":
|
|
243
|
+
message = cls.ErrorMessage[code]
|
|
244
|
+
return cls(code, message, data)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
import time
|
|
4
|
+
from collections import deque
|
|
5
|
+
from typing import Any, Optional, Union
|
|
6
|
+
|
|
7
|
+
from .video_frame import VideoFrame
|
|
8
|
+
from .audio_frame import AudioFrame
|
|
9
|
+
from .audio_source import AudioSource
|
|
10
|
+
from .video_source import VideoSource
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AVSynchronizer:
|
|
17
|
+
"""Synchronize audio and video capture.
|
|
18
|
+
|
|
19
|
+
Usage:
|
|
20
|
+
av_sync = AVSynchronizer(
|
|
21
|
+
audio_source=audio_source,
|
|
22
|
+
video_source=video_source,
|
|
23
|
+
video_fps=video_fps,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
async for video_frame, audio_frame in video_generator:
|
|
27
|
+
await av_sync.push(video_frame)
|
|
28
|
+
await av_sync.push(audio_frame)
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
*,
|
|
34
|
+
audio_source: AudioSource,
|
|
35
|
+
video_source: VideoSource,
|
|
36
|
+
video_fps: float,
|
|
37
|
+
video_queue_size_ms: float = 100,
|
|
38
|
+
_max_delay_tolerance_ms: float = 300,
|
|
39
|
+
):
|
|
40
|
+
self._audio_source = audio_source
|
|
41
|
+
self._video_source = video_source
|
|
42
|
+
self._video_fps = video_fps
|
|
43
|
+
self._video_queue_size_ms = video_queue_size_ms
|
|
44
|
+
self._max_delay_tolerance_ms = _max_delay_tolerance_ms
|
|
45
|
+
|
|
46
|
+
self._stopped = False
|
|
47
|
+
# the time of the last video/audio frame captured
|
|
48
|
+
self._last_video_time: float = 0
|
|
49
|
+
self._last_audio_time: float = 0
|
|
50
|
+
|
|
51
|
+
self._video_queue_max_size = int(self._video_fps * self._video_queue_size_ms / 1000)
|
|
52
|
+
if self._video_queue_size_ms > 0:
|
|
53
|
+
# ensure queue is bounded if queue size is specified
|
|
54
|
+
self._video_queue_max_size = max(1, self._video_queue_max_size)
|
|
55
|
+
|
|
56
|
+
self._video_queue = asyncio.Queue[tuple[VideoFrame, Optional[float]]](
|
|
57
|
+
maxsize=self._video_queue_max_size
|
|
58
|
+
)
|
|
59
|
+
self._fps_controller = _FPSController(
|
|
60
|
+
expected_fps=self._video_fps,
|
|
61
|
+
max_delay_tolerance_ms=self._max_delay_tolerance_ms,
|
|
62
|
+
)
|
|
63
|
+
self._capture_video_task = asyncio.create_task(self._capture_video())
|
|
64
|
+
|
|
65
|
+
async def push(
|
|
66
|
+
self, frame: Union[VideoFrame, AudioFrame], timestamp: Optional[float] = None
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Push a frame to the synchronizer
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
frame: The video or audio frame to push.
|
|
72
|
+
timestamp: (optional) The timestamp of the frame, for logging purposes for now.
|
|
73
|
+
For AudioFrame, it should be the end time of the frame.
|
|
74
|
+
"""
|
|
75
|
+
if isinstance(frame, AudioFrame):
|
|
76
|
+
await self._audio_source.capture_frame(frame)
|
|
77
|
+
if timestamp is not None:
|
|
78
|
+
self._last_audio_time = timestamp
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
await self._video_queue.put((frame, timestamp))
|
|
82
|
+
|
|
83
|
+
async def clear_queue(self) -> None:
|
|
84
|
+
self._audio_source.clear_queue()
|
|
85
|
+
while not self._video_queue.empty():
|
|
86
|
+
await self._video_queue.get()
|
|
87
|
+
self._video_queue.task_done()
|
|
88
|
+
|
|
89
|
+
async def wait_for_playout(self) -> None:
|
|
90
|
+
"""Wait until all video and audio frames are played out."""
|
|
91
|
+
await asyncio.gather(
|
|
92
|
+
self._audio_source.wait_for_playout(),
|
|
93
|
+
self._video_queue.join(),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def reset(self) -> None:
|
|
97
|
+
self._fps_controller.reset()
|
|
98
|
+
|
|
99
|
+
async def _capture_video(self) -> None:
|
|
100
|
+
while not self._stopped:
|
|
101
|
+
frame, timestamp = await self._video_queue.get()
|
|
102
|
+
async with self._fps_controller:
|
|
103
|
+
self._video_source.capture_frame(frame)
|
|
104
|
+
if timestamp is not None:
|
|
105
|
+
self._last_video_time = timestamp
|
|
106
|
+
self._video_queue.task_done()
|
|
107
|
+
|
|
108
|
+
async def aclose(self) -> None:
|
|
109
|
+
self._stopped = True
|
|
110
|
+
if self._capture_video_task:
|
|
111
|
+
self._capture_video_task.cancel()
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def actual_fps(self) -> float:
|
|
115
|
+
return self._fps_controller.actual_fps
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def last_video_time(self) -> float:
|
|
119
|
+
"""The time of the last video frame captured"""
|
|
120
|
+
return self._last_video_time
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def last_audio_time(self) -> float:
|
|
124
|
+
"""The time of the last audio frame played out"""
|
|
125
|
+
return self._last_audio_time - self._audio_source.queued_duration
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class _FPSController:
|
|
129
|
+
def __init__(self, *, expected_fps: float, max_delay_tolerance_ms: float = 300) -> None:
|
|
130
|
+
"""Controls frame rate by adjusting sleep time based on actual FPS.
|
|
131
|
+
|
|
132
|
+
Usage:
|
|
133
|
+
async with _FPSController(expected_fps=30):
|
|
134
|
+
# process frame
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
expected_fps: Target frames per second
|
|
139
|
+
max_delay_tolerance_ms: Maximum delay tolerance in milliseconds
|
|
140
|
+
"""
|
|
141
|
+
self._expected_fps = expected_fps
|
|
142
|
+
self._frame_interval = 1.0 / expected_fps
|
|
143
|
+
self._max_delay_tolerance_secs = max_delay_tolerance_ms / 1000
|
|
144
|
+
|
|
145
|
+
self._next_frame_time: Optional[float] = None
|
|
146
|
+
self._fps_calc_winsize = max(2, int(1.0 * expected_fps))
|
|
147
|
+
self._send_timestamps: deque[float] = deque(maxlen=self._fps_calc_winsize)
|
|
148
|
+
|
|
149
|
+
async def __aenter__(self) -> None:
|
|
150
|
+
await self.wait_next_process()
|
|
151
|
+
|
|
152
|
+
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
153
|
+
self.after_process()
|
|
154
|
+
|
|
155
|
+
def reset(self) -> None:
|
|
156
|
+
self._next_frame_time = None
|
|
157
|
+
self._send_timestamps.clear()
|
|
158
|
+
|
|
159
|
+
async def wait_next_process(self) -> None:
|
|
160
|
+
"""Wait until it's time for the next frame.
|
|
161
|
+
|
|
162
|
+
Adjusts sleep time based on actual FPS to maintain target rate.
|
|
163
|
+
"""
|
|
164
|
+
current_time = time.perf_counter()
|
|
165
|
+
|
|
166
|
+
# initialize the next frame time
|
|
167
|
+
if self._next_frame_time is None:
|
|
168
|
+
self._next_frame_time = current_time
|
|
169
|
+
|
|
170
|
+
# calculate sleep time
|
|
171
|
+
sleep_time = self._next_frame_time - current_time
|
|
172
|
+
if sleep_time > 0:
|
|
173
|
+
await asyncio.sleep(sleep_time)
|
|
174
|
+
else:
|
|
175
|
+
# check if significantly behind schedule
|
|
176
|
+
if -sleep_time > self._max_delay_tolerance_secs:
|
|
177
|
+
logger.warning(f"Frame capture was behind schedule for {-sleep_time * 1000:.2f} ms")
|
|
178
|
+
self._next_frame_time = time.perf_counter()
|
|
179
|
+
|
|
180
|
+
def after_process(self) -> None:
|
|
181
|
+
"""Update timing information after processing a frame."""
|
|
182
|
+
assert self._next_frame_time is not None, "wait_next_process must be called first"
|
|
183
|
+
|
|
184
|
+
# update timing information
|
|
185
|
+
self._send_timestamps.append(time.perf_counter())
|
|
186
|
+
|
|
187
|
+
# calculate next frame time
|
|
188
|
+
self._next_frame_time += self._frame_interval
|
|
189
|
+
|
|
190
|
+
@property
|
|
191
|
+
def expected_fps(self) -> float:
|
|
192
|
+
return self._expected_fps
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def actual_fps(self) -> float:
|
|
196
|
+
"""Get current average FPS."""
|
|
197
|
+
if len(self._send_timestamps) < 2:
|
|
198
|
+
return 0
|
|
199
|
+
|
|
200
|
+
return (len(self._send_timestamps) - 1) / (
|
|
201
|
+
self._send_timestamps[-1] - self._send_timestamps[0]
|
|
202
|
+
)
|