livekit 1.2.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit/rtc/__init__.py +264 -0
- livekit/rtc/_ffi_client.py +321 -0
- livekit/rtc/_proto/__init__.py +0 -0
- livekit/rtc/_proto/audio_frame_pb2.py +159 -0
- livekit/rtc/_proto/audio_frame_pb2.pyi +1560 -0
- livekit/rtc/_proto/capture_pb2.py +70 -0
- livekit/rtc/_proto/capture_pb2.pyi +559 -0
- livekit/rtc/_proto/data_stream_pb2.py +171 -0
- livekit/rtc/_proto/data_stream_pb2.pyi +1498 -0
- livekit/rtc/_proto/data_track_pb2.py +120 -0
- livekit/rtc/_proto/data_track_pb2.pyi +1073 -0
- livekit/rtc/_proto/e2ee_pb2.py +81 -0
- livekit/rtc/_proto/e2ee_pb2.pyi +591 -0
- livekit/rtc/_proto/ffi_pb2.py +55 -0
- livekit/rtc/_proto/ffi_pb2.pyi +1246 -0
- livekit/rtc/_proto/handle_pb2.py +27 -0
- livekit/rtc/_proto/handle_pb2.pyi +51 -0
- livekit/rtc/_proto/participant_pb2.py +45 -0
- livekit/rtc/_proto/participant_pb2.pyi +313 -0
- livekit/rtc/_proto/room_pb2.py +311 -0
- livekit/rtc/_proto/room_pb2.pyi +3136 -0
- livekit/rtc/_proto/rpc_pb2.py +47 -0
- livekit/rtc/_proto/rpc_pb2.pyi +261 -0
- livekit/rtc/_proto/stats_pb2.py +123 -0
- livekit/rtc/_proto/stats_pb2.pyi +1514 -0
- livekit/rtc/_proto/track_pb2.py +76 -0
- livekit/rtc/_proto/track_pb2.pyi +544 -0
- livekit/rtc/_proto/track_publication_pb2.py +39 -0
- livekit/rtc/_proto/track_publication_pb2.pyi +145 -0
- livekit/rtc/_proto/video_frame_pb2.py +97 -0
- livekit/rtc/_proto/video_frame_pb2.pyi +838 -0
- livekit/rtc/_utils.py +187 -0
- livekit/rtc/apm.py +119 -0
- livekit/rtc/audio_filter.py +23 -0
- livekit/rtc/audio_frame.py +242 -0
- livekit/rtc/audio_mixer.py +247 -0
- livekit/rtc/audio_resampler.py +174 -0
- livekit/rtc/audio_source.py +191 -0
- livekit/rtc/audio_stream.py +354 -0
- livekit/rtc/data_stream.py +640 -0
- livekit/rtc/data_track.py +296 -0
- livekit/rtc/e2ee.py +320 -0
- livekit/rtc/encoded_video.py +127 -0
- livekit/rtc/event_emitter.py +215 -0
- livekit/rtc/frame_processor.py +37 -0
- livekit/rtc/jupyter.py +68 -0
- livekit/rtc/log.py +3 -0
- livekit/rtc/media_devices.py +596 -0
- livekit/rtc/participant.py +1171 -0
- livekit/rtc/platform_audio.py +426 -0
- livekit/rtc/py.typed +0 -0
- livekit/rtc/resources/LICENSE.md +4055 -0
- livekit/rtc/resources/__init__.py +1 -0
- livekit/rtc/resources/jupyter-html/index.html +101 -0
- livekit/rtc/resources/livekit_ffi.dll +0 -0
- livekit/rtc/resources/livekit_ffi.h +42 -0
- livekit/rtc/room.py +1304 -0
- livekit/rtc/rpc.py +244 -0
- livekit/rtc/synchronizer.py +202 -0
- livekit/rtc/track.py +264 -0
- livekit/rtc/track_publication.py +145 -0
- livekit/rtc/transcription.py +19 -0
- livekit/rtc/utils.py +138 -0
- livekit/rtc/version.py +1 -0
- livekit/rtc/video_frame.py +347 -0
- livekit/rtc/video_source.py +74 -0
- livekit/rtc/video_stream.py +191 -0
- livekit-1.2.1.dist-info/METADATA +80 -0
- livekit-1.2.1.dist-info/RECORD +71 -0
- livekit-1.2.1.dist-info/WHEEL +5 -0
- livekit-1.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import ctypes
|
|
16
|
+
from typing import Union
|
|
17
|
+
from ._proto import video_frame_pb2 as proto_video
|
|
18
|
+
from ._proto import ffi_pb2 as proto
|
|
19
|
+
from typing import List, Optional
|
|
20
|
+
from ._ffi_client import FfiClient, FfiHandle
|
|
21
|
+
from ._utils import _ensure_compatible_buffer, get_address
|
|
22
|
+
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class VideoFrame:
|
|
27
|
+
"""
|
|
28
|
+
Represents a video frame with associated metadata and pixel data.
|
|
29
|
+
|
|
30
|
+
This class provides methods to access video frame properties such as width, height,
|
|
31
|
+
and pixel format, as well as methods for manipulating and converting video frames.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
width: int,
|
|
37
|
+
height: int,
|
|
38
|
+
type: proto_video.VideoBufferType.ValueType,
|
|
39
|
+
data: Union[bytes, bytearray, memoryview],
|
|
40
|
+
) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Initializes a new VideoFrame instance.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
width (int): The width of the video frame in pixels.
|
|
46
|
+
height (int): The height of the video frame in pixels.
|
|
47
|
+
type (proto_video.VideoBufferType.ValueType): The format type of the video frame data
|
|
48
|
+
(e.g., RGBA, BGRA, RGB24, etc.).
|
|
49
|
+
data (Union[bytes, bytearray, memoryview]): The raw pixel data for the video frame.
|
|
50
|
+
"""
|
|
51
|
+
data = _ensure_compatible_buffer(data)
|
|
52
|
+
|
|
53
|
+
self._width = width
|
|
54
|
+
self._height = height
|
|
55
|
+
self._type = type
|
|
56
|
+
self._data = data
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def width(self) -> int:
|
|
60
|
+
"""
|
|
61
|
+
Returns the width of the video frame in pixels.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
int: The width of the video frame.
|
|
65
|
+
"""
|
|
66
|
+
return self._width
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def height(self) -> int:
|
|
70
|
+
"""
|
|
71
|
+
Returns the height of the video frame in pixels.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
int: The height of the video frame.
|
|
75
|
+
"""
|
|
76
|
+
return self._height
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def type(self) -> proto_video.VideoBufferType.ValueType:
|
|
80
|
+
"""
|
|
81
|
+
Returns the height of the video frame in pixels.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
int: The height of the video frame.
|
|
85
|
+
"""
|
|
86
|
+
return self._type
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def data(self) -> memoryview:
|
|
90
|
+
"""
|
|
91
|
+
Returns a memoryview of the raw pixel data for the video frame.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
memoryview: The raw pixel data of the video frame as a memoryview object.
|
|
95
|
+
"""
|
|
96
|
+
return memoryview(self._data)
|
|
97
|
+
|
|
98
|
+
@staticmethod
|
|
99
|
+
def _from_owned_info(owned_info: proto_video.OwnedVideoBuffer) -> "VideoFrame":
|
|
100
|
+
info = owned_info.info
|
|
101
|
+
data_len = _get_plane_length(info.type, info.width, info.height)
|
|
102
|
+
cdata = (ctypes.c_uint8 * data_len).from_address(info.data_ptr)
|
|
103
|
+
data = bytearray(cdata)
|
|
104
|
+
frame = VideoFrame(
|
|
105
|
+
width=info.width,
|
|
106
|
+
height=info.height,
|
|
107
|
+
type=info.type,
|
|
108
|
+
data=data,
|
|
109
|
+
)
|
|
110
|
+
FfiHandle(owned_info.handle.id).dispose()
|
|
111
|
+
return frame
|
|
112
|
+
|
|
113
|
+
def _proto_info(self) -> proto_video.VideoBufferInfo:
|
|
114
|
+
info = proto_video.VideoBufferInfo()
|
|
115
|
+
addr = get_address(self.data)
|
|
116
|
+
info.components.extend(_get_plane_infos(addr, self.type, self.width, self.height))
|
|
117
|
+
info.width = self.width
|
|
118
|
+
info.height = self.height
|
|
119
|
+
info.type = self.type
|
|
120
|
+
info.data_ptr = addr
|
|
121
|
+
info.stride = 0
|
|
122
|
+
|
|
123
|
+
if self.type in [
|
|
124
|
+
proto_video.VideoBufferType.ARGB,
|
|
125
|
+
proto_video.VideoBufferType.ABGR,
|
|
126
|
+
proto_video.VideoBufferType.RGBA,
|
|
127
|
+
proto_video.VideoBufferType.BGRA,
|
|
128
|
+
]:
|
|
129
|
+
info.stride = self.width * 4
|
|
130
|
+
elif self.type == proto_video.VideoBufferType.RGB24:
|
|
131
|
+
info.stride = self.width * 3
|
|
132
|
+
|
|
133
|
+
return info
|
|
134
|
+
|
|
135
|
+
def get_plane(self, plane_nth: int) -> Optional[memoryview]:
|
|
136
|
+
"""
|
|
137
|
+
Returns the memoryview of a specific plane in the video frame, based on its index.
|
|
138
|
+
|
|
139
|
+
Some video formats (e.g., I420, NV12) contain multiple planes (Y, U, V channels).
|
|
140
|
+
This method allows access to individual planes by index.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
plane_nth (int): The index of the plane to retrieve (starting from 0).
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
Optional[memoryview]: A memoryview of the specified plane's data, or None if
|
|
147
|
+
the index is out of bounds for the format.
|
|
148
|
+
"""
|
|
149
|
+
plane_infos = _get_plane_infos(get_address(self.data), self.type, self.width, self.height)
|
|
150
|
+
if plane_nth >= len(plane_infos):
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
plane_info = plane_infos[plane_nth]
|
|
154
|
+
cdata = (ctypes.c_uint8 * plane_info.size).from_address(plane_info.data_ptr)
|
|
155
|
+
return memoryview(cdata)
|
|
156
|
+
|
|
157
|
+
def convert(
|
|
158
|
+
self, type: proto_video.VideoBufferType.ValueType, *, flip_y: bool = False
|
|
159
|
+
) -> "VideoFrame":
|
|
160
|
+
"""
|
|
161
|
+
Converts the current video frame to a different format type, optionally flipping
|
|
162
|
+
the frame vertically.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
type (proto_video.VideoBufferType.ValueType): The target format type to convert to
|
|
166
|
+
(e.g., RGBA, I420).
|
|
167
|
+
flip_y (bool, optional): If True, the frame will be flipped vertically. Defaults to False.
|
|
168
|
+
|
|
169
|
+
Returns:
|
|
170
|
+
VideoFrame: A new VideoFrame object in the specified format.
|
|
171
|
+
|
|
172
|
+
Raises:
|
|
173
|
+
Exception: If the conversion isn't supported.
|
|
174
|
+
|
|
175
|
+
Example:
|
|
176
|
+
Convert a frame from RGBA to I420 format:
|
|
177
|
+
|
|
178
|
+
>>> frame = VideoFrame(width=1920, height=1080, type=proto_video.VideoBufferType.RGBA, data=raw_data)
|
|
179
|
+
>>> converted_frame = frame.convert(proto_video.VideoBufferType.I420)
|
|
180
|
+
>>> print(converted_frame.type)
|
|
181
|
+
VideoBufferType.I420
|
|
182
|
+
|
|
183
|
+
Example:
|
|
184
|
+
Convert a frame from BGRA to RGB24 format and flip it vertically:
|
|
185
|
+
|
|
186
|
+
>>> frame = VideoFrame(width=1280, height=720, type=proto_video.VideoBufferType.BGRA, data=raw_data)
|
|
187
|
+
>>> converted_frame = frame.convert(proto_video.VideoBufferType.RGB24, flip_y=True)
|
|
188
|
+
>>> print(converted_frame.type)
|
|
189
|
+
VideoBufferType.RGB24
|
|
190
|
+
>>> print(converted_frame.width, converted_frame.height)
|
|
191
|
+
1280 720
|
|
192
|
+
"""
|
|
193
|
+
req = proto.FfiRequest()
|
|
194
|
+
req.video_convert.flip_y = flip_y
|
|
195
|
+
req.video_convert.dst_type = type
|
|
196
|
+
req.video_convert.buffer.CopyFrom(self._proto_info())
|
|
197
|
+
resp = FfiClient.instance.request(req)
|
|
198
|
+
if resp.video_convert.error:
|
|
199
|
+
raise Exception(resp.video_convert.error)
|
|
200
|
+
|
|
201
|
+
return VideoFrame._from_owned_info(resp.video_convert.buffer)
|
|
202
|
+
|
|
203
|
+
def __repr__(self) -> str:
|
|
204
|
+
return f"rtc.VideoFrame(width={self.width}, height={self.height}, type={proto_video.VideoBufferType.Name(self.type)})"
|
|
205
|
+
|
|
206
|
+
@classmethod
|
|
207
|
+
def __get_pydantic_core_schema__(cls, *_: Any) -> Any:
|
|
208
|
+
from pydantic_core import core_schema
|
|
209
|
+
import base64
|
|
210
|
+
|
|
211
|
+
def validate_video_frame(value: Any) -> "VideoFrame":
|
|
212
|
+
if isinstance(value, VideoFrame):
|
|
213
|
+
return value
|
|
214
|
+
|
|
215
|
+
if isinstance(value, tuple):
|
|
216
|
+
value = value[0]
|
|
217
|
+
|
|
218
|
+
if isinstance(value, dict):
|
|
219
|
+
return VideoFrame(
|
|
220
|
+
width=value["width"],
|
|
221
|
+
height=value["height"],
|
|
222
|
+
type=proto_video.VideoBufferType.Value(value["type"]),
|
|
223
|
+
data=base64.b64decode(value["data"]),
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
raise TypeError("Invalid type for VideoFrame")
|
|
227
|
+
|
|
228
|
+
return core_schema.json_or_python_schema(
|
|
229
|
+
json_schema=core_schema.chain_schema(
|
|
230
|
+
[
|
|
231
|
+
core_schema.model_fields_schema(
|
|
232
|
+
{
|
|
233
|
+
"width": core_schema.model_field(core_schema.int_schema()),
|
|
234
|
+
"height": core_schema.model_field(core_schema.int_schema()),
|
|
235
|
+
"type": core_schema.model_field(core_schema.str_schema()),
|
|
236
|
+
"data": core_schema.model_field(core_schema.str_schema()),
|
|
237
|
+
},
|
|
238
|
+
),
|
|
239
|
+
core_schema.no_info_plain_validator_function(validate_video_frame),
|
|
240
|
+
]
|
|
241
|
+
),
|
|
242
|
+
python_schema=core_schema.no_info_plain_validator_function(validate_video_frame),
|
|
243
|
+
serialization=core_schema.plain_serializer_function_ser_schema(
|
|
244
|
+
lambda instance: {
|
|
245
|
+
"width": instance.width,
|
|
246
|
+
"height": instance.height,
|
|
247
|
+
"type": proto_video.VideoBufferType.Name(instance.type),
|
|
248
|
+
"data": base64.b64encode(instance.data).decode("utf-8"),
|
|
249
|
+
}
|
|
250
|
+
),
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _component_info(
|
|
255
|
+
data_ptr: int, stride: int, size: int
|
|
256
|
+
) -> proto_video.VideoBufferInfo.ComponentInfo:
|
|
257
|
+
cmpt = proto_video.VideoBufferInfo.ComponentInfo()
|
|
258
|
+
cmpt.data_ptr = data_ptr
|
|
259
|
+
cmpt.stride = stride
|
|
260
|
+
cmpt.size = size
|
|
261
|
+
return cmpt
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _get_plane_length(type: proto_video.VideoBufferType.ValueType, width: int, height: int) -> int:
|
|
265
|
+
"""
|
|
266
|
+
Return the size in bytes of a participant video buffer type based on its size (This ignores the strides)
|
|
267
|
+
"""
|
|
268
|
+
if type in [
|
|
269
|
+
proto_video.VideoBufferType.ARGB,
|
|
270
|
+
proto_video.VideoBufferType.ABGR,
|
|
271
|
+
proto_video.VideoBufferType.RGBA,
|
|
272
|
+
proto_video.VideoBufferType.BGRA,
|
|
273
|
+
]:
|
|
274
|
+
return width * height * 4
|
|
275
|
+
elif type == proto_video.VideoBufferType.RGB24:
|
|
276
|
+
return width * height * 3
|
|
277
|
+
elif type == proto_video.VideoBufferType.I420:
|
|
278
|
+
chroma_width = (width + 1) // 2
|
|
279
|
+
chroma_height = (height + 1) // 2
|
|
280
|
+
return width * height + chroma_width * chroma_height * 2
|
|
281
|
+
elif type == proto_video.VideoBufferType.I420A:
|
|
282
|
+
chroma_width = (width + 1) // 2
|
|
283
|
+
chroma_height = (height + 1) // 2
|
|
284
|
+
return width * height * 2 + chroma_width * chroma_height * 2
|
|
285
|
+
elif type == proto_video.VideoBufferType.I422:
|
|
286
|
+
chroma_width = (width + 1) // 2
|
|
287
|
+
return width * height + chroma_width * height * 2
|
|
288
|
+
elif type == proto_video.VideoBufferType.I444:
|
|
289
|
+
return width * height * 3
|
|
290
|
+
elif type == proto_video.VideoBufferType.I010:
|
|
291
|
+
chroma_width = (width + 1) // 2
|
|
292
|
+
chroma_height = (height + 1) // 2
|
|
293
|
+
return width * height * 2 + chroma_width * chroma_height * 4
|
|
294
|
+
elif type == proto_video.VideoBufferType.NV12:
|
|
295
|
+
chroma_width = (width + 1) // 2
|
|
296
|
+
chroma_height = (height + 1) // 2
|
|
297
|
+
return width * height + chroma_width * chroma_height * 2
|
|
298
|
+
|
|
299
|
+
raise Exception(f"unsupported video buffer type: {type}")
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _get_plane_infos(
|
|
303
|
+
addr: int, type: proto_video.VideoBufferType.ValueType, width: int, height: int
|
|
304
|
+
) -> List[proto_video.VideoBufferInfo.ComponentInfo]:
|
|
305
|
+
if type == proto_video.VideoBufferType.I420:
|
|
306
|
+
chroma_width = (width + 1) // 2
|
|
307
|
+
chroma_height = (height + 1) // 2
|
|
308
|
+
y = _component_info(addr, width, width * height)
|
|
309
|
+
u = _component_info(y.data_ptr + y.size, chroma_width, chroma_width * chroma_height)
|
|
310
|
+
v = _component_info(u.data_ptr + u.size, chroma_width, chroma_width * chroma_height)
|
|
311
|
+
return [y, u, v]
|
|
312
|
+
elif type == proto_video.VideoBufferType.I420A:
|
|
313
|
+
chroma_width = (width + 1) // 2
|
|
314
|
+
chroma_height = (height + 1) // 2
|
|
315
|
+
y = _component_info(addr, width, width * height)
|
|
316
|
+
u = _component_info(y.data_ptr + y.size, chroma_width, chroma_width * chroma_height)
|
|
317
|
+
v = _component_info(u.data_ptr + u.size, chroma_width, chroma_width * chroma_height)
|
|
318
|
+
a = _component_info(v.data_ptr + v.size, width, width * height)
|
|
319
|
+
return [y, u, v, a]
|
|
320
|
+
elif type == proto_video.VideoBufferType.I422:
|
|
321
|
+
chroma_width = (width + 1) // 2
|
|
322
|
+
y = _component_info(addr, width, width * height)
|
|
323
|
+
u = _component_info(y.data_ptr + y.size, chroma_width, chroma_width * height)
|
|
324
|
+
v = _component_info(u.data_ptr + u.size, chroma_width, chroma_width * height)
|
|
325
|
+
return [y, u, v]
|
|
326
|
+
elif type == proto_video.VideoBufferType.I444:
|
|
327
|
+
y = _component_info(addr, width, width * height)
|
|
328
|
+
u = _component_info(y.data_ptr + y.size, width, width * height)
|
|
329
|
+
v = _component_info(u.data_ptr + u.size, width, width * height)
|
|
330
|
+
return [y, u, v]
|
|
331
|
+
elif type == proto_video.VideoBufferType.I010:
|
|
332
|
+
chroma_width = (width + 1) // 2
|
|
333
|
+
chroma_height = (height + 1) // 2
|
|
334
|
+
y = _component_info(addr, width * 2, width * height * 2)
|
|
335
|
+
u = _component_info(y.data_ptr + y.size, chroma_width * 2, chroma_width * chroma_height * 2)
|
|
336
|
+
v = _component_info(u.data_ptr + u.size, chroma_width * 2, chroma_width * chroma_height * 2)
|
|
337
|
+
return [y, u, v]
|
|
338
|
+
elif type == proto_video.VideoBufferType.NV12:
|
|
339
|
+
chroma_width = (width + 1) // 2
|
|
340
|
+
chroma_height = (height + 1) // 2
|
|
341
|
+
y = _component_info(addr, width, width * height)
|
|
342
|
+
uv = _component_info(
|
|
343
|
+
y.data_ptr + y.size, chroma_width * 2, chroma_width * chroma_height * 2
|
|
344
|
+
)
|
|
345
|
+
return [y, uv]
|
|
346
|
+
|
|
347
|
+
return []
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from ._ffi_client import FfiHandle, FfiClient
|
|
18
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
19
|
+
from ._proto import video_frame_pb2 as proto_video
|
|
20
|
+
from .video_frame import VideoFrame
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class VideoSource:
|
|
24
|
+
def __init__(self, width: int, height: int, *, is_screencast: bool = False) -> None:
|
|
25
|
+
"""
|
|
26
|
+
Create a new video source.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
width (int): Initial width of the video source.
|
|
30
|
+
height (int): Initial height of the video source.
|
|
31
|
+
is_screencast (bool, optional): Optimize the WebRTC pipeline for screen content.
|
|
32
|
+
Defaults to False.
|
|
33
|
+
|
|
34
|
+
When True, WebRTC will:
|
|
35
|
+
|
|
36
|
+
- Maintain resolution under congestion by dropping frames instead of
|
|
37
|
+
downscaling (keeps text crisp)
|
|
38
|
+
- Disable quality scaling and denoising to preserve text/UI readability
|
|
39
|
+
- Guarantee a minimum 1200 kbps bitrate floor
|
|
40
|
+
- Enable zero-hertz mode, stopping frame transmission when the screen
|
|
41
|
+
is static to save bandwidth
|
|
42
|
+
- Set content type to screen, adjusting encoder configuration throughout
|
|
43
|
+
the pipeline (VP9 inter-layer prediction, simulcast layer allocation, etc.)
|
|
44
|
+
"""
|
|
45
|
+
req = proto_ffi.FfiRequest()
|
|
46
|
+
req.new_video_source.type = proto_video.VideoSourceType.VIDEO_SOURCE_NATIVE
|
|
47
|
+
req.new_video_source.resolution.width = width
|
|
48
|
+
req.new_video_source.resolution.height = height
|
|
49
|
+
req.new_video_source.is_screencast = is_screencast
|
|
50
|
+
|
|
51
|
+
resp = FfiClient.instance.request(req)
|
|
52
|
+
self._info = resp.new_video_source.source
|
|
53
|
+
self._ffi_handle = FfiHandle(self._info.handle.id)
|
|
54
|
+
|
|
55
|
+
def capture_frame(
|
|
56
|
+
self,
|
|
57
|
+
frame: VideoFrame,
|
|
58
|
+
*,
|
|
59
|
+
timestamp_us: int = 0,
|
|
60
|
+
rotation: proto_video.VideoRotation.ValueType = proto_video.VideoRotation.VIDEO_ROTATION_0,
|
|
61
|
+
metadata: proto_video.FrameMetadata | None = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Capture a frame, optionally attaching frame metadata."""
|
|
64
|
+
req = proto_ffi.FfiRequest()
|
|
65
|
+
req.capture_video_frame.source_handle = self._ffi_handle.handle
|
|
66
|
+
req.capture_video_frame.buffer.CopyFrom(frame._proto_info())
|
|
67
|
+
req.capture_video_frame.rotation = rotation
|
|
68
|
+
req.capture_video_frame.timestamp_us = timestamp_us
|
|
69
|
+
if metadata is not None:
|
|
70
|
+
req.capture_video_frame.metadata.CopyFrom(metadata)
|
|
71
|
+
FfiClient.instance.request(req)
|
|
72
|
+
|
|
73
|
+
async def aclose(self) -> None:
|
|
74
|
+
self._ffi_handle.dispose()
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any, AsyncIterator, Optional
|
|
20
|
+
|
|
21
|
+
from ._ffi_client import FfiClient, FfiHandle
|
|
22
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
23
|
+
from ._proto import video_frame_pb2 as proto_video_frame
|
|
24
|
+
from ._proto.track_pb2 import TrackSource
|
|
25
|
+
from ._utils import RingQueue, task_done_logger
|
|
26
|
+
from .participant import Participant
|
|
27
|
+
from .track import Track
|
|
28
|
+
from .video_frame import VideoFrame
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class VideoFrameEvent:
|
|
33
|
+
frame: VideoFrame
|
|
34
|
+
timestamp_us: int
|
|
35
|
+
rotation: proto_video_frame.VideoRotation
|
|
36
|
+
metadata: proto_video_frame.FrameMetadata | None = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class VideoStream:
|
|
40
|
+
"""VideoStream is a stream of video frames received from a RemoteTrack."""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
track: Track,
|
|
45
|
+
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
46
|
+
capacity: int = 0,
|
|
47
|
+
format: Optional[proto_video_frame.VideoBufferType.ValueType] = None,
|
|
48
|
+
**kwargs: Any,
|
|
49
|
+
) -> None:
|
|
50
|
+
self._loop = loop or asyncio.get_event_loop()
|
|
51
|
+
# Subscribe before creating the stream so none of its events are missed. Its
|
|
52
|
+
# handle is only known once the request returns; until then every video stream
|
|
53
|
+
# event is let through, and _run drops the ones of other streams.
|
|
54
|
+
handle_id: int | None = None
|
|
55
|
+
self._ffi_queue = FfiClient.instance.queue.subscribe(
|
|
56
|
+
self._loop,
|
|
57
|
+
filter_fn=lambda e: (
|
|
58
|
+
e.WhichOneof("message") == "video_stream_event"
|
|
59
|
+
and (handle_id is None or e.video_stream_event.stream_handle == handle_id)
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
self._queue: RingQueue[VideoFrameEvent | None] = RingQueue(capacity)
|
|
63
|
+
self._track: Track | None = track
|
|
64
|
+
self._format = format
|
|
65
|
+
self._capacity = capacity
|
|
66
|
+
self._format = format
|
|
67
|
+
stream: Any = None
|
|
68
|
+
if "participant" in kwargs:
|
|
69
|
+
stream = self._create_owned_stream_from_participant(
|
|
70
|
+
participant=kwargs["participant"], track_source=kwargs["track_source"]
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
stream = self._create_owned_stream()
|
|
74
|
+
|
|
75
|
+
self._ffi_handle = FfiHandle(stream.handle.id)
|
|
76
|
+
handle_id = stream.handle.id
|
|
77
|
+
self._info = stream.info
|
|
78
|
+
|
|
79
|
+
self._task = self._loop.create_task(self._run())
|
|
80
|
+
self._task.add_done_callback(task_done_logger)
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def from_participant(
|
|
84
|
+
cls,
|
|
85
|
+
*,
|
|
86
|
+
participant: Participant,
|
|
87
|
+
track_source: TrackSource.ValueType,
|
|
88
|
+
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
89
|
+
format: Optional[proto_video_frame.VideoBufferType.ValueType] = None,
|
|
90
|
+
capacity: int = 0,
|
|
91
|
+
) -> VideoStream:
|
|
92
|
+
return VideoStream(
|
|
93
|
+
participant=participant,
|
|
94
|
+
track_source=track_source,
|
|
95
|
+
loop=loop,
|
|
96
|
+
capacity=capacity,
|
|
97
|
+
format=format,
|
|
98
|
+
track=None, # type: ignore
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
@classmethod
|
|
102
|
+
def from_track(
|
|
103
|
+
cls,
|
|
104
|
+
*,
|
|
105
|
+
track: Track,
|
|
106
|
+
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
107
|
+
format: Optional[proto_video_frame.VideoBufferType.ValueType] = None,
|
|
108
|
+
capacity: int = 0,
|
|
109
|
+
) -> VideoStream:
|
|
110
|
+
return VideoStream(
|
|
111
|
+
track=track,
|
|
112
|
+
loop=loop,
|
|
113
|
+
capacity=capacity,
|
|
114
|
+
format=format,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def __del__(self) -> None:
|
|
118
|
+
FfiClient.instance.queue.unsubscribe(self._ffi_queue)
|
|
119
|
+
|
|
120
|
+
def _create_owned_stream(self) -> Any:
|
|
121
|
+
assert self._track is not None
|
|
122
|
+
req = proto_ffi.FfiRequest()
|
|
123
|
+
new_video_stream = req.new_video_stream
|
|
124
|
+
new_video_stream.track_handle = self._track._ffi_handle.handle
|
|
125
|
+
new_video_stream.type = proto_video_frame.VideoStreamType.VIDEO_STREAM_NATIVE
|
|
126
|
+
if self._format is not None:
|
|
127
|
+
new_video_stream.format = self._format
|
|
128
|
+
new_video_stream.normalize_stride = True
|
|
129
|
+
resp = FfiClient.instance.request(req)
|
|
130
|
+
return resp.new_video_stream.stream
|
|
131
|
+
|
|
132
|
+
def _create_owned_stream_from_participant(
|
|
133
|
+
self, participant: Participant, track_source: TrackSource.ValueType
|
|
134
|
+
) -> Any:
|
|
135
|
+
req = proto_ffi.FfiRequest()
|
|
136
|
+
video_stream_from_participant = req.video_stream_from_participant
|
|
137
|
+
video_stream_from_participant.participant_handle = participant._ffi_handle.handle
|
|
138
|
+
video_stream_from_participant.type = proto_video_frame.VideoStreamType.VIDEO_STREAM_NATIVE
|
|
139
|
+
video_stream_from_participant.track_source = track_source
|
|
140
|
+
video_stream_from_participant.normalize_stride = True
|
|
141
|
+
if self._format is not None:
|
|
142
|
+
video_stream_from_participant.format = self._format
|
|
143
|
+
resp = FfiClient.instance.request(req)
|
|
144
|
+
return resp.video_stream_from_participant.stream
|
|
145
|
+
|
|
146
|
+
async def _run(self) -> None:
|
|
147
|
+
while True:
|
|
148
|
+
event = await self._ffi_queue.wait_for(self._is_event)
|
|
149
|
+
video_event = event.video_stream_event
|
|
150
|
+
|
|
151
|
+
if video_event.HasField("frame_received"):
|
|
152
|
+
frame_received = video_event.frame_received
|
|
153
|
+
owned_buffer_info = frame_received.buffer
|
|
154
|
+
frame = VideoFrame._from_owned_info(owned_buffer_info)
|
|
155
|
+
metadata: proto_video_frame.FrameMetadata | None = None
|
|
156
|
+
if frame_received.HasField("metadata"):
|
|
157
|
+
metadata = proto_video_frame.FrameMetadata()
|
|
158
|
+
metadata.CopyFrom(frame_received.metadata)
|
|
159
|
+
|
|
160
|
+
event = VideoFrameEvent(
|
|
161
|
+
frame=frame,
|
|
162
|
+
timestamp_us=frame_received.timestamp_us,
|
|
163
|
+
rotation=frame_received.rotation,
|
|
164
|
+
metadata=metadata,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
self._queue.put(event)
|
|
168
|
+
elif video_event.HasField("eos"):
|
|
169
|
+
break
|
|
170
|
+
|
|
171
|
+
FfiClient.instance.queue.unsubscribe(self._ffi_queue)
|
|
172
|
+
|
|
173
|
+
async def aclose(self) -> None:
|
|
174
|
+
self._ffi_handle.dispose()
|
|
175
|
+
await self._task
|
|
176
|
+
|
|
177
|
+
def _is_event(self, e: proto_ffi.FfiEvent) -> bool:
|
|
178
|
+
return e.video_stream_event.stream_handle == self._ffi_handle.handle
|
|
179
|
+
|
|
180
|
+
def __aiter__(self) -> AsyncIterator[VideoFrameEvent]:
|
|
181
|
+
return self
|
|
182
|
+
|
|
183
|
+
async def __anext__(self) -> VideoFrameEvent:
|
|
184
|
+
if self._task.done():
|
|
185
|
+
raise StopAsyncIteration
|
|
186
|
+
|
|
187
|
+
item = await self._queue.get()
|
|
188
|
+
if item is None:
|
|
189
|
+
raise StopAsyncIteration
|
|
190
|
+
|
|
191
|
+
return item
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: livekit
|
|
3
|
+
Version: 1.2.1
|
|
4
|
+
Summary: Python Real-time SDK for LiveKit
|
|
5
|
+
Author-email: LiveKit <support@livekit.io>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Documentation, https://docs.livekit.io
|
|
8
|
+
Project-URL: Website, https://livekit.io/
|
|
9
|
+
Project-URL: Source, https://github.com/livekit/python-sdks/
|
|
10
|
+
Keywords: webrtc,realtime,audio,video,livekit
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
13
|
+
Classifier: Topic :: Multimedia :: Video
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Requires-Python: >=3.9.0
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
Requires-Dist: protobuf>=5
|
|
26
|
+
Requires-Dist: types-protobuf>=5
|
|
27
|
+
Requires-Dist: aiofiles>=24
|
|
28
|
+
Requires-Dist: numpy>=1.26
|
|
29
|
+
|
|
30
|
+
# LiveKit SDK for Python
|
|
31
|
+
|
|
32
|
+
Python SDK to integrate LiveKit's real-time video, audio, and data capabilities into your Python applications using WebRTC. Designed for use with [LiveKit Agents](https://github.com/livekit/agents) to build powerful voice AI apps.
|
|
33
|
+
|
|
34
|
+
See https://docs.livekit.io/ for more information.
|
|
35
|
+
|
|
36
|
+
## Publishing pre-encoded video
|
|
37
|
+
|
|
38
|
+
Use `EncodedVideoSource` when an upstream encoder already provides compressed
|
|
39
|
+
frames. The source uses the same native passthrough encoder as the Rust SDK;
|
|
40
|
+
Python does not decode or re-encode the video.
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from livekit import rtc
|
|
44
|
+
|
|
45
|
+
source = rtc.EncodedVideoSource(width=640, height=360)
|
|
46
|
+
track = rtc.LocalVideoTrack.create_video_track("camera", source)
|
|
47
|
+
await room.local_participant.publish_track(
|
|
48
|
+
track,
|
|
49
|
+
rtc.TrackPublishOptions(
|
|
50
|
+
video_codec=rtc.VideoCodec.VP9,
|
|
51
|
+
video_encoder=rtc.VideoEncoderBackend.ENCODER_BACKEND_PRE_ENCODED,
|
|
52
|
+
simulcast=False,
|
|
53
|
+
),
|
|
54
|
+
)
|
|
55
|
+
# Supply each complete encoded frame at its playback time.
|
|
56
|
+
source.capture_frame(rtc.EncodedVideoFrame(
|
|
57
|
+
data=encoded_vp9_frame,
|
|
58
|
+
width=640,
|
|
59
|
+
height=360,
|
|
60
|
+
codec=rtc.VideoCodec.VP9,
|
|
61
|
+
frame_type=rtc.EncodedFrameType.ENCODED_FRAME_KEY,
|
|
62
|
+
timestamp_us=capture_timestamp_us,
|
|
63
|
+
))
|
|
64
|
+
feedback = source.take_feedback()
|
|
65
|
+
# Forward feedback.keyframe_requested and feedback.rate_control to your encoder.
|
|
66
|
+
# At shutdown, unpublish the track and release the source:
|
|
67
|
+
await room.local_participant.unpublish_track(track.sid)
|
|
68
|
+
await source.aclose()
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Supported codecs are H.264, H.265, VP8, VP9 and AV1, subject to receiver support.
|
|
72
|
+
Submit complete access units (Annex B for H.264/H.265), not WebM/MP4 container
|
|
73
|
+
bytes or RTP packets. A container must first be demuxed; demuxing extracts
|
|
74
|
+
compressed frames and does not reconstruct pixels. The codec must match the
|
|
75
|
+
publication and stay fixed, and simulcast must be disabled.
|
|
76
|
+
|
|
77
|
+
The caller controls pacing and handles keyframe and bitrate feedback. In
|
|
78
|
+
particular, a prerecorded file cannot generate a new keyframe on demand for a
|
|
79
|
+
late subscriber or after packet loss. WebM's auxiliary alpha data is not part of
|
|
80
|
+
the VP9 color access unit and is not transported by this API.
|