livekit 1.2.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit/rtc/__init__.py +264 -0
- livekit/rtc/_ffi_client.py +321 -0
- livekit/rtc/_proto/__init__.py +0 -0
- livekit/rtc/_proto/audio_frame_pb2.py +159 -0
- livekit/rtc/_proto/audio_frame_pb2.pyi +1560 -0
- livekit/rtc/_proto/capture_pb2.py +70 -0
- livekit/rtc/_proto/capture_pb2.pyi +559 -0
- livekit/rtc/_proto/data_stream_pb2.py +171 -0
- livekit/rtc/_proto/data_stream_pb2.pyi +1498 -0
- livekit/rtc/_proto/data_track_pb2.py +120 -0
- livekit/rtc/_proto/data_track_pb2.pyi +1073 -0
- livekit/rtc/_proto/e2ee_pb2.py +81 -0
- livekit/rtc/_proto/e2ee_pb2.pyi +591 -0
- livekit/rtc/_proto/ffi_pb2.py +55 -0
- livekit/rtc/_proto/ffi_pb2.pyi +1246 -0
- livekit/rtc/_proto/handle_pb2.py +27 -0
- livekit/rtc/_proto/handle_pb2.pyi +51 -0
- livekit/rtc/_proto/participant_pb2.py +45 -0
- livekit/rtc/_proto/participant_pb2.pyi +313 -0
- livekit/rtc/_proto/room_pb2.py +311 -0
- livekit/rtc/_proto/room_pb2.pyi +3136 -0
- livekit/rtc/_proto/rpc_pb2.py +47 -0
- livekit/rtc/_proto/rpc_pb2.pyi +261 -0
- livekit/rtc/_proto/stats_pb2.py +123 -0
- livekit/rtc/_proto/stats_pb2.pyi +1514 -0
- livekit/rtc/_proto/track_pb2.py +76 -0
- livekit/rtc/_proto/track_pb2.pyi +544 -0
- livekit/rtc/_proto/track_publication_pb2.py +39 -0
- livekit/rtc/_proto/track_publication_pb2.pyi +145 -0
- livekit/rtc/_proto/video_frame_pb2.py +97 -0
- livekit/rtc/_proto/video_frame_pb2.pyi +838 -0
- livekit/rtc/_utils.py +187 -0
- livekit/rtc/apm.py +119 -0
- livekit/rtc/audio_filter.py +23 -0
- livekit/rtc/audio_frame.py +242 -0
- livekit/rtc/audio_mixer.py +247 -0
- livekit/rtc/audio_resampler.py +174 -0
- livekit/rtc/audio_source.py +191 -0
- livekit/rtc/audio_stream.py +354 -0
- livekit/rtc/data_stream.py +640 -0
- livekit/rtc/data_track.py +296 -0
- livekit/rtc/e2ee.py +320 -0
- livekit/rtc/encoded_video.py +127 -0
- livekit/rtc/event_emitter.py +215 -0
- livekit/rtc/frame_processor.py +37 -0
- livekit/rtc/jupyter.py +68 -0
- livekit/rtc/log.py +3 -0
- livekit/rtc/media_devices.py +596 -0
- livekit/rtc/participant.py +1171 -0
- livekit/rtc/platform_audio.py +426 -0
- livekit/rtc/py.typed +0 -0
- livekit/rtc/resources/LICENSE.md +4055 -0
- livekit/rtc/resources/__init__.py +1 -0
- livekit/rtc/resources/jupyter-html/index.html +101 -0
- livekit/rtc/resources/livekit_ffi.dll +0 -0
- livekit/rtc/resources/livekit_ffi.h +42 -0
- livekit/rtc/room.py +1304 -0
- livekit/rtc/rpc.py +244 -0
- livekit/rtc/synchronizer.py +202 -0
- livekit/rtc/track.py +264 -0
- livekit/rtc/track_publication.py +145 -0
- livekit/rtc/transcription.py +19 -0
- livekit/rtc/utils.py +138 -0
- livekit/rtc/version.py +1 -0
- livekit/rtc/video_frame.py +347 -0
- livekit/rtc/video_source.py +74 -0
- livekit/rtc/video_stream.py +191 -0
- livekit-1.2.1.dist-info/METADATA +80 -0
- livekit-1.2.1.dist-info/RECORD +71 -0
- livekit-1.2.1.dist-info/WHEEL +5 -0
- livekit-1.2.1.dist-info/top_level.txt +1 -0
livekit/rtc/_utils.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import logging
|
|
17
|
+
from collections import deque
|
|
18
|
+
import ctypes
|
|
19
|
+
import random
|
|
20
|
+
from typing import Any, Callable, Generator, Generic, List, TypeVar, Union
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger("livekit")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class classproperty(object):
|
|
26
|
+
def __init__(self, f: Callable[..., Any]) -> None:
|
|
27
|
+
self.f: Any = classmethod(f)
|
|
28
|
+
|
|
29
|
+
def __get__(self, *a: Any) -> Any:
|
|
30
|
+
return self.f.__get__(*a)()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def task_done_logger(task: asyncio.Task) -> None:
|
|
34
|
+
if task.cancelled():
|
|
35
|
+
logger.info("task cancelled: %s", task)
|
|
36
|
+
return
|
|
37
|
+
|
|
38
|
+
if task.exception():
|
|
39
|
+
logger.error("task exception: %s", task, exc_info=task.exception())
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _ensure_compatible_buffer(
|
|
44
|
+
data: Union[bytes, bytearray, memoryview],
|
|
45
|
+
) -> Union[bytes, bytearray, memoryview]:
|
|
46
|
+
"""Validate and normalize a buffer for FFI use.
|
|
47
|
+
|
|
48
|
+
Sliced memoryviews are materialized because get_address cannot
|
|
49
|
+
reliably resolve their offset for all buffer types.
|
|
50
|
+
"""
|
|
51
|
+
if isinstance(data, memoryview):
|
|
52
|
+
if not data.contiguous:
|
|
53
|
+
raise ValueError("memoryview must be contiguous")
|
|
54
|
+
if data.nbytes != len(data.obj): # type: ignore[arg-type]
|
|
55
|
+
return bytearray(data) if not data.readonly else bytes(data)
|
|
56
|
+
elif not isinstance(data, (bytes, bytearray)):
|
|
57
|
+
raise TypeError(f"expected bytes, bytearray, or memoryview, got {type(data)}")
|
|
58
|
+
return data
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_address(data: Union[bytes, bytearray, memoryview]) -> int:
|
|
62
|
+
if isinstance(data, memoryview):
|
|
63
|
+
if not data.readonly:
|
|
64
|
+
return ctypes.addressof(ctypes.c_char.from_buffer(data))
|
|
65
|
+
data = data.obj # type: ignore[assignment]
|
|
66
|
+
if isinstance(data, bytearray):
|
|
67
|
+
return ctypes.addressof(ctypes.c_char.from_buffer(data))
|
|
68
|
+
if isinstance(data, bytes):
|
|
69
|
+
addr = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value
|
|
70
|
+
assert addr is not None
|
|
71
|
+
return addr
|
|
72
|
+
raise TypeError(f"expected bytes, bytearray, or memoryview, got {type(data)}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
T = TypeVar("T")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class RingQueue(Generic[T]):
|
|
79
|
+
def __init__(self, capacity: int = 0) -> None:
|
|
80
|
+
self._capacity = capacity
|
|
81
|
+
self._queue: deque[T] = deque()
|
|
82
|
+
self._event = asyncio.Event()
|
|
83
|
+
|
|
84
|
+
def put(self, item: T) -> None:
|
|
85
|
+
if self._capacity > 0 and len(self._queue) == self._capacity:
|
|
86
|
+
self._queue.popleft()
|
|
87
|
+
self._queue.append(item)
|
|
88
|
+
self._event.set()
|
|
89
|
+
|
|
90
|
+
async def get(self) -> T:
|
|
91
|
+
while len(self._queue) == 0:
|
|
92
|
+
await self._event.wait()
|
|
93
|
+
self._event.clear()
|
|
94
|
+
return self._queue.popleft()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class Queue(asyncio.Queue[T]):
|
|
98
|
+
"""asyncio.Queue with utility functions."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, maxsize: int = 0) -> None:
|
|
101
|
+
super().__init__(maxsize)
|
|
102
|
+
|
|
103
|
+
async def wait_for(self, fnc: Callable[[T], bool]) -> T:
|
|
104
|
+
"""Wait for an event that matches the given function.
|
|
105
|
+
The previous events are discarded.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
while True:
|
|
109
|
+
event = await self.get()
|
|
110
|
+
if fnc(event):
|
|
111
|
+
# task_done must be manually called for the returned item
|
|
112
|
+
return event
|
|
113
|
+
|
|
114
|
+
self.task_done()
|
|
115
|
+
|
|
116
|
+
def release(self) -> None:
|
|
117
|
+
"""Mark every unfinished item as done.
|
|
118
|
+
|
|
119
|
+
Lets a ``join()`` that is already waiting on this queue return once there
|
|
120
|
+
is no consumer left for it.
|
|
121
|
+
"""
|
|
122
|
+
while True:
|
|
123
|
+
try:
|
|
124
|
+
self.task_done()
|
|
125
|
+
except ValueError:
|
|
126
|
+
# task_done() raises once every item is accounted for
|
|
127
|
+
return
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class BroadcastQueue(Generic[T]):
|
|
131
|
+
"""Queue with multiple subscribers."""
|
|
132
|
+
|
|
133
|
+
def __init__(self) -> None:
|
|
134
|
+
self._lock = asyncio.Lock()
|
|
135
|
+
self._subscribers: List[Queue[T]] = []
|
|
136
|
+
|
|
137
|
+
def len_subscribers(self) -> int:
|
|
138
|
+
return len(self._subscribers)
|
|
139
|
+
|
|
140
|
+
def put_nowait(self, item: T) -> None:
|
|
141
|
+
for queue in self._subscribers:
|
|
142
|
+
queue.put_nowait(item)
|
|
143
|
+
|
|
144
|
+
def subscribe(self) -> Queue[T]:
|
|
145
|
+
queue = Queue[T]()
|
|
146
|
+
self._subscribers.append(queue)
|
|
147
|
+
return queue
|
|
148
|
+
|
|
149
|
+
def unsubscribe(self, queue: Queue[T]) -> None:
|
|
150
|
+
self._subscribers.remove(queue)
|
|
151
|
+
# A join() may already hold this queue. Events that are still queued, or that
|
|
152
|
+
# the subscriber took but never finished, have no consumer anymore, so
|
|
153
|
+
# release them instead of leaving the joiner waiting forever.
|
|
154
|
+
queue.release()
|
|
155
|
+
|
|
156
|
+
async def join(self) -> None:
|
|
157
|
+
async with self._lock:
|
|
158
|
+
subs = self._subscribers.copy()
|
|
159
|
+
for queue in subs:
|
|
160
|
+
await queue.join()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
_base62_characters = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def generate_random_base62(length: int = 12) -> str:
|
|
167
|
+
"""
|
|
168
|
+
Generate a random base62 encoded string of a specified length.
|
|
169
|
+
|
|
170
|
+
:param length: The desired length of the base62 encoded string.
|
|
171
|
+
:return: A base62 encoded string.
|
|
172
|
+
"""
|
|
173
|
+
global _base62_characters
|
|
174
|
+
return "".join(random.choice(_base62_characters) for _ in range(length))
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# adapted from https://stackoverflow.com/a/6043797
|
|
178
|
+
def split_utf8(s: str, n: int) -> Generator[bytes, None, None]:
|
|
179
|
+
"""Split UTF-8 s into chunks of maximum length n."""
|
|
180
|
+
encoded = s.encode()
|
|
181
|
+
while len(encoded) > n:
|
|
182
|
+
k = n
|
|
183
|
+
while (encoded[k] & 0xC0) == 0x80:
|
|
184
|
+
k -= 1
|
|
185
|
+
yield encoded[:k]
|
|
186
|
+
encoded = encoded[k:]
|
|
187
|
+
yield encoded
|
livekit/rtc/apm.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from ._ffi_client import FfiClient, FfiHandle
|
|
4
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
5
|
+
from ._utils import get_address
|
|
6
|
+
from .audio_frame import AudioFrame
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AudioProcessingModule:
|
|
10
|
+
"""
|
|
11
|
+
Provides WebRTC audio processing capabilities including echo cancellation, noise suppression,
|
|
12
|
+
high-pass filtering, and gain control.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
*,
|
|
18
|
+
echo_cancellation: bool = False,
|
|
19
|
+
noise_suppression: bool = False,
|
|
20
|
+
high_pass_filter: bool = False,
|
|
21
|
+
auto_gain_control: bool = False,
|
|
22
|
+
) -> None:
|
|
23
|
+
"""
|
|
24
|
+
Initialize an AudioProcessingModule instance with the specified audio processing features.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
echo_cancellation (bool, optional): Whether to enable echo cancellation.
|
|
28
|
+
noise_suppression (bool, optional): Whether to enable noise suppression.
|
|
29
|
+
high_pass_filter (bool, optional): Whether to enable a high-pass filter.
|
|
30
|
+
auto_gain_control (bool, optional): Whether to enable auto gain control.
|
|
31
|
+
"""
|
|
32
|
+
req = proto_ffi.FfiRequest()
|
|
33
|
+
req.new_apm.echo_canceller_enabled = echo_cancellation
|
|
34
|
+
req.new_apm.noise_suppression_enabled = noise_suppression
|
|
35
|
+
req.new_apm.high_pass_filter_enabled = high_pass_filter
|
|
36
|
+
req.new_apm.gain_controller_enabled = auto_gain_control
|
|
37
|
+
|
|
38
|
+
resp = FfiClient.instance.request(req)
|
|
39
|
+
self._ffi_handle = FfiHandle(resp.new_apm.apm.handle.id)
|
|
40
|
+
|
|
41
|
+
def process_stream(self, data: AudioFrame) -> None:
|
|
42
|
+
"""
|
|
43
|
+
Process the provided audio frame using the configured audio processing features.
|
|
44
|
+
|
|
45
|
+
The input audio frame is modified in-place (if applicable) by the underlying audio
|
|
46
|
+
processing module (e.g., echo cancellation, noise suppression, etc.).
|
|
47
|
+
|
|
48
|
+
Important:
|
|
49
|
+
Audio frames must be exactly 10 ms in duration.
|
|
50
|
+
"""
|
|
51
|
+
if isinstance(data._data, bytes) or (
|
|
52
|
+
isinstance(data._data, memoryview) and data._data.readonly
|
|
53
|
+
):
|
|
54
|
+
data._data = bytearray(data._data)
|
|
55
|
+
|
|
56
|
+
req = proto_ffi.FfiRequest()
|
|
57
|
+
req.apm_process_stream.apm_handle = self._ffi_handle.handle
|
|
58
|
+
req.apm_process_stream.data_ptr = get_address(data._data)
|
|
59
|
+
req.apm_process_stream.size = len(data._data)
|
|
60
|
+
req.apm_process_stream.sample_rate = data.sample_rate
|
|
61
|
+
req.apm_process_stream.num_channels = data.num_channels
|
|
62
|
+
|
|
63
|
+
resp = FfiClient.instance.request(req)
|
|
64
|
+
|
|
65
|
+
if resp.apm_process_stream.error:
|
|
66
|
+
raise RuntimeError(resp.apm_process_stream.error)
|
|
67
|
+
|
|
68
|
+
def process_reverse_stream(self, data: AudioFrame) -> None:
|
|
69
|
+
"""
|
|
70
|
+
Process the reverse audio frame (typically used for echo cancellation in a full-duplex setup).
|
|
71
|
+
|
|
72
|
+
In an echo cancellation scenario, this method is used to process the "far-end" audio
|
|
73
|
+
prior to mixing or feeding it into the echo canceller. Like `process_stream`, the
|
|
74
|
+
input audio frame is modified in-place by the underlying processing module.
|
|
75
|
+
|
|
76
|
+
Important:
|
|
77
|
+
Audio frames must be exactly 10 ms in duration.
|
|
78
|
+
"""
|
|
79
|
+
if isinstance(data._data, bytes) or (
|
|
80
|
+
isinstance(data._data, memoryview) and data._data.readonly
|
|
81
|
+
):
|
|
82
|
+
data._data = bytearray(data._data)
|
|
83
|
+
|
|
84
|
+
req = proto_ffi.FfiRequest()
|
|
85
|
+
req.apm_process_reverse_stream.apm_handle = self._ffi_handle.handle
|
|
86
|
+
req.apm_process_reverse_stream.data_ptr = get_address(data._data)
|
|
87
|
+
req.apm_process_reverse_stream.size = len(data._data)
|
|
88
|
+
req.apm_process_reverse_stream.sample_rate = data.sample_rate
|
|
89
|
+
req.apm_process_reverse_stream.num_channels = data.num_channels
|
|
90
|
+
|
|
91
|
+
resp = FfiClient.instance.request(req)
|
|
92
|
+
|
|
93
|
+
if resp.apm_process_reverse_stream.error:
|
|
94
|
+
raise RuntimeError(resp.apm_process_reverse_stream.error)
|
|
95
|
+
|
|
96
|
+
def set_stream_delay_ms(self, delay_ms: int) -> None:
|
|
97
|
+
"""
|
|
98
|
+
This must be called if and only if echo processing is enabled.
|
|
99
|
+
|
|
100
|
+
Sets the `delay` in ms between `process_reverse_stream()` receiving a far-end
|
|
101
|
+
frame and `process_stream()` receiving a near-end frame containing the
|
|
102
|
+
corresponding echo. On the client-side this can be expressed as
|
|
103
|
+
delay = (t_render - t_analyze) + (t_process - t_capture)
|
|
104
|
+
where,
|
|
105
|
+
- t_analyze is the time a frame is passed to `process_reverse_stream()` and
|
|
106
|
+
t_render is the time the first sample of the same frame is rendered by
|
|
107
|
+
the audio hardware.
|
|
108
|
+
- t_capture is the time the first sample of a frame is captured by the
|
|
109
|
+
audio hardware and t_process is the time the same frame is passed to
|
|
110
|
+
`process_stream()`.
|
|
111
|
+
"""
|
|
112
|
+
req = proto_ffi.FfiRequest()
|
|
113
|
+
req.apm_set_stream_delay.apm_handle = self._ffi_handle.handle
|
|
114
|
+
req.apm_set_stream_delay.delay_ms = delay_ms
|
|
115
|
+
|
|
116
|
+
resp = FfiClient.instance.request(req)
|
|
117
|
+
|
|
118
|
+
if resp.apm_set_stream_delay.error:
|
|
119
|
+
raise RuntimeError(resp.apm_set_stream_delay.error)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from typing import Optional, List
|
|
2
|
+
|
|
3
|
+
from ._ffi_client import FfiClient
|
|
4
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class AudioFilter:
|
|
8
|
+
def __init__(self, module_id: str, path: str, dependencies: Optional[List[str]] = None) -> None:
|
|
9
|
+
self._path = path
|
|
10
|
+
|
|
11
|
+
req = proto_ffi.FfiRequest()
|
|
12
|
+
req.load_audio_filter_plugin.module_id = module_id
|
|
13
|
+
req.load_audio_filter_plugin.plugin_path = path
|
|
14
|
+
req.load_audio_filter_plugin.dependencies[:] = (
|
|
15
|
+
dependencies if dependencies is not None else []
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
resp = FfiClient.instance.request(req)
|
|
19
|
+
|
|
20
|
+
if resp.load_audio_filter_plugin.error:
|
|
21
|
+
raise Exception(
|
|
22
|
+
f"failed to initialize audio filter #{resp.load_audio_filter_plugin.error}"
|
|
23
|
+
)
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import ctypes
|
|
16
|
+
from ._ffi_client import FfiHandle
|
|
17
|
+
from ._proto import audio_frame_pb2 as proto_audio
|
|
18
|
+
from ._utils import _ensure_compatible_buffer, get_address
|
|
19
|
+
from typing import Any, Union
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AudioFrame:
|
|
23
|
+
"""
|
|
24
|
+
A class that represents a frame of audio data with specific properties such as sample rate,
|
|
25
|
+
number of channels, and samples per channel.
|
|
26
|
+
|
|
27
|
+
The format of the audio data is 16-bit signed integers (int16) interleaved by channel.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
data: Union[bytes, bytearray, memoryview],
|
|
33
|
+
sample_rate: int,
|
|
34
|
+
num_channels: int,
|
|
35
|
+
samples_per_channel: int,
|
|
36
|
+
*,
|
|
37
|
+
userdata: Union[dict[str, Any], None] = None,
|
|
38
|
+
) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Initialize an AudioFrame instance.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
data (Union[bytes, bytearray, memoryview]): The raw audio data, which must be at least
|
|
44
|
+
`num_channels * samples_per_channel * sizeof(int16)` bytes long.
|
|
45
|
+
sample_rate (int): The sample rate of the audio in Hz.
|
|
46
|
+
num_channels (int): The number of audio channels (e.g., 1 for mono, 2 for stereo).
|
|
47
|
+
samples_per_channel (int): The number of samples per channel.
|
|
48
|
+
|
|
49
|
+
Raises:
|
|
50
|
+
ValueError: If the length of `data` is smaller than the required size.
|
|
51
|
+
"""
|
|
52
|
+
if isinstance(data, memoryview):
|
|
53
|
+
data = data.cast("B")
|
|
54
|
+
|
|
55
|
+
data = _ensure_compatible_buffer(data)
|
|
56
|
+
|
|
57
|
+
min_size = num_channels * samples_per_channel * ctypes.sizeof(ctypes.c_int16)
|
|
58
|
+
data_len = len(data)
|
|
59
|
+
|
|
60
|
+
if data_len < min_size:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
"data length must be >= num_channels * samples_per_channel * sizeof(int16)"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
if data_len % ctypes.sizeof(ctypes.c_int16) != 0:
|
|
66
|
+
# can happen if data is bigger than needed
|
|
67
|
+
raise ValueError("data length must be a multiple of sizeof(int16)")
|
|
68
|
+
|
|
69
|
+
self._data = data
|
|
70
|
+
|
|
71
|
+
self._sample_rate = sample_rate
|
|
72
|
+
self._num_channels = num_channels
|
|
73
|
+
self._samples_per_channel = samples_per_channel
|
|
74
|
+
self._userdata = {} if userdata is None else userdata
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def create(sample_rate: int, num_channels: int, samples_per_channel: int) -> "AudioFrame":
|
|
78
|
+
"""
|
|
79
|
+
Create a new empty AudioFrame instance with specified sample rate, number of channels,
|
|
80
|
+
and samples per channel.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
sample_rate (int): The sample rate of the audio in Hz.
|
|
84
|
+
num_channels (int): The number of audio channels (e.g., 1 for mono, 2 for stereo).
|
|
85
|
+
samples_per_channel (int): The number of samples per channel.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
AudioFrame: A new AudioFrame instance with uninitialized (zeroed) data.
|
|
89
|
+
"""
|
|
90
|
+
size = num_channels * samples_per_channel * ctypes.sizeof(ctypes.c_int16)
|
|
91
|
+
data = bytearray(size)
|
|
92
|
+
return AudioFrame(data, sample_rate, num_channels, samples_per_channel)
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def _from_owned_info(owned_info: proto_audio.OwnedAudioFrameBuffer) -> "AudioFrame":
|
|
96
|
+
info = owned_info.info
|
|
97
|
+
size = info.num_channels * info.samples_per_channel
|
|
98
|
+
cdata = (ctypes.c_int16 * size).from_address(info.data_ptr)
|
|
99
|
+
data = bytearray(cdata)
|
|
100
|
+
FfiHandle(owned_info.handle.id).dispose()
|
|
101
|
+
return AudioFrame(data, info.sample_rate, info.num_channels, info.samples_per_channel)
|
|
102
|
+
|
|
103
|
+
def _proto_info(self) -> proto_audio.AudioFrameBufferInfo:
|
|
104
|
+
audio_info = proto_audio.AudioFrameBufferInfo()
|
|
105
|
+
audio_info.data_ptr = get_address(self._data)
|
|
106
|
+
audio_info.sample_rate = self.sample_rate
|
|
107
|
+
audio_info.num_channels = self.num_channels
|
|
108
|
+
audio_info.samples_per_channel = self.samples_per_channel
|
|
109
|
+
return audio_info
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def userdata(self) -> dict[str, Any]:
|
|
113
|
+
"""
|
|
114
|
+
Returns the user data associated with the audio frame.
|
|
115
|
+
"""
|
|
116
|
+
return self._userdata
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def data(self) -> memoryview:
|
|
120
|
+
"""
|
|
121
|
+
Returns a memory view of the audio data as 16-bit signed integers.
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
memoryview: A memory view of the audio data.
|
|
125
|
+
"""
|
|
126
|
+
return memoryview(self._data).cast("B").cast("h")
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def sample_rate(self) -> int:
|
|
130
|
+
"""
|
|
131
|
+
Returns the sample rate of the audio frame.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
int: The sample rate in Hz.
|
|
135
|
+
"""
|
|
136
|
+
return self._sample_rate
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def num_channels(self) -> int:
|
|
140
|
+
"""
|
|
141
|
+
Returns the number of channels in the audio frame.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
int: The number of audio channels (e.g., 1 for mono, 2 for stereo).
|
|
145
|
+
"""
|
|
146
|
+
return self._num_channels
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def samples_per_channel(self) -> int:
|
|
150
|
+
"""
|
|
151
|
+
Returns the number of samples per channel.
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
int: The number of samples per channel.
|
|
155
|
+
"""
|
|
156
|
+
return self._samples_per_channel
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def duration(self) -> float:
|
|
160
|
+
"""
|
|
161
|
+
Returns the duration of the audio frame in seconds.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
float: The duration in seconds.
|
|
165
|
+
"""
|
|
166
|
+
return self.samples_per_channel / self.sample_rate
|
|
167
|
+
|
|
168
|
+
def to_wav_bytes(self) -> bytes:
|
|
169
|
+
"""
|
|
170
|
+
Convert the audio frame data to a WAV-formatted byte stream.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
bytes: The audio data encoded in WAV format.
|
|
174
|
+
"""
|
|
175
|
+
import wave
|
|
176
|
+
import io
|
|
177
|
+
|
|
178
|
+
with io.BytesIO() as wav_file:
|
|
179
|
+
with wave.open(wav_file, "wb") as wav:
|
|
180
|
+
wav.setnchannels(self.num_channels)
|
|
181
|
+
wav.setsampwidth(2)
|
|
182
|
+
wav.setframerate(self.sample_rate)
|
|
183
|
+
wav.writeframes(self._data)
|
|
184
|
+
|
|
185
|
+
return wav_file.getvalue()
|
|
186
|
+
|
|
187
|
+
def __repr__(self) -> str:
|
|
188
|
+
return (
|
|
189
|
+
f"rtc.AudioFrame(sample_rate={self.sample_rate}, "
|
|
190
|
+
f"num_channels={self.num_channels}, "
|
|
191
|
+
f"samples_per_channel={self.samples_per_channel}, "
|
|
192
|
+
f"duration={self.duration:.3f})"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
@classmethod
|
|
196
|
+
def __get_pydantic_core_schema__(cls, *_: Any) -> Any:
|
|
197
|
+
from pydantic_core import core_schema
|
|
198
|
+
import base64
|
|
199
|
+
|
|
200
|
+
def validate_audio_frame(value: Any) -> "AudioFrame":
|
|
201
|
+
if isinstance(value, AudioFrame):
|
|
202
|
+
return value
|
|
203
|
+
|
|
204
|
+
if isinstance(value, tuple):
|
|
205
|
+
value = value[0]
|
|
206
|
+
|
|
207
|
+
if isinstance(value, dict):
|
|
208
|
+
return AudioFrame(
|
|
209
|
+
data=base64.b64decode(value["data"]),
|
|
210
|
+
sample_rate=value["sample_rate"],
|
|
211
|
+
num_channels=value["num_channels"],
|
|
212
|
+
samples_per_channel=value["samples_per_channel"],
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
raise TypeError("Invalid type for AudioFrame")
|
|
216
|
+
|
|
217
|
+
return core_schema.json_or_python_schema(
|
|
218
|
+
json_schema=core_schema.chain_schema(
|
|
219
|
+
[
|
|
220
|
+
core_schema.model_fields_schema(
|
|
221
|
+
{
|
|
222
|
+
"data": core_schema.model_field(core_schema.str_schema()),
|
|
223
|
+
"sample_rate": core_schema.model_field(core_schema.int_schema()),
|
|
224
|
+
"num_channels": core_schema.model_field(core_schema.int_schema()),
|
|
225
|
+
"samples_per_channel": core_schema.model_field(
|
|
226
|
+
core_schema.int_schema()
|
|
227
|
+
),
|
|
228
|
+
},
|
|
229
|
+
),
|
|
230
|
+
core_schema.no_info_plain_validator_function(validate_audio_frame),
|
|
231
|
+
]
|
|
232
|
+
),
|
|
233
|
+
python_schema=core_schema.no_info_plain_validator_function(validate_audio_frame),
|
|
234
|
+
serialization=core_schema.plain_serializer_function_ser_schema(
|
|
235
|
+
lambda instance: {
|
|
236
|
+
"data": base64.b64encode(instance.data).decode("utf-8"),
|
|
237
|
+
"sample_rate": instance.sample_rate,
|
|
238
|
+
"num_channels": instance.num_channels,
|
|
239
|
+
"samples_per_channel": instance.samples_per_channel,
|
|
240
|
+
}
|
|
241
|
+
),
|
|
242
|
+
)
|