livekit 1.2.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit/rtc/__init__.py +264 -0
- livekit/rtc/_ffi_client.py +321 -0
- livekit/rtc/_proto/__init__.py +0 -0
- livekit/rtc/_proto/audio_frame_pb2.py +159 -0
- livekit/rtc/_proto/audio_frame_pb2.pyi +1560 -0
- livekit/rtc/_proto/capture_pb2.py +70 -0
- livekit/rtc/_proto/capture_pb2.pyi +559 -0
- livekit/rtc/_proto/data_stream_pb2.py +171 -0
- livekit/rtc/_proto/data_stream_pb2.pyi +1498 -0
- livekit/rtc/_proto/data_track_pb2.py +120 -0
- livekit/rtc/_proto/data_track_pb2.pyi +1073 -0
- livekit/rtc/_proto/e2ee_pb2.py +81 -0
- livekit/rtc/_proto/e2ee_pb2.pyi +591 -0
- livekit/rtc/_proto/ffi_pb2.py +55 -0
- livekit/rtc/_proto/ffi_pb2.pyi +1246 -0
- livekit/rtc/_proto/handle_pb2.py +27 -0
- livekit/rtc/_proto/handle_pb2.pyi +51 -0
- livekit/rtc/_proto/participant_pb2.py +45 -0
- livekit/rtc/_proto/participant_pb2.pyi +313 -0
- livekit/rtc/_proto/room_pb2.py +311 -0
- livekit/rtc/_proto/room_pb2.pyi +3136 -0
- livekit/rtc/_proto/rpc_pb2.py +47 -0
- livekit/rtc/_proto/rpc_pb2.pyi +261 -0
- livekit/rtc/_proto/stats_pb2.py +123 -0
- livekit/rtc/_proto/stats_pb2.pyi +1514 -0
- livekit/rtc/_proto/track_pb2.py +76 -0
- livekit/rtc/_proto/track_pb2.pyi +544 -0
- livekit/rtc/_proto/track_publication_pb2.py +39 -0
- livekit/rtc/_proto/track_publication_pb2.pyi +145 -0
- livekit/rtc/_proto/video_frame_pb2.py +97 -0
- livekit/rtc/_proto/video_frame_pb2.pyi +838 -0
- livekit/rtc/_utils.py +187 -0
- livekit/rtc/apm.py +119 -0
- livekit/rtc/audio_filter.py +23 -0
- livekit/rtc/audio_frame.py +242 -0
- livekit/rtc/audio_mixer.py +247 -0
- livekit/rtc/audio_resampler.py +174 -0
- livekit/rtc/audio_source.py +191 -0
- livekit/rtc/audio_stream.py +354 -0
- livekit/rtc/data_stream.py +640 -0
- livekit/rtc/data_track.py +296 -0
- livekit/rtc/e2ee.py +320 -0
- livekit/rtc/encoded_video.py +127 -0
- livekit/rtc/event_emitter.py +215 -0
- livekit/rtc/frame_processor.py +37 -0
- livekit/rtc/jupyter.py +68 -0
- livekit/rtc/log.py +3 -0
- livekit/rtc/media_devices.py +596 -0
- livekit/rtc/participant.py +1171 -0
- livekit/rtc/platform_audio.py +426 -0
- livekit/rtc/py.typed +0 -0
- livekit/rtc/resources/LICENSE.md +4055 -0
- livekit/rtc/resources/__init__.py +1 -0
- livekit/rtc/resources/jupyter-html/index.html +101 -0
- livekit/rtc/resources/livekit_ffi.dll +0 -0
- livekit/rtc/resources/livekit_ffi.h +42 -0
- livekit/rtc/room.py +1304 -0
- livekit/rtc/rpc.py +244 -0
- livekit/rtc/synchronizer.py +202 -0
- livekit/rtc/track.py +264 -0
- livekit/rtc/track_publication.py +145 -0
- livekit/rtc/transcription.py +19 -0
- livekit/rtc/utils.py +138 -0
- livekit/rtc/version.py +1 -0
- livekit/rtc/video_frame.py +347 -0
- livekit/rtc/video_source.py +74 -0
- livekit/rtc/video_stream.py +191 -0
- livekit-1.2.1.dist-info/METADATA +80 -0
- livekit-1.2.1.dist-info/RECORD +71 -0
- livekit-1.2.1.dist-info/WHEEL +5 -0
- livekit-1.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import numpy as np
|
|
3
|
+
import contextlib
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import AsyncIterator, Optional
|
|
6
|
+
from .audio_frame import AudioFrame
|
|
7
|
+
from .log import logger
|
|
8
|
+
|
|
9
|
+
_Stream = AsyncIterator[AudioFrame]
|
|
10
|
+
|
|
11
|
+
# consecutive timeouts after which a stream is dropped once end_input() was called
|
|
12
|
+
_MAX_STALLED_ROUNDS = 10
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class _Contribution:
|
|
17
|
+
stream: _Stream
|
|
18
|
+
data: np.ndarray
|
|
19
|
+
buffer: np.ndarray
|
|
20
|
+
had_data: bool
|
|
21
|
+
exhausted: bool
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AudioMixer:
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
sample_rate: int,
|
|
28
|
+
num_channels: int,
|
|
29
|
+
*,
|
|
30
|
+
blocksize: int = 0,
|
|
31
|
+
stream_timeout_ms: int = 100,
|
|
32
|
+
capacity: int = 100,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""
|
|
35
|
+
Initialize the AudioMixer.
|
|
36
|
+
|
|
37
|
+
The mixer accepts multiple async audio streams and mixes them into a single output stream.
|
|
38
|
+
Each output frame is generated with a fixed chunk size determined by the blocksize (in samples).
|
|
39
|
+
If blocksize is not provided (or 0), it defaults to 100ms.
|
|
40
|
+
|
|
41
|
+
Each input stream is processed in parallel, accumulating audio data until at least one chunk
|
|
42
|
+
of samples is available. If an input stream does not provide data within the specified timeout,
|
|
43
|
+
a warning is logged. The mixer can be closed immediately
|
|
44
|
+
(dropping unconsumed frames) or allowed to flush remaining data using end_input().
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
sample_rate (int): The audio sample rate in Hz.
|
|
48
|
+
num_channels (int): The number of audio channels.
|
|
49
|
+
blocksize (int, optional): The size of the audio block (in samples) for mixing. If not provided,
|
|
50
|
+
defaults to sample_rate // 10.
|
|
51
|
+
stream_timeout_ms (int, optional): The maximum wait time in milliseconds for each stream to provide
|
|
52
|
+
audio data before timing out. Defaults to 100 ms.
|
|
53
|
+
capacity (int, optional): The maximum number of mixed frames to store in the output queue.
|
|
54
|
+
Defaults to 100.
|
|
55
|
+
"""
|
|
56
|
+
self._streams: set[_Stream] = set()
|
|
57
|
+
self._buffers: dict[_Stream, np.ndarray] = {}
|
|
58
|
+
# A read that timed out is kept for the next round: cancelling it would close an
|
|
59
|
+
# async generator stream for good.
|
|
60
|
+
self._pending_reads: dict[_Stream, asyncio.Future[AudioFrame]] = {}
|
|
61
|
+
self._stalled_rounds: dict[_Stream, int] = {}
|
|
62
|
+
self._sample_rate: int = sample_rate
|
|
63
|
+
self._num_channels: int = num_channels
|
|
64
|
+
self._chunk_size: int = blocksize if blocksize > 0 else int(sample_rate // 10)
|
|
65
|
+
self._stream_timeout_ms: int = stream_timeout_ms
|
|
66
|
+
self._queue: asyncio.Queue[Optional[AudioFrame]] = asyncio.Queue(maxsize=capacity)
|
|
67
|
+
# _ending signals that no new streams will be added,
|
|
68
|
+
# but we continue processing until all streams are exhausted.
|
|
69
|
+
self._ending: bool = False
|
|
70
|
+
self._mixer_task: asyncio.Task = asyncio.create_task(self._mixer())
|
|
71
|
+
|
|
72
|
+
def add_stream(self, stream: AsyncIterator[AudioFrame]) -> None:
|
|
73
|
+
"""
|
|
74
|
+
Add an audio stream to the mixer.
|
|
75
|
+
|
|
76
|
+
The stream is added to the internal set of streams and an empty buffer is initialized for it,
|
|
77
|
+
if not already present.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
stream (AsyncIterator[AudioFrame]): An async iterator that produces AudioFrame objects.
|
|
81
|
+
"""
|
|
82
|
+
if self._ending:
|
|
83
|
+
raise RuntimeError("Cannot add stream after mixer has been closed")
|
|
84
|
+
|
|
85
|
+
self._streams.add(stream)
|
|
86
|
+
if stream not in self._buffers:
|
|
87
|
+
self._buffers[stream] = np.empty((0, self._num_channels), dtype=np.int16)
|
|
88
|
+
|
|
89
|
+
def remove_stream(self, stream: AsyncIterator[AudioFrame]) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Remove an audio stream from the mixer.
|
|
92
|
+
|
|
93
|
+
This method removes the specified stream and its associated buffer from the mixer.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
stream (AsyncIterator[AudioFrame]): The audio stream to remove.
|
|
97
|
+
"""
|
|
98
|
+
self._streams.discard(stream)
|
|
99
|
+
self._buffers.pop(stream, None)
|
|
100
|
+
self._stalled_rounds.pop(stream, None)
|
|
101
|
+
pending = self._pending_reads.pop(stream, None)
|
|
102
|
+
if pending is not None:
|
|
103
|
+
_discard_read(pending)
|
|
104
|
+
|
|
105
|
+
def __aiter__(self) -> "AudioMixer":
|
|
106
|
+
return self
|
|
107
|
+
|
|
108
|
+
async def __anext__(self) -> AudioFrame:
|
|
109
|
+
item = await self._queue.get()
|
|
110
|
+
if item is None:
|
|
111
|
+
raise StopAsyncIteration
|
|
112
|
+
return item
|
|
113
|
+
|
|
114
|
+
async def aclose(self) -> None:
|
|
115
|
+
"""
|
|
116
|
+
Immediately stop mixing and close the mixer.
|
|
117
|
+
|
|
118
|
+
This cancels the mixing task, and any unconsumed output in the queue may be dropped.
|
|
119
|
+
Pending stream reads are cancelled and awaited, so a stream must let
|
|
120
|
+
asyncio.CancelledError propagate once its cleanup is done.
|
|
121
|
+
"""
|
|
122
|
+
self._ending = True
|
|
123
|
+
self._mixer_task.cancel()
|
|
124
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
125
|
+
await self._mixer_task
|
|
126
|
+
pending = list(self._pending_reads.values())
|
|
127
|
+
self._pending_reads.clear()
|
|
128
|
+
for read in pending:
|
|
129
|
+
_discard_read(read)
|
|
130
|
+
# let the streams finish their cleanup before the mixer counts as closed
|
|
131
|
+
await asyncio.gather(*pending, return_exceptions=True)
|
|
132
|
+
|
|
133
|
+
def end_input(self) -> None:
|
|
134
|
+
"""
|
|
135
|
+
Signal that no more streams will be added.
|
|
136
|
+
|
|
137
|
+
This method marks the mixer as closed so that it flushes any remaining buffered output before ending.
|
|
138
|
+
Note that existing streams will still be processed until exhausted.
|
|
139
|
+
"""
|
|
140
|
+
self._ending = True
|
|
141
|
+
|
|
142
|
+
async def _mixer(self) -> None:
|
|
143
|
+
while True:
|
|
144
|
+
# If we're in ending mode and there are no more streams, exit.
|
|
145
|
+
if self._ending and not self._streams:
|
|
146
|
+
break
|
|
147
|
+
|
|
148
|
+
if not self._streams:
|
|
149
|
+
await asyncio.sleep(0.01)
|
|
150
|
+
continue
|
|
151
|
+
|
|
152
|
+
tasks = [
|
|
153
|
+
self._get_contribution(
|
|
154
|
+
stream,
|
|
155
|
+
self._buffers.get(stream, np.empty((0, self._num_channels), dtype=np.int16)),
|
|
156
|
+
)
|
|
157
|
+
for stream in list(self._streams)
|
|
158
|
+
]
|
|
159
|
+
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
160
|
+
contributions = []
|
|
161
|
+
any_data = False
|
|
162
|
+
removals = []
|
|
163
|
+
for contrib in results:
|
|
164
|
+
if not isinstance(contrib, _Contribution):
|
|
165
|
+
continue
|
|
166
|
+
|
|
167
|
+
contributions.append(contrib.data.astype(np.float32))
|
|
168
|
+
self._buffers[contrib.stream] = contrib.buffer
|
|
169
|
+
if contrib.had_data:
|
|
170
|
+
any_data = True
|
|
171
|
+
if contrib.exhausted and contrib.buffer.shape[0] == 0:
|
|
172
|
+
removals.append(contrib.stream)
|
|
173
|
+
|
|
174
|
+
for stream in removals:
|
|
175
|
+
self.remove_stream(stream)
|
|
176
|
+
|
|
177
|
+
if not any_data:
|
|
178
|
+
await asyncio.sleep(0.001)
|
|
179
|
+
continue
|
|
180
|
+
|
|
181
|
+
mixed = np.sum(np.stack(contributions, axis=0), axis=0)
|
|
182
|
+
mixed = np.clip(mixed, -32768, 32767).astype(np.int16)
|
|
183
|
+
frame = AudioFrame(
|
|
184
|
+
mixed.tobytes(), self._sample_rate, self._num_channels, self._chunk_size
|
|
185
|
+
)
|
|
186
|
+
await self._queue.put(frame)
|
|
187
|
+
|
|
188
|
+
await self._queue.put(None)
|
|
189
|
+
|
|
190
|
+
async def _get_contribution(
|
|
191
|
+
self, stream: AsyncIterator[AudioFrame], buf: np.ndarray
|
|
192
|
+
) -> _Contribution:
|
|
193
|
+
had_data = buf.shape[0] > 0
|
|
194
|
+
exhausted = False
|
|
195
|
+
# a round scheduled before remove_stream() must not start a new read on that stream
|
|
196
|
+
while buf.shape[0] < self._chunk_size and not exhausted and stream in self._streams:
|
|
197
|
+
# The read stays in _pending_reads until it completes, so remove_stream()
|
|
198
|
+
# and aclose() can always cancel it.
|
|
199
|
+
read = self._pending_reads.get(stream)
|
|
200
|
+
if read is None:
|
|
201
|
+
read = asyncio.ensure_future(stream.__anext__())
|
|
202
|
+
self._pending_reads[stream] = read
|
|
203
|
+
done, _ = await asyncio.wait({read}, timeout=self._stream_timeout_ms / 1000)
|
|
204
|
+
if not done:
|
|
205
|
+
logger.warning(f"AudioMixer: stream {stream} timeout, ignoring")
|
|
206
|
+
stalled = self._stalled_rounds.get(stream, 0) + 1
|
|
207
|
+
self._stalled_rounds[stream] = stalled
|
|
208
|
+
# after end_input() a stream that keeps stalling is dropped, so the drain ends
|
|
209
|
+
exhausted = self._ending and stalled >= _MAX_STALLED_ROUNDS
|
|
210
|
+
break
|
|
211
|
+
self._stalled_rounds.pop(stream, None)
|
|
212
|
+
if self._pending_reads.get(stream) is read:
|
|
213
|
+
del self._pending_reads[stream]
|
|
214
|
+
try:
|
|
215
|
+
frame = read.result()
|
|
216
|
+
except StopAsyncIteration:
|
|
217
|
+
exhausted = True
|
|
218
|
+
break
|
|
219
|
+
except asyncio.TimeoutError:
|
|
220
|
+
# raised by the stream itself: handled like the mixer's own timeout, as before
|
|
221
|
+
logger.warning(f"AudioMixer: stream {stream} timeout, ignoring")
|
|
222
|
+
break
|
|
223
|
+
new_data = np.frombuffer(frame.data.tobytes(), dtype=np.int16).reshape(
|
|
224
|
+
-1, self._num_channels
|
|
225
|
+
)
|
|
226
|
+
buf = np.concatenate((buf, new_data), axis=0) if buf.size else new_data
|
|
227
|
+
had_data = True
|
|
228
|
+
if buf.shape[0] >= self._chunk_size:
|
|
229
|
+
contrib, buf = buf[: self._chunk_size], buf[self._chunk_size :]
|
|
230
|
+
else:
|
|
231
|
+
pad = np.zeros((self._chunk_size - buf.shape[0], self._num_channels), dtype=np.int16)
|
|
232
|
+
contrib, buf = (
|
|
233
|
+
np.concatenate((buf, pad), axis=0),
|
|
234
|
+
np.empty((0, self._num_channels), dtype=np.int16),
|
|
235
|
+
)
|
|
236
|
+
return _Contribution(stream, contrib, buf, had_data, exhausted)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _discard_read(read: "asyncio.Future[AudioFrame]") -> None:
|
|
240
|
+
"""Cancel a read that is no longer wanted and consume its outcome whenever it ends."""
|
|
241
|
+
read.cancel()
|
|
242
|
+
read.add_done_callback(_consume_outcome)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _consume_outcome(read: "asyncio.Future[AudioFrame]") -> None:
|
|
246
|
+
if not read.cancelled():
|
|
247
|
+
read.exception() # mark a failed read as handled
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ctypes
|
|
4
|
+
from enum import Enum, unique
|
|
5
|
+
|
|
6
|
+
from ._proto import audio_frame_pb2 as proto_audio_frame
|
|
7
|
+
from ._ffi_client import FfiClient, FfiHandle
|
|
8
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
9
|
+
from ._utils import get_address
|
|
10
|
+
from .audio_frame import AudioFrame
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@unique
|
|
14
|
+
class AudioResamplerQuality(str, Enum):
|
|
15
|
+
QUICK = "quick"
|
|
16
|
+
LOW = "low"
|
|
17
|
+
MEDIUM = "medium"
|
|
18
|
+
HIGH = "high"
|
|
19
|
+
VERY_HIGH = "very_high"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AudioResampler:
|
|
23
|
+
"""
|
|
24
|
+
A class for resampling audio data from one sample rate to another.
|
|
25
|
+
|
|
26
|
+
`AudioResampler` provides functionality to resample audio data from an input sample rate to an output
|
|
27
|
+
sample rate using the Sox resampling library. It supports multiple channels and configurable resampling quality.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
input_rate: int,
|
|
33
|
+
output_rate: int,
|
|
34
|
+
*,
|
|
35
|
+
num_channels: int = 1,
|
|
36
|
+
quality: AudioResamplerQuality = AudioResamplerQuality.MEDIUM,
|
|
37
|
+
) -> None:
|
|
38
|
+
"""
|
|
39
|
+
Initialize an `AudioResampler` instance for resampling audio data.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
input_rate (int): The sample rate of the input audio data (in Hz).
|
|
43
|
+
output_rate (int): The desired sample rate of the output audio data (in Hz).
|
|
44
|
+
num_channels (int, optional): The number of audio channels (e.g., 1 for mono, 2 for stereo). Defaults to 1.
|
|
45
|
+
quality (AudioResamplerQuality, optional): The quality setting for the resampler. Can be one of the
|
|
46
|
+
`AudioResamplerQuality` enum values: `QUICK`, `LOW`, `MEDIUM`, `HIGH`, `VERY_HIGH`. Higher quality settings
|
|
47
|
+
result in better audio quality but require more processing power. Defaults to `AudioResamplerQuality.MEDIUM`.
|
|
48
|
+
|
|
49
|
+
Raises:
|
|
50
|
+
Exception: If there is an error creating the resampler.
|
|
51
|
+
"""
|
|
52
|
+
self._input_rate = input_rate
|
|
53
|
+
self._output_rate = output_rate
|
|
54
|
+
self._num_channels = num_channels
|
|
55
|
+
|
|
56
|
+
req = proto_ffi.FfiRequest()
|
|
57
|
+
req.new_sox_resampler.input_rate = input_rate
|
|
58
|
+
req.new_sox_resampler.output_rate = output_rate
|
|
59
|
+
req.new_sox_resampler.num_channels = num_channels
|
|
60
|
+
req.new_sox_resampler.quality_recipe = _to_proto_quality(quality)
|
|
61
|
+
|
|
62
|
+
# not exposed for now
|
|
63
|
+
req.new_sox_resampler.input_data_type = (
|
|
64
|
+
proto_audio_frame.SoxResamplerDataType.SOXR_DATATYPE_INT16I
|
|
65
|
+
)
|
|
66
|
+
req.new_sox_resampler.output_data_type = (
|
|
67
|
+
proto_audio_frame.SoxResamplerDataType.SOXR_DATATYPE_INT16I
|
|
68
|
+
)
|
|
69
|
+
req.new_sox_resampler.flags = 0 # default
|
|
70
|
+
|
|
71
|
+
resp = FfiClient.instance.request(req)
|
|
72
|
+
|
|
73
|
+
if resp.new_sox_resampler.error:
|
|
74
|
+
raise Exception(resp.new_sox_resampler.error)
|
|
75
|
+
|
|
76
|
+
self._ffi_handle = FfiHandle(resp.new_sox_resampler.resampler.handle.id)
|
|
77
|
+
|
|
78
|
+
def push(self, data: bytearray | AudioFrame) -> list[AudioFrame]:
|
|
79
|
+
"""
|
|
80
|
+
Push audio data into the resampler and retrieve any available resampled data.
|
|
81
|
+
|
|
82
|
+
This method accepts audio data, resamples it according to the configured input and output rates,
|
|
83
|
+
and returns any resampled data that is available after processing the input.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
data (bytearray | AudioFrame): The audio data to resample. This can be a `bytearray` containing
|
|
87
|
+
raw audio bytes in int16le format or an `AudioFrame` object.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
list[AudioFrame]: A list of `AudioFrame` objects containing the resampled audio data.
|
|
91
|
+
The list may be empty if no output data is available yet.
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
Exception: If there is an error during resampling.
|
|
95
|
+
"""
|
|
96
|
+
bdata = data if isinstance(data, bytearray) else data.data.cast("b")
|
|
97
|
+
if not bdata:
|
|
98
|
+
return []
|
|
99
|
+
|
|
100
|
+
req = proto_ffi.FfiRequest()
|
|
101
|
+
req.push_sox_resampler.resampler_handle = self._ffi_handle.handle
|
|
102
|
+
req.push_sox_resampler.data_ptr = get_address(memoryview(bdata))
|
|
103
|
+
req.push_sox_resampler.size = len(bdata)
|
|
104
|
+
|
|
105
|
+
resp = FfiClient.instance.request(req)
|
|
106
|
+
|
|
107
|
+
if resp.push_sox_resampler.error:
|
|
108
|
+
raise Exception(resp.push_sox_resampler.error)
|
|
109
|
+
|
|
110
|
+
if not resp.push_sox_resampler.output_ptr:
|
|
111
|
+
return []
|
|
112
|
+
|
|
113
|
+
cdata = (ctypes.c_int8 * resp.push_sox_resampler.size).from_address(
|
|
114
|
+
resp.push_sox_resampler.output_ptr
|
|
115
|
+
)
|
|
116
|
+
output_data = bytearray(cdata)
|
|
117
|
+
return [
|
|
118
|
+
AudioFrame(
|
|
119
|
+
output_data,
|
|
120
|
+
self._output_rate,
|
|
121
|
+
self._num_channels,
|
|
122
|
+
len(output_data) // (self._num_channels * ctypes.sizeof(ctypes.c_int16)),
|
|
123
|
+
)
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
def flush(self) -> list[AudioFrame]:
|
|
127
|
+
"""
|
|
128
|
+
Flush any remaining audio data through the resampler and retrieve the resampled data.
|
|
129
|
+
|
|
130
|
+
This method should be called when no more input data will be provided to ensure that all internal
|
|
131
|
+
buffers are processed and all resampled data is output.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
list[AudioFrame]: A list of `AudioFrame` objects containing the remaining resampled audio data after flushing.
|
|
135
|
+
The list may be empty if no output data remains.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
Exception: If there is an error during flushing.
|
|
139
|
+
"""
|
|
140
|
+
req = proto_ffi.FfiRequest()
|
|
141
|
+
req.flush_sox_resampler.resampler_handle = self._ffi_handle.handle
|
|
142
|
+
|
|
143
|
+
resp = FfiClient.instance.request(req)
|
|
144
|
+
|
|
145
|
+
if not resp.flush_sox_resampler.output_ptr:
|
|
146
|
+
return []
|
|
147
|
+
|
|
148
|
+
cdata = (ctypes.c_int8 * resp.flush_sox_resampler.size).from_address(
|
|
149
|
+
resp.flush_sox_resampler.output_ptr
|
|
150
|
+
)
|
|
151
|
+
output_data = bytearray(cdata)
|
|
152
|
+
return [
|
|
153
|
+
AudioFrame(
|
|
154
|
+
output_data,
|
|
155
|
+
self._output_rate,
|
|
156
|
+
self._num_channels,
|
|
157
|
+
len(output_data) // (self._num_channels * ctypes.sizeof(ctypes.c_int16)),
|
|
158
|
+
)
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _to_proto_quality(
|
|
163
|
+
quality: AudioResamplerQuality,
|
|
164
|
+
) -> proto_audio_frame.SoxQualityRecipe.ValueType:
|
|
165
|
+
if quality == AudioResamplerQuality.QUICK:
|
|
166
|
+
return proto_audio_frame.SoxQualityRecipe.SOXR_QUALITY_QUICK
|
|
167
|
+
elif quality == AudioResamplerQuality.LOW:
|
|
168
|
+
return proto_audio_frame.SoxQualityRecipe.SOXR_QUALITY_LOW
|
|
169
|
+
elif quality == AudioResamplerQuality.MEDIUM:
|
|
170
|
+
return proto_audio_frame.SoxQualityRecipe.SOXR_QUALITY_MEDIUM
|
|
171
|
+
elif quality == AudioResamplerQuality.HIGH:
|
|
172
|
+
return proto_audio_frame.SoxQualityRecipe.SOXR_QUALITY_HIGH
|
|
173
|
+
elif quality == AudioResamplerQuality.VERY_HIGH:
|
|
174
|
+
return proto_audio_frame.SoxQualityRecipe.SOXR_QUALITY_VERYHIGH
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# Copyright 2023 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import time
|
|
18
|
+
import asyncio
|
|
19
|
+
|
|
20
|
+
from ._ffi_client import FfiHandle, FfiClient
|
|
21
|
+
from ._proto import audio_frame_pb2 as proto_audio_frame
|
|
22
|
+
from ._proto import ffi_pb2 as proto_ffi
|
|
23
|
+
from .audio_frame import AudioFrame
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AudioSource:
|
|
27
|
+
"""
|
|
28
|
+
Represents a real-time audio source with an internal audio queue.
|
|
29
|
+
|
|
30
|
+
The `AudioSource` class allows you to push audio frames into a real-time audio
|
|
31
|
+
source, managing an internal queue of audio data up to a maximum duration defined
|
|
32
|
+
by `queue_size_ms`. It supports asynchronous operations to capture audio frames
|
|
33
|
+
and to wait for the playback of all queued audio data.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
sample_rate: int,
|
|
39
|
+
num_channels: int,
|
|
40
|
+
queue_size_ms: int = 1000,
|
|
41
|
+
loop: asyncio.AbstractEventLoop | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""
|
|
44
|
+
Initializes a new instance of the audio source.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
sample_rate (int): The sample rate of the audio source in Hz.
|
|
48
|
+
num_channels (int): The number of audio channels.
|
|
49
|
+
queue_size_ms (int, optional): The buffer size of the audio queue in milliseconds.
|
|
50
|
+
Defaults to 1000 ms.
|
|
51
|
+
loop (asyncio.AbstractEventLoop, optional): The event loop to use. Defaults to
|
|
52
|
+
`asyncio.get_event_loop()`.
|
|
53
|
+
"""
|
|
54
|
+
self._sample_rate = sample_rate
|
|
55
|
+
self._num_channels = num_channels
|
|
56
|
+
self._loop = loop or asyncio.get_event_loop()
|
|
57
|
+
|
|
58
|
+
req = proto_ffi.FfiRequest()
|
|
59
|
+
req.new_audio_source.type = proto_audio_frame.AudioSourceType.AUDIO_SOURCE_NATIVE
|
|
60
|
+
req.new_audio_source.sample_rate = sample_rate
|
|
61
|
+
req.new_audio_source.num_channels = num_channels
|
|
62
|
+
req.new_audio_source.queue_size_ms = queue_size_ms
|
|
63
|
+
|
|
64
|
+
resp = FfiClient.instance.request(req)
|
|
65
|
+
self._info = resp.new_audio_source.source
|
|
66
|
+
self._ffi_handle = FfiHandle(self._info.handle.id)
|
|
67
|
+
|
|
68
|
+
self._last_capture = 0.0
|
|
69
|
+
self._q_size = 0.0
|
|
70
|
+
self._join_handle: asyncio.TimerHandle | None = None
|
|
71
|
+
self._join_fut: asyncio.Future[None] | None = None
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def sample_rate(self) -> int:
|
|
75
|
+
"""The sample rate of the audio source in Hz."""
|
|
76
|
+
return self._sample_rate
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def num_channels(self) -> int:
|
|
80
|
+
"""The number of audio channels."""
|
|
81
|
+
return self._num_channels
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def queued_duration(self) -> float:
|
|
85
|
+
"""The current duration (in seconds) of audio data queued for playback."""
|
|
86
|
+
return max(self._q_size - time.monotonic() + self._last_capture, 0.0)
|
|
87
|
+
|
|
88
|
+
def clear_queue(self) -> None:
|
|
89
|
+
"""
|
|
90
|
+
Clears the internal audio queue, discarding all buffered audio data.
|
|
91
|
+
|
|
92
|
+
This method immediately removes all audio data currently queued for playback,
|
|
93
|
+
effectively resetting the audio source's buffer. Any audio frames that have been
|
|
94
|
+
captured but not yet played will be discarded. This is useful in scenarios where
|
|
95
|
+
you need to stop playback abruptly or prevent outdated audio data from being played.
|
|
96
|
+
"""
|
|
97
|
+
req = proto_ffi.FfiRequest()
|
|
98
|
+
req.clear_audio_buffer.source_handle = self._ffi_handle.handle
|
|
99
|
+
_ = FfiClient.instance.request(req)
|
|
100
|
+
self._release_waiter()
|
|
101
|
+
|
|
102
|
+
async def capture_frame(self, frame: AudioFrame) -> None:
|
|
103
|
+
"""
|
|
104
|
+
Captures an `AudioFrame` and queues it for playback.
|
|
105
|
+
|
|
106
|
+
This method is used to push new audio data into the audio source. The audio data
|
|
107
|
+
will be processed and queued. If the size of the audio frame exceeds the internal
|
|
108
|
+
queue size, the method will wait until there is enough space in the queue to
|
|
109
|
+
accommodate the frame. The method returns only when all of the data in the buffer
|
|
110
|
+
has been pushed.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
frame (AudioFrame): The audio frame to capture and queue.
|
|
114
|
+
|
|
115
|
+
Raises:
|
|
116
|
+
Exception: If there is an error during frame capture.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
if frame.samples_per_channel == 0 or self._ffi_handle.disposed:
|
|
120
|
+
return
|
|
121
|
+
|
|
122
|
+
now = time.monotonic()
|
|
123
|
+
elapsed = 0.0 if self._last_capture == 0.0 else now - self._last_capture
|
|
124
|
+
self._q_size += frame.samples_per_channel / self.sample_rate - elapsed
|
|
125
|
+
self._last_capture = now
|
|
126
|
+
|
|
127
|
+
if self._join_handle:
|
|
128
|
+
self._join_handle.cancel()
|
|
129
|
+
|
|
130
|
+
if self._join_fut is None:
|
|
131
|
+
self._join_fut = self._loop.create_future()
|
|
132
|
+
|
|
133
|
+
self._join_handle = self._loop.call_later(self._q_size, self._release_waiter)
|
|
134
|
+
|
|
135
|
+
req = proto_ffi.FfiRequest()
|
|
136
|
+
req.capture_audio_frame.source_handle = self._ffi_handle.handle
|
|
137
|
+
req.capture_audio_frame.buffer.CopyFrom(frame._proto_info())
|
|
138
|
+
|
|
139
|
+
# The async_id is only known once the request returns; until then every capture
|
|
140
|
+
# callback is let through, and wait_for drops the ones of other captures.
|
|
141
|
+
async_id: int | None = None
|
|
142
|
+
queue = FfiClient.instance.queue.subscribe(
|
|
143
|
+
loop=self._loop,
|
|
144
|
+
filter_fn=lambda e: (
|
|
145
|
+
e.WhichOneof("message") == "capture_audio_frame"
|
|
146
|
+
and (async_id is None or e.capture_audio_frame.async_id == async_id)
|
|
147
|
+
),
|
|
148
|
+
)
|
|
149
|
+
try:
|
|
150
|
+
resp = FfiClient.instance.request(req)
|
|
151
|
+
async_id = resp.capture_audio_frame.async_id
|
|
152
|
+
cb: proto_ffi.FfiEvent = await queue.wait_for(
|
|
153
|
+
lambda e: e.capture_audio_frame.async_id == resp.capture_audio_frame.async_id
|
|
154
|
+
)
|
|
155
|
+
finally:
|
|
156
|
+
FfiClient.instance.queue.unsubscribe(queue)
|
|
157
|
+
|
|
158
|
+
if cb.capture_audio_frame.error:
|
|
159
|
+
raise Exception(cb.capture_audio_frame.error)
|
|
160
|
+
|
|
161
|
+
async def wait_for_playout(self) -> None:
|
|
162
|
+
"""
|
|
163
|
+
Waits for the audio source to finish playing out all audio data.
|
|
164
|
+
|
|
165
|
+
This method ensures that all queued audio data has been played out before returning.
|
|
166
|
+
It can be used to synchronize events after audio playback or to ensure that the
|
|
167
|
+
audio queue is empty.
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
if self._join_fut is None:
|
|
171
|
+
return
|
|
172
|
+
|
|
173
|
+
await asyncio.shield(self._join_fut)
|
|
174
|
+
|
|
175
|
+
def _release_waiter(self) -> None:
|
|
176
|
+
if self._join_fut is None:
|
|
177
|
+
return # could be None when clear_queue is called
|
|
178
|
+
|
|
179
|
+
if not self._join_fut.done():
|
|
180
|
+
self._join_fut.set_result(None)
|
|
181
|
+
|
|
182
|
+
self._last_capture = 0.0
|
|
183
|
+
self._q_size = 0.0
|
|
184
|
+
self._join_fut = None
|
|
185
|
+
|
|
186
|
+
async def aclose(self) -> None:
|
|
187
|
+
"""Close the audio source
|
|
188
|
+
|
|
189
|
+
This method cleans up resources associated with the audio source.
|
|
190
|
+
"""
|
|
191
|
+
self._ffi_handle.dispose()
|