pipecat-effects 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipecat_effects-0.1.0/LICENSE +24 -0
- pipecat_effects-0.1.0/PKG-INFO +293 -0
- pipecat_effects-0.1.0/README.md +271 -0
- pipecat_effects-0.1.0/pyproject.toml +102 -0
- pipecat_effects-0.1.0/pyproject.toml.orig +68 -0
- pipecat_effects-0.1.0/src/pipecat_effects/__init__.py +39 -0
- pipecat_effects-0.1.0/src/pipecat_effects/effects.py +270 -0
- pipecat_effects-0.1.0/src/pipecat_effects/filter.py +139 -0
- pipecat_effects-0.1.0/src/pipecat_effects/meter.py +55 -0
- pipecat_effects-0.1.0/src/pipecat_effects/mixer.py +54 -0
- pipecat_effects-0.1.0/src/pipecat_effects/primitives.py +303 -0
- pipecat_effects-0.1.0/src/pipecat_effects/py.typed +0 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Softcery OÜ
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pipecat-effects
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Output audio effects for pipecat. One filter, 8 effects and a loudness meter.
|
|
5
|
+
Keywords: pipecat,audio,effects,voice
|
|
6
|
+
Author: Softcery
|
|
7
|
+
License-Expression: BSD-2-Clause
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: pipecat-ai>=1.10,<2
|
|
16
|
+
Requires-Dist: numpy>=2.0
|
|
17
|
+
Requires-Dist: scipy>=1.14
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Project-URL: Source, https://github.com/softcery/pipecat-effects
|
|
20
|
+
Project-URL: Issues, https://github.com/softcery/pipecat-effects/issues
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# pipecat-effects
|
|
24
|
+
|
|
25
|
+
Output audio effects for [pipecat](https://github.com/pipecat-ai/pipecat). One filter, 8 effects
|
|
26
|
+
and one loudness meter. You build the chain. The chain adds 0 samples of latency.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
pip install pipecat-effects
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Use
|
|
35
|
+
|
|
36
|
+
Pipecat 1.10.0 has no output filter field. `FilterMixer` runs the filter as the output mixer.
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from pipecat.transports.base_transport import TransportParams
|
|
40
|
+
from pipecat_effects import (
|
|
41
|
+
AGC,
|
|
42
|
+
Biquad,
|
|
43
|
+
Compressor,
|
|
44
|
+
DeEsser,
|
|
45
|
+
Effects,
|
|
46
|
+
EffectsFilter,
|
|
47
|
+
FilterMixer,
|
|
48
|
+
Limiter,
|
|
49
|
+
Saturation,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
CHAIN: Effects = (
|
|
53
|
+
AGC(target_lufs=-20.0),
|
|
54
|
+
Biquad(kind="highpass", hz=90.0),
|
|
55
|
+
Biquad(kind="lowshelf", hz=200.0, gain_db=2.5),
|
|
56
|
+
Biquad(kind="peak", hz=3200.0, q=1.2, gain_db=-2.0),
|
|
57
|
+
DeEsser(hz=6500.0, threshold_db=-32.0, ratio=4.0),
|
|
58
|
+
Compressor(threshold_db=-20.0, ratio=3.0, attack_ms=8.0, release_ms=120.0, makeup_db=2.0),
|
|
59
|
+
Saturation(drive=1.2, mix=0.15),
|
|
60
|
+
Limiter(ceiling_db=-1.0, knee_db=3.0),
|
|
61
|
+
)
|
|
62
|
+
CHANNELS = 1
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def transport_params() -> TransportParams:
|
|
66
|
+
return TransportParams(
|
|
67
|
+
audio_in_enabled=True,
|
|
68
|
+
audio_out_enabled=True,
|
|
69
|
+
audio_out_channels=CHANNELS,
|
|
70
|
+
audio_out_mixer=FilterMixer(EffectsFilter(CHAIN), channels=CHANNELS),
|
|
71
|
+
)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
`examples/bot.py` runs one voice bot with that chain. It needs 4 pipecat extras and
|
|
75
|
+
`OPENAI_API_KEY`. The wheel has no `examples/`, so run it from a clone of this repository.
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
pip install pipecat-effects "pipecat-ai[runner,webrtc,openai,silero]"
|
|
79
|
+
python examples/bot.py
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Listen
|
|
83
|
+
|
|
84
|
+
One sentence of Cartesia sonic-3.5 speech, 24 kHz mono, run in 40 ms chunks, the stock output
|
|
85
|
+
chunk of pipecat. `examples/chains.py` holds each chain. `examples/clips.py` renders the clips
|
|
86
|
+
with ffmpeg. GitHub mutes each player at load. Unmute it to listen. On PyPI each player is a
|
|
87
|
+
link. Loudness is integrated, measured by ffmpeg `ebur128`.
|
|
88
|
+
|
|
89
|
+
### Unprocessed
|
|
90
|
+
|
|
91
|
+
https://github.com/user-attachments/assets/9b4a4b80-8168-4380-a12d-b98f9fdcaaa7
|
|
92
|
+
|
|
93
|
+
No filter. -18.3 LUFS, -1.3 dBTP.
|
|
94
|
+
|
|
95
|
+
### EQ and compression
|
|
96
|
+
|
|
97
|
+
https://github.com/user-attachments/assets/33bbbafc-3627-416c-84f0-baf398f4922f
|
|
98
|
+
|
|
99
|
+
The chain in [Use](#use): 2.5 dB low shelf at 200 Hz, 2 dB cut at 3.2 kHz, de-esser, 3:1
|
|
100
|
+
compressor. -23.2 LUFS, -5.5 dBTP.
|
|
101
|
+
|
|
102
|
+
### Telephone
|
|
103
|
+
|
|
104
|
+
https://github.com/user-attachments/assets/6fff6b7a-ef53-44a1-b275-5e29ca4ab1bb
|
|
105
|
+
|
|
106
|
+
300 Hz to 3400 Hz band, 4:1 compressor, drive 3.0 at 0.3 mix. -22.0 LUFS, -5.1 dBTP.
|
|
107
|
+
|
|
108
|
+
### Broadcast
|
|
109
|
+
|
|
110
|
+
https://github.com/user-attachments/assets/0e336600-fde6-4ddb-8210-bd74be7fe0b4
|
|
111
|
+
|
|
112
|
+
100 Hz high-pass, de-esser, 4:1 compressor with 0.5 ms attack and 18 dB makeup, +4 dB at 3.5 kHz.
|
|
113
|
+
-16.3 LUFS, -1.4 dBTP. Against the unprocessed clip: +2.0 LU, and +3.7 dB in the share of power
|
|
114
|
+
from 2 to 4 kHz.
|
|
115
|
+
|
|
116
|
+
### Room
|
|
117
|
+
|
|
118
|
+
https://github.com/user-attachments/assets/6cbc794c-5a33-4cb4-a2be-ccfebffe1f79
|
|
119
|
+
|
|
120
|
+
Schroeder reverb, 400 ms decay, 0.3 mix. -21.2 LUFS, -3.5 dBTP.
|
|
121
|
+
|
|
122
|
+
## Build a chain
|
|
123
|
+
|
|
124
|
+
- A chain is any sequence of effects. The filter runs them in order and adds no stage.
|
|
125
|
+
- Build one filter per session. Each effect holds its state across chunks.
|
|
126
|
+
- Put `AGC` first. It reads the level before any stage changes it.
|
|
127
|
+
- If a chain ends without `Limiter`, the int16 cast hard clips each sample over full scale. End
|
|
128
|
+
each chain with `Limiter`.
|
|
129
|
+
- Give `FilterMixer` the `audio_out_channels` value as `channels`. `start` raises over 1 channel.
|
|
130
|
+
- Each effect validates its values at build time. A value outside the range raises `ValueError`
|
|
131
|
+
that names the field and the range.
|
|
132
|
+
|
|
133
|
+
## Effects
|
|
134
|
+
|
|
135
|
+
| effect | field | default | range |
|
|
136
|
+
| --- | --- | --- | --- |
|
|
137
|
+
| `Gain` | `db` | 0.0 | -60 to 24 |
|
|
138
|
+
| `Biquad` | `kind` | required | one of lowpass, highpass, bandpass, peak, lowshelf, highshelf |
|
|
139
|
+
| | `hz` | required | 10 to 20000 |
|
|
140
|
+
| | `q` | 0.7071 | 0.1 to 20 |
|
|
141
|
+
| | `gain_db` | 0.0 | -24 to 24 |
|
|
142
|
+
| `Saturation` | `drive` | 2.0 | 0.1 to 20 |
|
|
143
|
+
| | `mix` | 1.0 | 0 to 1 |
|
|
144
|
+
| `Compressor` | `threshold_db` | -18.0 | -60 to 0 |
|
|
145
|
+
| | `ratio` | 3.0 | 1 to 20 |
|
|
146
|
+
| | `attack_ms` | 5.0 | 0 to 200 |
|
|
147
|
+
| | `release_ms` | 80.0 | 1 to 2000 |
|
|
148
|
+
| | `makeup_db` | 0.0 | -24 to 24 |
|
|
149
|
+
| `AGC` | `target_lufs` | -20.0 | -40 to -10 |
|
|
150
|
+
| | `max_db_per_second` | 6.0 | 0.1 to 20 |
|
|
151
|
+
| `Limiter` | `ceiling_db` | -1.0 | -24 to 0 |
|
|
152
|
+
| | `knee_db` | 3.0 | 0 to 12 |
|
|
153
|
+
| `Reverb` | `decay_ms` | 200.0 | 10 to 500 |
|
|
154
|
+
| | `mix` | 0.15 | 0 to 1 |
|
|
155
|
+
| `DeEsser` | `hz` | 6500.0 | 1000 to 20000 |
|
|
156
|
+
| | `q` | 1.5 | 0.1 to 20 |
|
|
157
|
+
| | `threshold_db` | -30.0 | -60 to 0 |
|
|
158
|
+
| | `ratio` | 4.0 | 1 to 20 |
|
|
159
|
+
| | `attack_ms` | 1.0 | 0 to 200 |
|
|
160
|
+
| | `release_ms` | 40.0 | 1 to 2000 |
|
|
161
|
+
|
|
162
|
+
`mix` sets the dry/wet ratio. 0 gives only the input, the dry signal. 1 gives only the processed
|
|
163
|
+
signal, the wet signal.
|
|
164
|
+
|
|
165
|
+
Method of each effect:
|
|
166
|
+
|
|
167
|
+
| effect | method |
|
|
168
|
+
| --- | --- |
|
|
169
|
+
| `Gain` | one multiply |
|
|
170
|
+
| `Biquad` | one second-order section, Audio EQ Cookbook by Robert Bristow-Johnson |
|
|
171
|
+
| `Saturation` | tanh waveshaper, `tanh(drive * x) / tanh(drive)`, dry/wet mix |
|
|
172
|
+
| `Compressor` | one envelope follower |
|
|
173
|
+
| `AGC` | K-weighted loudness over 400 ms, gain ramped at `max_db_per_second` or less |
|
|
174
|
+
| `Limiter` | memoryless soft clipper, soft knee, hard ceiling on sample peaks |
|
|
175
|
+
| `Reverb` | Schroeder reverberator, 4 comb filters and 2 allpass filters |
|
|
176
|
+
| `DeEsser` | split-band, the envelope of one band-pass band sets the cut of that band |
|
|
177
|
+
|
|
178
|
+
## Control
|
|
179
|
+
|
|
180
|
+
Change the chain at runtime with 2 stock frames.
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame
|
|
184
|
+
from pipecat_effects import Gain, Limiter
|
|
185
|
+
|
|
186
|
+
await worker.queue_frame(MixerEnableFrame(enable=False))
|
|
187
|
+
await worker.queue_frame(MixerUpdateSettingsFrame(settings={"effects": (Gain(db=-3.0), Limiter())}))
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
- `MixerEnableFrame(enable=False)` bypasses the chain. The filter gives the input, and the chain
|
|
191
|
+
keeps running on it, so envelopes and delay lines stay current.
|
|
192
|
+
- `MixerUpdateSettingsFrame` with an `effects` key builds a new chain and swaps it in.
|
|
193
|
+
- Each change fades over one chunk, from the output last heard to the new output. If 2 updates
|
|
194
|
+
arrive before one chunk, the chain last heard fades to the last chain. The chain between them
|
|
195
|
+
is not heard.
|
|
196
|
+
- `FilterMixer` maps the 2 mixer frames to `FilterEnableFrame` and `FilterUpdateSettingsFrame`.
|
|
197
|
+
Call `EffectsFilter.process_frame` with those 2 frames when you hold the filter directly.
|
|
198
|
+
- An `effects` value outside a sequence of effects raises `TypeError` that names the field. An
|
|
199
|
+
item whose `start` gives no callable raises `TypeError` that names the field and the index.
|
|
200
|
+
- A chain that fails to build raises `ValueError` that names the `effects` field. The old chain
|
|
201
|
+
keeps running.
|
|
202
|
+
- An update before `start` builds at `start`. A failed build then raises at `start` and names the
|
|
203
|
+
field of the effect, not `effects`.
|
|
204
|
+
|
|
205
|
+
## Meter
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
reading = mixer.read() # Reading(lufs=-19.92, dbtp=-1.04), or None on silence
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
- `FilterMixer` meters each chunk it gives to the transport, after the int16 cast.
|
|
212
|
+
- `FilterMixer.read()` gives a `Reading` with `lufs` and `dbtp` since the last read, then clears
|
|
213
|
+
both. Silence gives `None`.
|
|
214
|
+
- `lufs` is the K-weighted loudness of ITU-R BS.1770, the loudness standard. The published 48 kHz filter moves to the
|
|
215
|
+
session rate by the bilinear transform. A 997 Hz sine at 0 dBFS reads -3.01 LUFS.
|
|
216
|
+
- `dbtp` is the true peak, on 4 times oversampling with 48 taps.
|
|
217
|
+
- `Meter` reads float chunks for any other caller. Its `read` gives the floor, -120.0 dB, in
|
|
218
|
+
place of `None`.
|
|
219
|
+
|
|
220
|
+
## Cost
|
|
221
|
+
|
|
222
|
+
One 40 ms chunk at 24 kHz, the stock output chunk of pipecat. The 8 effect chain above, mean of
|
|
223
|
+
1000 chunks, median of 4 runs. Python 3.13.9, macOS arm64.
|
|
224
|
+
|
|
225
|
+
| path | mean | 95th |
|
|
226
|
+
| --- | --- | --- |
|
|
227
|
+
| filter | 0.245 ms | 0.265 ms |
|
|
228
|
+
| mixer on silence with the meter | 0.315 ms | 0.347 ms |
|
|
229
|
+
| true peak | 0.028 ms | 0.030 ms |
|
|
230
|
+
| loudness and true peak | 0.063 ms | 0.069 ms |
|
|
231
|
+
|
|
232
|
+
`python examples/bench.py --out rows.jsonl --sha <commit>` writes one row.
|
|
233
|
+
|
|
234
|
+
## Limits
|
|
235
|
+
|
|
236
|
+
### Chain
|
|
237
|
+
|
|
238
|
+
- The filter runs on mono. `FilterMixer.start` raises on more than 1 channel.
|
|
239
|
+
- The chain runs on each chunk, silence included, so attack and release stay continuous through
|
|
240
|
+
silence. At the stock `audio_out_10ms_chunks` of 4, one idle session costs 25 silent chunks of
|
|
241
|
+
40 ms a second, 7.9 ms of compute a second on the [Cost](#cost) machine.
|
|
242
|
+
- A bypassed chain keeps running, so it costs the same as an active chain.
|
|
243
|
+
- Outside a bypass, the chunk after an update runs the old chain and the new chain.
|
|
244
|
+
- A chunk of 0 samples passes through. An odd byte count raises `ValueError` that names `audio`.
|
|
245
|
+
- A `Biquad` or `DeEsser` centre over 0.45 of the sample rate raises at `start`, with the highest
|
|
246
|
+
allowed centre and the rate. The error names `hz`, not the effect.
|
|
247
|
+
- At 8 kHz the highest centre is 3600 Hz. The chain in [Use](#use) raises there, since its
|
|
248
|
+
`DeEsser` sits at 6500 Hz.
|
|
249
|
+
- Not included: lookahead, convolution, pitch shift, formant shift. The broadcast clip holds a
|
|
250
|
+
peak-to-loudness ratio, true peak minus loudness, of 14.9 dB. The unprocessed clip holds
|
|
251
|
+
17.0 dB. A hall reverb needs convolution.
|
|
252
|
+
|
|
253
|
+
### Effects
|
|
254
|
+
|
|
255
|
+
- `Compressor` and `DeEsser` set the gain of each 1 ms block from the peak of that block. `AGC`
|
|
256
|
+
sets one target per chunk from the loudness at the chunk end. Each gain reads ahead inside its
|
|
257
|
+
chunk, by up to 1 ms or 1 chunk.
|
|
258
|
+
- `AGC` reads momentary loudness without the gate of BS.1770. Under -50 LUFS the gain holds.
|
|
259
|
+
- `AGC` sets the level at its place in the chain. Later stages move the output level. The EQ and
|
|
260
|
+
compression clip holds `AGC(target_lufs=-20.0)` and measures -23.2 LUFS.
|
|
261
|
+
- `Limiter` is a memoryless soft clipper. It has no attack or release, so a signal driven far
|
|
262
|
+
over the ceiling distorts.
|
|
263
|
+
- The -1 dB ceiling leaves the margin for inter-sample peaks at a codec resampler. The true peak
|
|
264
|
+
meter only reports.
|
|
265
|
+
- One section falls 12 dB per octave. A lowpass at 6000 Hz drops an 8 kHz tone by 10.0 dB at a
|
|
266
|
+
24 kHz rate.
|
|
267
|
+
- `Reverb` takes `decay_ms` to 500.
|
|
268
|
+
|
|
269
|
+
### Meter
|
|
270
|
+
|
|
271
|
+
- The true peak meter cuts at the input Nyquist. At a 24 kHz rate it reads a 10 kHz sine 0.5 dB
|
|
272
|
+
under its peak.
|
|
273
|
+
- The K-weighting at 8 kHz differs from the standard by 0.43 dB at most from 100 Hz to 3 kHz. At
|
|
274
|
+
24 kHz it differs by 0.040 dB at most.
|
|
275
|
+
|
|
276
|
+
### Pipecat
|
|
277
|
+
|
|
278
|
+
- With an output mixer, a flush that fails to drain does not time out. Each mixer chunk reaches
|
|
279
|
+
the pipeline sink and counts as progress. A flush that drains returns. The defect is in
|
|
280
|
+
pipecat 1.10.0 and holds for each output mixer.
|
|
281
|
+
|
|
282
|
+
## Develop
|
|
283
|
+
|
|
284
|
+
- The package ships a `py.typed` marker. It exports `Effect`, `Apply` and `Samples` for a caller
|
|
285
|
+
who writes an effect.
|
|
286
|
+
- `make lint` checks the lock, the format, the lint rules, and the types with pyright.
|
|
287
|
+
- `make test` runs the tests. `make test-lowest` runs them on the lowest allowed pipecat-ai,
|
|
288
|
+
numpy and scipy. CI runs both.
|
|
289
|
+
- `make audit` checks `uv.lock` for known vulnerabilities.
|
|
290
|
+
|
|
291
|
+
## License
|
|
292
|
+
|
|
293
|
+
BSD 2-Clause.
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# pipecat-effects
|
|
2
|
+
|
|
3
|
+
Output audio effects for [pipecat](https://github.com/pipecat-ai/pipecat). One filter, 8 effects
|
|
4
|
+
and one loudness meter. You build the chain. The chain adds 0 samples of latency.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
pip install pipecat-effects
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Use
|
|
13
|
+
|
|
14
|
+
Pipecat 1.10.0 has no output filter field. `FilterMixer` runs the filter as the output mixer.
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from pipecat.transports.base_transport import TransportParams
|
|
18
|
+
from pipecat_effects import (
|
|
19
|
+
AGC,
|
|
20
|
+
Biquad,
|
|
21
|
+
Compressor,
|
|
22
|
+
DeEsser,
|
|
23
|
+
Effects,
|
|
24
|
+
EffectsFilter,
|
|
25
|
+
FilterMixer,
|
|
26
|
+
Limiter,
|
|
27
|
+
Saturation,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
CHAIN: Effects = (
|
|
31
|
+
AGC(target_lufs=-20.0),
|
|
32
|
+
Biquad(kind="highpass", hz=90.0),
|
|
33
|
+
Biquad(kind="lowshelf", hz=200.0, gain_db=2.5),
|
|
34
|
+
Biquad(kind="peak", hz=3200.0, q=1.2, gain_db=-2.0),
|
|
35
|
+
DeEsser(hz=6500.0, threshold_db=-32.0, ratio=4.0),
|
|
36
|
+
Compressor(threshold_db=-20.0, ratio=3.0, attack_ms=8.0, release_ms=120.0, makeup_db=2.0),
|
|
37
|
+
Saturation(drive=1.2, mix=0.15),
|
|
38
|
+
Limiter(ceiling_db=-1.0, knee_db=3.0),
|
|
39
|
+
)
|
|
40
|
+
CHANNELS = 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def transport_params() -> TransportParams:
|
|
44
|
+
return TransportParams(
|
|
45
|
+
audio_in_enabled=True,
|
|
46
|
+
audio_out_enabled=True,
|
|
47
|
+
audio_out_channels=CHANNELS,
|
|
48
|
+
audio_out_mixer=FilterMixer(EffectsFilter(CHAIN), channels=CHANNELS),
|
|
49
|
+
)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`examples/bot.py` runs one voice bot with that chain. It needs 4 pipecat extras and
|
|
53
|
+
`OPENAI_API_KEY`. The wheel has no `examples/`, so run it from a clone of this repository.
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
pip install pipecat-effects "pipecat-ai[runner,webrtc,openai,silero]"
|
|
57
|
+
python examples/bot.py
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Listen
|
|
61
|
+
|
|
62
|
+
One sentence of Cartesia sonic-3.5 speech, 24 kHz mono, run in 40 ms chunks, the stock output
|
|
63
|
+
chunk of pipecat. `examples/chains.py` holds each chain. `examples/clips.py` renders the clips
|
|
64
|
+
with ffmpeg. GitHub mutes each player at load. Unmute it to listen. On PyPI each player is a
|
|
65
|
+
link. Loudness is integrated, measured by ffmpeg `ebur128`.
|
|
66
|
+
|
|
67
|
+
### Unprocessed
|
|
68
|
+
|
|
69
|
+
https://github.com/user-attachments/assets/9b4a4b80-8168-4380-a12d-b98f9fdcaaa7
|
|
70
|
+
|
|
71
|
+
No filter. -18.3 LUFS, -1.3 dBTP.
|
|
72
|
+
|
|
73
|
+
### EQ and compression
|
|
74
|
+
|
|
75
|
+
https://github.com/user-attachments/assets/33bbbafc-3627-416c-84f0-baf398f4922f
|
|
76
|
+
|
|
77
|
+
The chain in [Use](#use): 2.5 dB low shelf at 200 Hz, 2 dB cut at 3.2 kHz, de-esser, 3:1
|
|
78
|
+
compressor. -23.2 LUFS, -5.5 dBTP.
|
|
79
|
+
|
|
80
|
+
### Telephone
|
|
81
|
+
|
|
82
|
+
https://github.com/user-attachments/assets/6fff6b7a-ef53-44a1-b275-5e29ca4ab1bb
|
|
83
|
+
|
|
84
|
+
300 Hz to 3400 Hz band, 4:1 compressor, drive 3.0 at 0.3 mix. -22.0 LUFS, -5.1 dBTP.
|
|
85
|
+
|
|
86
|
+
### Broadcast
|
|
87
|
+
|
|
88
|
+
https://github.com/user-attachments/assets/0e336600-fde6-4ddb-8210-bd74be7fe0b4
|
|
89
|
+
|
|
90
|
+
100 Hz high-pass, de-esser, 4:1 compressor with 0.5 ms attack and 18 dB makeup, +4 dB at 3.5 kHz.
|
|
91
|
+
-16.3 LUFS, -1.4 dBTP. Against the unprocessed clip: +2.0 LU, and +3.7 dB in the share of power
|
|
92
|
+
from 2 to 4 kHz.
|
|
93
|
+
|
|
94
|
+
### Room
|
|
95
|
+
|
|
96
|
+
https://github.com/user-attachments/assets/6cbc794c-5a33-4cb4-a2be-ccfebffe1f79
|
|
97
|
+
|
|
98
|
+
Schroeder reverb, 400 ms decay, 0.3 mix. -21.2 LUFS, -3.5 dBTP.
|
|
99
|
+
|
|
100
|
+
## Build a chain
|
|
101
|
+
|
|
102
|
+
- A chain is any sequence of effects. The filter runs them in order and adds no stage.
|
|
103
|
+
- Build one filter per session. Each effect holds its state across chunks.
|
|
104
|
+
- Put `AGC` first. It reads the level before any stage changes it.
|
|
105
|
+
- If a chain ends without `Limiter`, the int16 cast hard clips each sample over full scale. End
|
|
106
|
+
each chain with `Limiter`.
|
|
107
|
+
- Give `FilterMixer` the `audio_out_channels` value as `channels`. `start` raises over 1 channel.
|
|
108
|
+
- Each effect validates its values at build time. A value outside the range raises `ValueError`
|
|
109
|
+
that names the field and the range.
|
|
110
|
+
|
|
111
|
+
## Effects
|
|
112
|
+
|
|
113
|
+
| effect | field | default | range |
|
|
114
|
+
| --- | --- | --- | --- |
|
|
115
|
+
| `Gain` | `db` | 0.0 | -60 to 24 |
|
|
116
|
+
| `Biquad` | `kind` | required | one of lowpass, highpass, bandpass, peak, lowshelf, highshelf |
|
|
117
|
+
| | `hz` | required | 10 to 20000 |
|
|
118
|
+
| | `q` | 0.7071 | 0.1 to 20 |
|
|
119
|
+
| | `gain_db` | 0.0 | -24 to 24 |
|
|
120
|
+
| `Saturation` | `drive` | 2.0 | 0.1 to 20 |
|
|
121
|
+
| | `mix` | 1.0 | 0 to 1 |
|
|
122
|
+
| `Compressor` | `threshold_db` | -18.0 | -60 to 0 |
|
|
123
|
+
| | `ratio` | 3.0 | 1 to 20 |
|
|
124
|
+
| | `attack_ms` | 5.0 | 0 to 200 |
|
|
125
|
+
| | `release_ms` | 80.0 | 1 to 2000 |
|
|
126
|
+
| | `makeup_db` | 0.0 | -24 to 24 |
|
|
127
|
+
| `AGC` | `target_lufs` | -20.0 | -40 to -10 |
|
|
128
|
+
| | `max_db_per_second` | 6.0 | 0.1 to 20 |
|
|
129
|
+
| `Limiter` | `ceiling_db` | -1.0 | -24 to 0 |
|
|
130
|
+
| | `knee_db` | 3.0 | 0 to 12 |
|
|
131
|
+
| `Reverb` | `decay_ms` | 200.0 | 10 to 500 |
|
|
132
|
+
| | `mix` | 0.15 | 0 to 1 |
|
|
133
|
+
| `DeEsser` | `hz` | 6500.0 | 1000 to 20000 |
|
|
134
|
+
| | `q` | 1.5 | 0.1 to 20 |
|
|
135
|
+
| | `threshold_db` | -30.0 | -60 to 0 |
|
|
136
|
+
| | `ratio` | 4.0 | 1 to 20 |
|
|
137
|
+
| | `attack_ms` | 1.0 | 0 to 200 |
|
|
138
|
+
| | `release_ms` | 40.0 | 1 to 2000 |
|
|
139
|
+
|
|
140
|
+
`mix` sets the dry/wet ratio. 0 gives only the input, the dry signal. 1 gives only the processed
|
|
141
|
+
signal, the wet signal.
|
|
142
|
+
|
|
143
|
+
Method of each effect:
|
|
144
|
+
|
|
145
|
+
| effect | method |
|
|
146
|
+
| --- | --- |
|
|
147
|
+
| `Gain` | one multiply |
|
|
148
|
+
| `Biquad` | one second-order section, Audio EQ Cookbook by Robert Bristow-Johnson |
|
|
149
|
+
| `Saturation` | tanh waveshaper, `tanh(drive * x) / tanh(drive)`, dry/wet mix |
|
|
150
|
+
| `Compressor` | one envelope follower |
|
|
151
|
+
| `AGC` | K-weighted loudness over 400 ms, gain ramped at `max_db_per_second` or less |
|
|
152
|
+
| `Limiter` | memoryless soft clipper, soft knee, hard ceiling on sample peaks |
|
|
153
|
+
| `Reverb` | Schroeder reverberator, 4 comb filters and 2 allpass filters |
|
|
154
|
+
| `DeEsser` | split-band, the envelope of one band-pass band sets the cut of that band |
|
|
155
|
+
|
|
156
|
+
## Control
|
|
157
|
+
|
|
158
|
+
Change the chain at runtime with 2 stock frames.
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame
|
|
162
|
+
from pipecat_effects import Gain, Limiter
|
|
163
|
+
|
|
164
|
+
await worker.queue_frame(MixerEnableFrame(enable=False))
|
|
165
|
+
await worker.queue_frame(MixerUpdateSettingsFrame(settings={"effects": (Gain(db=-3.0), Limiter())}))
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
- `MixerEnableFrame(enable=False)` bypasses the chain. The filter gives the input, and the chain
|
|
169
|
+
keeps running on it, so envelopes and delay lines stay current.
|
|
170
|
+
- `MixerUpdateSettingsFrame` with an `effects` key builds a new chain and swaps it in.
|
|
171
|
+
- Each change fades over one chunk, from the output last heard to the new output. If 2 updates
|
|
172
|
+
arrive before one chunk, the chain last heard fades to the last chain. The chain between them
|
|
173
|
+
is not heard.
|
|
174
|
+
- `FilterMixer` maps the 2 mixer frames to `FilterEnableFrame` and `FilterUpdateSettingsFrame`.
|
|
175
|
+
Call `EffectsFilter.process_frame` with those 2 frames when you hold the filter directly.
|
|
176
|
+
- An `effects` value outside a sequence of effects raises `TypeError` that names the field. An
|
|
177
|
+
item whose `start` gives no callable raises `TypeError` that names the field and the index.
|
|
178
|
+
- A chain that fails to build raises `ValueError` that names the `effects` field. The old chain
|
|
179
|
+
keeps running.
|
|
180
|
+
- An update before `start` builds at `start`. A failed build then raises at `start` and names the
|
|
181
|
+
field of the effect, not `effects`.
|
|
182
|
+
|
|
183
|
+
## Meter
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
reading = mixer.read() # Reading(lufs=-19.92, dbtp=-1.04), or None on silence
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
- `FilterMixer` meters each chunk it gives to the transport, after the int16 cast.
|
|
190
|
+
- `FilterMixer.read()` gives a `Reading` with `lufs` and `dbtp` since the last read, then clears
|
|
191
|
+
both. Silence gives `None`.
|
|
192
|
+
- `lufs` is the K-weighted loudness of ITU-R BS.1770, the loudness standard. The published 48 kHz filter moves to the
|
|
193
|
+
session rate by the bilinear transform. A 997 Hz sine at 0 dBFS reads -3.01 LUFS.
|
|
194
|
+
- `dbtp` is the true peak, on 4 times oversampling with 48 taps.
|
|
195
|
+
- `Meter` reads float chunks for any other caller. Its `read` gives the floor, -120.0 dB, in
|
|
196
|
+
place of `None`.
|
|
197
|
+
|
|
198
|
+
## Cost
|
|
199
|
+
|
|
200
|
+
One 40 ms chunk at 24 kHz, the stock output chunk of pipecat. The 8 effect chain above, mean of
|
|
201
|
+
1000 chunks, median of 4 runs. Python 3.13.9, macOS arm64.
|
|
202
|
+
|
|
203
|
+
| path | mean | 95th |
|
|
204
|
+
| --- | --- | --- |
|
|
205
|
+
| filter | 0.245 ms | 0.265 ms |
|
|
206
|
+
| mixer on silence with the meter | 0.315 ms | 0.347 ms |
|
|
207
|
+
| true peak | 0.028 ms | 0.030 ms |
|
|
208
|
+
| loudness and true peak | 0.063 ms | 0.069 ms |
|
|
209
|
+
|
|
210
|
+
`python examples/bench.py --out rows.jsonl --sha <commit>` writes one row.
|
|
211
|
+
|
|
212
|
+
## Limits
|
|
213
|
+
|
|
214
|
+
### Chain
|
|
215
|
+
|
|
216
|
+
- The filter runs on mono. `FilterMixer.start` raises on more than 1 channel.
|
|
217
|
+
- The chain runs on each chunk, silence included, so attack and release stay continuous through
|
|
218
|
+
silence. At the stock `audio_out_10ms_chunks` of 4, one idle session costs 25 silent chunks of
|
|
219
|
+
40 ms a second, 7.9 ms of compute a second on the [Cost](#cost) machine.
|
|
220
|
+
- A bypassed chain keeps running, so it costs the same as an active chain.
|
|
221
|
+
- Outside a bypass, the chunk after an update runs the old chain and the new chain.
|
|
222
|
+
- A chunk of 0 samples passes through. An odd byte count raises `ValueError` that names `audio`.
|
|
223
|
+
- A `Biquad` or `DeEsser` centre over 0.45 of the sample rate raises at `start`, with the highest
|
|
224
|
+
allowed centre and the rate. The error names `hz`, not the effect.
|
|
225
|
+
- At 8 kHz the highest centre is 3600 Hz. The chain in [Use](#use) raises there, since its
|
|
226
|
+
`DeEsser` sits at 6500 Hz.
|
|
227
|
+
- Not included: lookahead, convolution, pitch shift, formant shift. The broadcast clip holds a
|
|
228
|
+
peak-to-loudness ratio, true peak minus loudness, of 14.9 dB. The unprocessed clip holds
|
|
229
|
+
17.0 dB. A hall reverb needs convolution.
|
|
230
|
+
|
|
231
|
+
### Effects
|
|
232
|
+
|
|
233
|
+
- `Compressor` and `DeEsser` set the gain of each 1 ms block from the peak of that block. `AGC`
|
|
234
|
+
sets one target per chunk from the loudness at the chunk end. Each gain reads ahead inside its
|
|
235
|
+
chunk, by up to 1 ms or 1 chunk.
|
|
236
|
+
- `AGC` reads momentary loudness without the gate of BS.1770. Under -50 LUFS the gain holds.
|
|
237
|
+
- `AGC` sets the level at its place in the chain. Later stages move the output level. The EQ and
|
|
238
|
+
compression clip holds `AGC(target_lufs=-20.0)` and measures -23.2 LUFS.
|
|
239
|
+
- `Limiter` is a memoryless soft clipper. It has no attack or release, so a signal driven far
|
|
240
|
+
over the ceiling distorts.
|
|
241
|
+
- The -1 dB ceiling leaves the margin for inter-sample peaks at a codec resampler. The true peak
|
|
242
|
+
meter only reports.
|
|
243
|
+
- One section falls 12 dB per octave. A lowpass at 6000 Hz drops an 8 kHz tone by 10.0 dB at a
|
|
244
|
+
24 kHz rate.
|
|
245
|
+
- `Reverb` takes `decay_ms` to 500.
|
|
246
|
+
|
|
247
|
+
### Meter
|
|
248
|
+
|
|
249
|
+
- The true peak meter cuts at the input Nyquist. At a 24 kHz rate it reads a 10 kHz sine 0.5 dB
|
|
250
|
+
under its peak.
|
|
251
|
+
- The K-weighting at 8 kHz differs from the standard by 0.43 dB at most from 100 Hz to 3 kHz. At
|
|
252
|
+
24 kHz it differs by 0.040 dB at most.
|
|
253
|
+
|
|
254
|
+
### Pipecat
|
|
255
|
+
|
|
256
|
+
- With an output mixer, a flush that fails to drain does not time out. Each mixer chunk reaches
|
|
257
|
+
the pipeline sink and counts as progress. A flush that drains returns. The defect is in
|
|
258
|
+
pipecat 1.10.0 and holds for each output mixer.
|
|
259
|
+
|
|
260
|
+
## Develop
|
|
261
|
+
|
|
262
|
+
- The package ships a `py.typed` marker. It exports `Effect`, `Apply` and `Samples` for a caller
|
|
263
|
+
who writes an effect.
|
|
264
|
+
- `make lint` checks the lock, the format, the lint rules, and the types with pyright.
|
|
265
|
+
- `make test` runs the tests. `make test-lowest` runs them on the lowest allowed pipecat-ai,
|
|
266
|
+
numpy and scipy. CI runs both.
|
|
267
|
+
- `make audit` checks `uv.lock` for known vulnerabilities.
|
|
268
|
+
|
|
269
|
+
## License
|
|
270
|
+
|
|
271
|
+
BSD 2-Clause.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pipecat-effects"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Output audio effects for pipecat. One filter, 8 effects and a loudness meter."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "BSD-2-Clause"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.12"
|
|
9
|
+
keywords = [
|
|
10
|
+
"pipecat",
|
|
11
|
+
"audio",
|
|
12
|
+
"effects",
|
|
13
|
+
"voice",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Topic :: Multimedia :: Sound/Audio",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"pipecat-ai>=1.10,<2",
|
|
25
|
+
"numpy>=2.0",
|
|
26
|
+
"scipy>=1.14",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[[project.authors]]
|
|
30
|
+
name = "Softcery"
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Source = "https://github.com/softcery/pipecat-effects"
|
|
34
|
+
Issues = "https://github.com/softcery/pipecat-effects/issues"
|
|
35
|
+
|
|
36
|
+
[dependency-groups]
|
|
37
|
+
dev = [
|
|
38
|
+
"pyright[nodejs]>=1.1.414",
|
|
39
|
+
"pytest>=9.1.1",
|
|
40
|
+
"pytest-asyncio>=1.4.0",
|
|
41
|
+
"ruff>=0.16.7",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[tool.uv]
|
|
45
|
+
required-version = ">=0.12.13,<0.13"
|
|
46
|
+
|
|
47
|
+
[tool.ruff]
|
|
48
|
+
line-length = 100
|
|
49
|
+
src = [
|
|
50
|
+
"src",
|
|
51
|
+
"examples",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[tool.ruff.lint]
|
|
55
|
+
select = [
|
|
56
|
+
"E",
|
|
57
|
+
"W",
|
|
58
|
+
"F",
|
|
59
|
+
"I",
|
|
60
|
+
"UP",
|
|
61
|
+
"B",
|
|
62
|
+
"SIM",
|
|
63
|
+
"C4",
|
|
64
|
+
"RUF",
|
|
65
|
+
"PTH",
|
|
66
|
+
"RET",
|
|
67
|
+
"ASYNC",
|
|
68
|
+
"T20",
|
|
69
|
+
"ERA",
|
|
70
|
+
"S",
|
|
71
|
+
"BLE",
|
|
72
|
+
"A",
|
|
73
|
+
"N",
|
|
74
|
+
"PIE",
|
|
75
|
+
"PERF",
|
|
76
|
+
"FURB",
|
|
77
|
+
"NPY",
|
|
78
|
+
"PT",
|
|
79
|
+
"PGH",
|
|
80
|
+
"PLE",
|
|
81
|
+
"PLW",
|
|
82
|
+
"TRY",
|
|
83
|
+
]
|
|
84
|
+
ignore = ["TRY003"]
|
|
85
|
+
|
|
86
|
+
[tool.ruff.lint.isort]
|
|
87
|
+
known-first-party = ["pipecat_effects"]
|
|
88
|
+
|
|
89
|
+
[tool.ruff.lint.per-file-ignores]
|
|
90
|
+
"tests/**" = ["S101"]
|
|
91
|
+
|
|
92
|
+
[tool.pyright]
|
|
93
|
+
pythonVersion = "3.12"
|
|
94
|
+
|
|
95
|
+
[tool.pytest]
|
|
96
|
+
asyncio_mode = "auto"
|
|
97
|
+
strict = true
|
|
98
|
+
testpaths = ["tests"]
|
|
99
|
+
|
|
100
|
+
[build-system]
|
|
101
|
+
requires = ["uv_build>=0.12.13,<0.13"]
|
|
102
|
+
build-backend = "uv_build"
|