stenograf 0.4.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stenograf/__init__.py +14 -0
- stenograf/__main__.py +11 -0
- stenograf/aec.py +397 -0
- stenograf/asr/__init__.py +23 -0
- stenograf/asr/base.py +81 -0
- stenograf/asr/biasing.py +368 -0
- stenograf/asr/parakeet.py +254 -0
- stenograf/asr/parakeet_onnx.py +257 -0
- stenograf/asr/providers.py +97 -0
- stenograf/asr/registry.py +122 -0
- stenograf/asr/tokens.py +146 -0
- stenograf/assets/Stenograf.app/Contents/Info.plist +50 -0
- stenograf/assets/Stenograf.app/Contents/MacOS/Stenograf +0 -0
- stenograf/assets/Stenograf.app/Contents/Resources/Stenograf.icns +0 -0
- stenograf/assets/Stenograf.app/Contents/_CodeSignature/CodeResources +128 -0
- stenograf/assets/icon.png +0 -0
- stenograf/assets/tray.svg +28 -0
- stenograf/audio.py +216 -0
- stenograf/bin/stenodiar.exe +0 -0
- stenograf/captions.py +161 -0
- stenograf/capture/__init__.py +9 -0
- stenograf/capture/base.py +160 -0
- stenograf/capture/file.py +75 -0
- stenograf/capture/linux.py +213 -0
- stenograf/capture/macos.py +316 -0
- stenograf/capture/streaming.py +258 -0
- stenograf/capture/windows.py +266 -0
- stenograf/cli/__init__.py +99 -0
- stenograf/cli/doctor_cmd.py +154 -0
- stenograf/cli/format.py +81 -0
- stenograf/cli/notes.py +270 -0
- stenograf/cli/profiles.py +169 -0
- stenograf/cli/run.py +391 -0
- stenograf/cli/settings_cmd.py +170 -0
- stenograf/cli/start.py +641 -0
- stenograf/cli/transcribe.py +432 -0
- stenograf/config.py +130 -0
- stenograf/diarization/__init__.py +29 -0
- stenograf/diarization/base.py +57 -0
- stenograf/diarization/sherpa.py +172 -0
- stenograf/diarization/speakrs.py +148 -0
- stenograf/doctor.py +370 -0
- stenograf/flow.py +525 -0
- stenograf/glossary.py +273 -0
- stenograf/gui/__init__.py +41 -0
- stenograf/gui/app.py +517 -0
- stenograf/gui/meeting.py +316 -0
- stenograf/gui/qml/Btn.qml +37 -0
- stenograf/gui/qml/Combo.qml +93 -0
- stenograf/gui/qml/Doctor.qml +79 -0
- stenograf/gui/qml/Field.qml +23 -0
- stenograf/gui/qml/Home.qml +92 -0
- stenograf/gui/qml/Main.qml +79 -0
- stenograf/gui/qml/Meeting.qml +301 -0
- stenograf/gui/qml/Notes.qml +78 -0
- stenograf/gui/qml/Panel.qml +77 -0
- stenograf/gui/qml/Settings.qml +58 -0
- stenograf/gui/qml/Setup.qml +166 -0
- stenograf/gui/qml/Theme.qml +34 -0
- stenograf/gui/qml/Toggle.qml +79 -0
- stenograf/gui/qml/Transcribe.qml +94 -0
- stenograf/gui/qml/qmldir +3 -0
- stenograf/gui/screens.py +309 -0
- stenograf/gui/tray.py +386 -0
- stenograf/lid.py +69 -0
- stenograf/live.py +673 -0
- stenograf/loaders.py +337 -0
- stenograf/models.py +156 -0
- stenograf/notes/__init__.py +45 -0
- stenograf/notes/backend.py +166 -0
- stenograf/notes/command.py +111 -0
- stenograf/notes/export.py +111 -0
- stenograf/notes/generate.py +173 -0
- stenograf/notes/mlx.py +191 -0
- stenograf/notes/model.py +150 -0
- stenograf/notes/ollama.py +135 -0
- stenograf/notes/prompt.py +197 -0
- stenograf/output.py +219 -0
- stenograf/pipeline.py +417 -0
- stenograf/profiles.py +285 -0
- stenograf/recording.py +199 -0
- stenograf/session.py +1355 -0
- stenograf/settings.py +480 -0
- stenograf/shortcut.py +401 -0
- stenograf/transcript.py +313 -0
- stenograf/ui/__init__.py +40 -0
- stenograf/ui/_fps.py +28 -0
- stenograf/ui/app.py +66 -0
- stenograf/ui/doctor.py +119 -0
- stenograf/ui/flow.py +71 -0
- stenograf/ui/home.py +122 -0
- stenograf/ui/meeting.py +475 -0
- stenograf/ui/notes.py +192 -0
- stenograf/ui/settings.py +114 -0
- stenograf/ui/setup.py +173 -0
- stenograf/ui/transcribe.py +204 -0
- stenograf/ui/widgets.py +97 -0
- stenograf/vad.py +267 -0
- stenograf/view.py +209 -0
- stenograf-0.4.0.dist-info/METADATA +445 -0
- stenograf-0.4.0.dist-info/RECORD +104 -0
- stenograf-0.4.0.dist-info/WHEEL +4 -0
- stenograf-0.4.0.dist-info/entry_points.txt +3 -0
- stenograf-0.4.0.dist-info/licenses/LICENSE +21 -0
stenograf/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""stenograf — accuracy-first local meeting transcription. Audio never touches disk."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
ASSETS = Path(__file__).parent / "assets"
|
|
7
|
+
"""Brand assets shipped in the wheel: ``icon.png`` (the window, Dock and
|
|
8
|
+
application-menu icon) and the frozen ``Stenograf.app`` bundle template
|
|
9
|
+
(:mod:`stenograf.shortcut`). Generated by ``native/appbundle/build.sh``."""
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
__version__ = version("stenograf")
|
|
13
|
+
except PackageNotFoundError: # running from a source tree without installation
|
|
14
|
+
__version__ = "0.0.0.dev0"
|
stenograf/__main__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""``python -m stenograf`` — same entry as the ``steno`` console script.
|
|
2
|
+
|
|
3
|
+
The desktop launchers (:mod:`stenograf.shortcut`) run ``"<absolute python>"
|
|
4
|
+
-m stenograf`` instead of ``steno``: a double-clicked launcher gets a fresh
|
|
5
|
+
login-shell environment whose PATH may not include uv's shim directory.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from stenograf.cli import main
|
|
9
|
+
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
main()
|
stenograf/aec.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Acoustic echo cancellation: subtract the speakers from the microphone.
|
|
2
|
+
|
|
3
|
+
The default way to sit in an online meeting is speakers + built-in mic. The
|
|
4
|
+
remote participants come out of the speakers, back into the mic, and land on the
|
|
5
|
+
mic channel — where the pipeline attributes them to *you*. Measured on this
|
|
6
|
+
machine, that echo sits ~24 dB above the mic's noise floor: loud enough to
|
|
7
|
+
transcribe, so it produces duplicate lines under a ``Local-N`` label.
|
|
8
|
+
|
|
9
|
+
We already capture the far end: the system channel *is* the audio the speakers
|
|
10
|
+
played. So this is a textbook echo-cancellation setup — near end (mic), far end
|
|
11
|
+
(system reference) — and we hand both to WebRTC's AEC3 via livekit's
|
|
12
|
+
``AudioProcessingModule``. The mic channel is replaced by its cleaned version;
|
|
13
|
+
the system channel passes through untouched (nothing echoes into the tap).
|
|
14
|
+
|
|
15
|
+
Why AEC3 and not Apple's Voice Processing IO, which is one flag away in the
|
|
16
|
+
capture helper: VPIO ducks other applications' audio, which would attenuate the
|
|
17
|
+
remote speech we are trying to transcribe (measured: −36 dB on the system
|
|
18
|
+
channel), and its AVAudioEngine binding delivered no mic frames at all on macOS
|
|
19
|
+
26. See native/README.md. Chrome ships both and defaults to this one.
|
|
20
|
+
|
|
21
|
+
Two properties of AEC3 shape the design:
|
|
22
|
+
|
|
23
|
+
- It consumes **exactly 10 ms frames** (160 samples at 16 kHz), far end first,
|
|
24
|
+
then near end, for the same instant. Hence the tick pump below.
|
|
25
|
+
- Its internal delay estimator does the real work; ``set_stream_delay_ms`` is a
|
|
26
|
+
hint. Feeding a deliberately wrong 500 ms hint measured the same 26 dB ERLE as
|
|
27
|
+
the correct 25 ms, so the constant below is a nicety, not a load-bearing value.
|
|
28
|
+
|
|
29
|
+
Measured across the AEC scenario matrix (quiet/loud, batch/live,
|
|
30
|
+
built-in/Bluetooth, double-talk), a canceller with a live reference leaks
|
|
31
|
+
nothing the ASR decodes. What residual echo survives comes from *losing* the
|
|
32
|
+
reference — a stalled or mis-clocked tap. Loss arrives in two shapes, and
|
|
33
|
+
``far_end_missing_ticks`` counts both: reference frames that stop arriving,
|
|
34
|
+
and a dead tap that keeps delivering frames of bit-exact-zero PCM (the energy
|
|
35
|
+
check in ``_check_dead_tap``). A cross-channel text backstop at merge time
|
|
36
|
+
arms itself on that signal (``session.MeetingRecorder``), and the CLI warns
|
|
37
|
+
when it had to act.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
from typing import TYPE_CHECKING
|
|
43
|
+
|
|
44
|
+
import numpy as np
|
|
45
|
+
|
|
46
|
+
from stenograf.capture.base import (
|
|
47
|
+
ORDER_TOLERANCE_SAMPLES,
|
|
48
|
+
SAMPLE_RATE,
|
|
49
|
+
AudioFrame,
|
|
50
|
+
CaptureProvider,
|
|
51
|
+
Channel,
|
|
52
|
+
GapPaddedBuffer,
|
|
53
|
+
)
|
|
54
|
+
from stenograf.recording import WavTee
|
|
55
|
+
|
|
56
|
+
if TYPE_CHECKING:
|
|
57
|
+
from collections.abc import Iterator
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
|
|
60
|
+
TICK_SAMPLES = SAMPLE_RATE // 100
|
|
61
|
+
"""AEC3 processes exactly 10 ms at a time; neither channel may be fed anything else."""
|
|
62
|
+
|
|
63
|
+
DEFAULT_DELAY_MS = 25
|
|
64
|
+
"""Measured speaker → air → mic → tap round trip on a MacBook Pro (24.6 ms)."""
|
|
65
|
+
|
|
66
|
+
_FAR_HISTORY_S = 0.5
|
|
67
|
+
"""Reference kept behind the current tick, so a late mic frame can still pair up."""
|
|
68
|
+
|
|
69
|
+
_MAX_HOLD_S = 0.5
|
|
70
|
+
"""Mic backlog tolerated while waiting for the reference. Past this we cancel
|
|
71
|
+
against silence rather than stall the live captions: a stalled tap (device
|
|
72
|
+
change, sleep/wake) must degrade to "no cancellation", never to "no captions"."""
|
|
73
|
+
|
|
74
|
+
_DEAD_TAP_S = 30.0
|
|
75
|
+
"""A reference that is bit-exact zero this long, while the mic is live, is a
|
|
76
|
+
dead tap, not a quiet meeting: a live tap carries the meeting app's dither and
|
|
77
|
+
comfort noise, and the mic always carries its own noise floor. Zeros are what
|
|
78
|
+
the known long-session tap failure delivers — frames keep arriving, so absent-
|
|
79
|
+
frame counting alone never notices."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _Track(GapPaddedBuffer):
|
|
83
|
+
"""A contiguous run of one channel's samples, and the timestamp of sample 0.
|
|
84
|
+
|
|
85
|
+
Providers deliver frames monotonically per channel, but a device that drops
|
|
86
|
+
buffers leaves a hole. The shared buffer pads it with silence, keeping the
|
|
87
|
+
sample index and the timeline in agreement — the only reason the far-end
|
|
88
|
+
reference can be looked up by timestamp at all. Jitter-sized gaps are
|
|
89
|
+
absorbed (``pad_gaps_over``) rather than zero-stuffed mid-speech.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(self, channel: Channel) -> None:
|
|
93
|
+
super().__init__(label=channel.value, pad_gaps_over=ORDER_TOLERANCE_SAMPLES)
|
|
94
|
+
self._buf = np.zeros(0, dtype=np.int16)
|
|
95
|
+
|
|
96
|
+
def _place(self, samples: np.ndarray) -> None:
|
|
97
|
+
if self._buf.size:
|
|
98
|
+
self._buf = np.concatenate([self._buf, samples])
|
|
99
|
+
else:
|
|
100
|
+
self._buf = samples.astype(np.int16, copy=True)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def available(self) -> int:
|
|
104
|
+
return self._buf.size
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def start_ts(self) -> float | None:
|
|
108
|
+
"""Timestamp of sample 0 — derived, so ``take`` advances it implicitly."""
|
|
109
|
+
if self._end is None:
|
|
110
|
+
return None
|
|
111
|
+
return (self._end - self._buf.size) / SAMPLE_RATE
|
|
112
|
+
|
|
113
|
+
def take(self, count: int) -> np.ndarray:
|
|
114
|
+
"""Remove and return the first ``count`` samples, advancing the timeline."""
|
|
115
|
+
assert self._end is not None
|
|
116
|
+
head, self._buf = self._buf[:count], self._buf[count:]
|
|
117
|
+
return head
|
|
118
|
+
|
|
119
|
+
def window(self, ts: float, count: int) -> np.ndarray | None:
|
|
120
|
+
"""``count`` samples starting at ``ts``, or None if they have not arrived.
|
|
121
|
+
|
|
122
|
+
A window starting before this track does is padded with silence: the mic
|
|
123
|
+
opens after the tap, so the reverse case (mic first) only happens when
|
|
124
|
+
the tap is late, and silence is the honest reference for audio nobody
|
|
125
|
+
has heard yet.
|
|
126
|
+
"""
|
|
127
|
+
if self.start_ts is None:
|
|
128
|
+
return None
|
|
129
|
+
index = int(round((ts - self.start_ts) * SAMPLE_RATE))
|
|
130
|
+
if index >= 0:
|
|
131
|
+
if index + count > self._buf.size:
|
|
132
|
+
return None
|
|
133
|
+
return self._buf[index : index + count]
|
|
134
|
+
pad = min(-index, count)
|
|
135
|
+
if pad == count:
|
|
136
|
+
return np.zeros(count, dtype=np.int16)
|
|
137
|
+
tail = self._buf[: count - pad]
|
|
138
|
+
if tail.size < count - pad:
|
|
139
|
+
return None
|
|
140
|
+
return np.concatenate([np.zeros(pad, dtype=np.int16), tail])
|
|
141
|
+
|
|
142
|
+
def trim_before(self, ts: float) -> None:
|
|
143
|
+
start = self.start_ts
|
|
144
|
+
if start is None or ts <= start:
|
|
145
|
+
return
|
|
146
|
+
drop = min(int((ts - start) * SAMPLE_RATE), self._buf.size)
|
|
147
|
+
if drop > 0:
|
|
148
|
+
self._buf = self._buf[drop:] # start_ts is derived; it advances with the slice
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class EchoCanceller:
|
|
152
|
+
"""Replaces mic frames with echo-cancelled ones, using system as reference.
|
|
153
|
+
|
|
154
|
+
Feed it every frame in arrival order; it returns the frames to forward. The
|
|
155
|
+
system channel is returned immediately and unchanged, so remote transcription
|
|
156
|
+
is never delayed or altered. Mic frames are buffered until the reference
|
|
157
|
+
covers the same instant, then emitted 10 ms at a time (re-aggregated into one
|
|
158
|
+
frame per call, preserving the input's timestamps).
|
|
159
|
+
|
|
160
|
+
With no system channel there is nothing to cancel against, and every frame
|
|
161
|
+
passes straight through.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
def __init__(
|
|
165
|
+
self,
|
|
166
|
+
channels: set[Channel],
|
|
167
|
+
*,
|
|
168
|
+
delay_ms: int = DEFAULT_DELAY_MS,
|
|
169
|
+
noise_suppression: bool = False,
|
|
170
|
+
cancel: bool = True,
|
|
171
|
+
) -> None:
|
|
172
|
+
self.enabled = cancel and Channel.MIC in channels and Channel.SYSTEM in channels
|
|
173
|
+
self.far_end_missing_ticks = 0
|
|
174
|
+
self._zero_run = 0 # consecutive all-zero reference ticks under a live mic
|
|
175
|
+
self._delay_ms = delay_ms
|
|
176
|
+
self._near = _Track(Channel.MIC)
|
|
177
|
+
self._far = _Track(Channel.SYSTEM)
|
|
178
|
+
self._apm = None
|
|
179
|
+
if self.enabled:
|
|
180
|
+
# Imported here: the native lib is ~9 MB and no other path needs it.
|
|
181
|
+
from livekit import rtc
|
|
182
|
+
|
|
183
|
+
self._rtc = rtc
|
|
184
|
+
self._apm = rtc.AudioProcessingModule(
|
|
185
|
+
echo_cancellation=True,
|
|
186
|
+
# AGC pumps the gain and fights the ASR front end; NS is off by
|
|
187
|
+
# default because this is an accuracy-first transcriber and the
|
|
188
|
+
# suppressor colours speech. Neither is needed to cancel echo.
|
|
189
|
+
auto_gain_control=False,
|
|
190
|
+
noise_suppression=noise_suppression,
|
|
191
|
+
high_pass_filter=True,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
def process(self, frame: AudioFrame) -> list[AudioFrame]:
|
|
195
|
+
if not self.enabled:
|
|
196
|
+
return [frame]
|
|
197
|
+
if frame.channel is Channel.SYSTEM:
|
|
198
|
+
self._far.add(frame.timestamp, frame.samples)
|
|
199
|
+
return [frame]
|
|
200
|
+
self._near.add(frame.timestamp, frame.samples)
|
|
201
|
+
return self._drain()
|
|
202
|
+
|
|
203
|
+
def drain(self) -> list[AudioFrame]:
|
|
204
|
+
"""Flush the tail at end of stream, padding the last partial tick."""
|
|
205
|
+
if not self.enabled or self._near.available == 0:
|
|
206
|
+
return []
|
|
207
|
+
return self._drain(flush=True)
|
|
208
|
+
|
|
209
|
+
def _drain(self, *, flush: bool = False) -> list[AudioFrame]:
|
|
210
|
+
hold = int(_MAX_HOLD_S * SAMPLE_RATE)
|
|
211
|
+
ticks: list[np.ndarray] = []
|
|
212
|
+
first_ts: float | None = None
|
|
213
|
+
|
|
214
|
+
while self._near.available > 0:
|
|
215
|
+
partial = self._near.available < TICK_SAMPLES
|
|
216
|
+
if partial and not flush:
|
|
217
|
+
break
|
|
218
|
+
ts = self._near.start_ts
|
|
219
|
+
assert ts is not None
|
|
220
|
+
|
|
221
|
+
far = self._far.window(ts, TICK_SAMPLES)
|
|
222
|
+
live = far is not None
|
|
223
|
+
if far is None:
|
|
224
|
+
# The reference has not caught up. Wait — unless the mic backlog
|
|
225
|
+
# says the tap has stopped, in which case forward uncancelled
|
|
226
|
+
# audio rather than freezing the captions.
|
|
227
|
+
if not flush and self._near.available < hold:
|
|
228
|
+
break
|
|
229
|
+
far = np.zeros(TICK_SAMPLES, dtype=np.int16)
|
|
230
|
+
if not flush:
|
|
231
|
+
# Only mid-capture counts as reference loss. At end-of-stream
|
|
232
|
+
# flush the far end has legitimately ended a hair before the
|
|
233
|
+
# mic tail; counting those ticks made every clean meeting
|
|
234
|
+
# report a phantom "ran without its reference for 0.0s" and
|
|
235
|
+
# spuriously armed the echo-text backstop.
|
|
236
|
+
self.far_end_missing_ticks += 1
|
|
237
|
+
|
|
238
|
+
count = min(TICK_SAMPLES, self._near.available)
|
|
239
|
+
near = self._near.take(count)
|
|
240
|
+
if live and not flush:
|
|
241
|
+
self._check_dead_tap(far, near)
|
|
242
|
+
if count < TICK_SAMPLES:
|
|
243
|
+
near = np.concatenate([near, np.zeros(TICK_SAMPLES - count, dtype=np.int16)])
|
|
244
|
+
|
|
245
|
+
cleaned = self._tick(far, near)[:count]
|
|
246
|
+
if first_ts is None:
|
|
247
|
+
first_ts = ts
|
|
248
|
+
ticks.append(cleaned)
|
|
249
|
+
self._far.trim_before(ts - _FAR_HISTORY_S)
|
|
250
|
+
|
|
251
|
+
if first_ts is None:
|
|
252
|
+
return []
|
|
253
|
+
return [AudioFrame(channel=Channel.MIC, timestamp=first_ts, samples=np.concatenate(ticks))]
|
|
254
|
+
|
|
255
|
+
def _check_dead_tap(self, far: np.ndarray, near: np.ndarray) -> None:
|
|
256
|
+
"""Fold a tap that delivers only zeros into the reference-loss count.
|
|
257
|
+
|
|
258
|
+
``far_end_missing_ticks`` sees *absent* reference frames; the known
|
|
259
|
+
long-session failure keeps delivering frames of bit-exact-zero PCM, so
|
|
260
|
+
without this check the counter stays 0, the text backstop never arms,
|
|
261
|
+
and AEC3 adapts against silence while echo passes straight through to
|
|
262
|
+
the ASR. A zero reference tick extends the run only
|
|
263
|
+
while the mic tick is non-zero (an all-zero mic is no evidence either
|
|
264
|
+
way — it neither extends nor resets); any far-end energy resets it.
|
|
265
|
+
A run that reaches ``_DEAD_TAP_S`` lands in ``far_end_missing_ticks``
|
|
266
|
+
retroactively — the reference was already dead for the whole run —
|
|
267
|
+
and tick by tick from there.
|
|
268
|
+
"""
|
|
269
|
+
if far.any():
|
|
270
|
+
self._zero_run = 0
|
|
271
|
+
return
|
|
272
|
+
if not near.any():
|
|
273
|
+
return
|
|
274
|
+
self._zero_run += 1
|
|
275
|
+
threshold = int(_DEAD_TAP_S * 100)
|
|
276
|
+
if self._zero_run == threshold:
|
|
277
|
+
self.far_end_missing_ticks += threshold
|
|
278
|
+
elif self._zero_run > threshold:
|
|
279
|
+
self.far_end_missing_ticks += 1
|
|
280
|
+
|
|
281
|
+
def _tick(self, far: np.ndarray, near: np.ndarray) -> np.ndarray:
|
|
282
|
+
"""One 10 ms step: reference first, then the mic, per AEC3's contract."""
|
|
283
|
+
assert self._apm is not None
|
|
284
|
+
reverse = self._rtc.AudioFrame(far.tobytes(), SAMPLE_RATE, 1, TICK_SAMPLES)
|
|
285
|
+
self._apm.process_reverse_stream(reverse)
|
|
286
|
+
self._apm.set_stream_delay_ms(self._delay_ms)
|
|
287
|
+
capture = self._rtc.AudioFrame(near.tobytes(), SAMPLE_RATE, 1, TICK_SAMPLES)
|
|
288
|
+
self._apm.process_stream(capture)
|
|
289
|
+
return np.frombuffer(bytes(capture.data), dtype=np.int16)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
class AecDump:
|
|
293
|
+
"""The mic/lpb/enh WAV triple that ``eval/aec_score.py`` scores.
|
|
294
|
+
|
|
295
|
+
AECMOS naming: ``mic.wav`` is the near end as the device heard it,
|
|
296
|
+
``lpb.wav`` (loopback) is the far-end reference the canceller saw, and
|
|
297
|
+
``enh.wav`` is the mic as the ASR receives it. All three are mono 16 kHz
|
|
298
|
+
and share the capture clock's t=0 (``WavTee`` pads each file's head up to
|
|
299
|
+
its first frame's timestamp), so they are sample-aligned for scoring.
|
|
300
|
+
|
|
301
|
+
Opt-in via ``--aec-dump``: like ``--record-audio``, this writes meeting
|
|
302
|
+
audio to disk.
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
def __init__(self, directory: Path) -> None:
|
|
306
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
307
|
+
self._mic = WavTee(directory / "mic.wav", {Channel.MIC})
|
|
308
|
+
self._lpb = WavTee(directory / "lpb.wav", {Channel.SYSTEM})
|
|
309
|
+
self._enh = WavTee(directory / "enh.wav", {Channel.MIC})
|
|
310
|
+
|
|
311
|
+
def add_input(self, frame: AudioFrame) -> None:
|
|
312
|
+
(self._mic if frame.channel is Channel.MIC else self._lpb).add(frame)
|
|
313
|
+
|
|
314
|
+
def add_output(self, frame: AudioFrame) -> None:
|
|
315
|
+
if frame.channel is Channel.MIC:
|
|
316
|
+
self._enh.add(frame)
|
|
317
|
+
|
|
318
|
+
def close(self) -> None:
|
|
319
|
+
self._mic.close()
|
|
320
|
+
self._lpb.close()
|
|
321
|
+
self._enh.close()
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class EchoCancellingProvider(CaptureProvider):
|
|
325
|
+
"""Wraps a provider, cancelling speaker bleed out of its mic channel.
|
|
326
|
+
|
|
327
|
+
``cancel=False`` keeps the wrapper as a pure pass-through — used with
|
|
328
|
+
``dump_dir`` to record the uncancelled baseline the eval rig compares
|
|
329
|
+
against (``--no-aec --aec-dump``).
|
|
330
|
+
"""
|
|
331
|
+
|
|
332
|
+
def __init__(
|
|
333
|
+
self,
|
|
334
|
+
inner: CaptureProvider,
|
|
335
|
+
*,
|
|
336
|
+
delay_ms: int = DEFAULT_DELAY_MS,
|
|
337
|
+
noise_suppression: bool = False,
|
|
338
|
+
cancel: bool = True,
|
|
339
|
+
dump_dir: Path | None = None,
|
|
340
|
+
) -> None:
|
|
341
|
+
self._inner = inner
|
|
342
|
+
self._delay_ms = delay_ms
|
|
343
|
+
self._noise_suppression = noise_suppression
|
|
344
|
+
self._cancel = cancel
|
|
345
|
+
self._dump_dir = dump_dir
|
|
346
|
+
self._dump: AecDump | None = None
|
|
347
|
+
self._canceller: EchoCanceller | None = None
|
|
348
|
+
self._iterating = False
|
|
349
|
+
|
|
350
|
+
@property
|
|
351
|
+
def canceller(self) -> EchoCanceller | None:
|
|
352
|
+
return self._canceller
|
|
353
|
+
|
|
354
|
+
def start(self, channels: set[Channel]) -> None:
|
|
355
|
+
self._canceller = EchoCanceller(
|
|
356
|
+
channels,
|
|
357
|
+
delay_ms=self._delay_ms,
|
|
358
|
+
noise_suppression=self._noise_suppression,
|
|
359
|
+
cancel=self._cancel,
|
|
360
|
+
)
|
|
361
|
+
if self._dump_dir is not None:
|
|
362
|
+
self._dump = AecDump(self._dump_dir)
|
|
363
|
+
self._inner.start(channels)
|
|
364
|
+
|
|
365
|
+
def frames(self) -> Iterator[AudioFrame]:
|
|
366
|
+
assert self._canceller is not None, "frames() called before start()"
|
|
367
|
+
# This iterator owns the dump while it runs: a stop() mid-meeting must
|
|
368
|
+
# not close the WAVs under it — the canceller tail drained after the
|
|
369
|
+
# inner stream ends still lands in enh.wav.
|
|
370
|
+
self._iterating = True
|
|
371
|
+
try:
|
|
372
|
+
for frame in self._inner.frames():
|
|
373
|
+
if self._dump is not None:
|
|
374
|
+
self._dump.add_input(frame)
|
|
375
|
+
for produced in self._canceller.process(frame):
|
|
376
|
+
if self._dump is not None:
|
|
377
|
+
self._dump.add_output(produced)
|
|
378
|
+
yield produced
|
|
379
|
+
for produced in self._canceller.drain():
|
|
380
|
+
if self._dump is not None:
|
|
381
|
+
self._dump.add_output(produced)
|
|
382
|
+
yield produced
|
|
383
|
+
finally:
|
|
384
|
+
self._iterating = False
|
|
385
|
+
self._close_dump()
|
|
386
|
+
|
|
387
|
+
def stop(self) -> None:
|
|
388
|
+
self._inner.stop()
|
|
389
|
+
# start() → stop() without ever iterating frames(): nobody else will
|
|
390
|
+
# close the dump's three WAV handles.
|
|
391
|
+
if not self._iterating:
|
|
392
|
+
self._close_dump()
|
|
393
|
+
|
|
394
|
+
def _close_dump(self) -> None:
|
|
395
|
+
dump, self._dump = self._dump, None
|
|
396
|
+
if dump is not None:
|
|
397
|
+
dump.close()
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from stenograf.asr.base import ASRBackend, Segment, Word
|
|
2
|
+
from stenograf.asr.registry import (
|
|
3
|
+
BackendSpec,
|
|
4
|
+
available_backends,
|
|
5
|
+
backend_model_id,
|
|
6
|
+
create_backend,
|
|
7
|
+
default_backend_name,
|
|
8
|
+
get_spec,
|
|
9
|
+
register_backend,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"ASRBackend",
|
|
14
|
+
"BackendSpec",
|
|
15
|
+
"Segment",
|
|
16
|
+
"Word",
|
|
17
|
+
"available_backends",
|
|
18
|
+
"backend_model_id",
|
|
19
|
+
"create_backend",
|
|
20
|
+
"default_backend_name",
|
|
21
|
+
"get_spec",
|
|
22
|
+
"register_backend",
|
|
23
|
+
]
|
stenograf/asr/base.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""ASR backend interface.
|
|
2
|
+
|
|
3
|
+
Backends wrap one model + runtime combination. Implementations:
|
|
4
|
+
|
|
5
|
+
- ``parakeet`` — Parakeet-TDT-0.6B-v3 via parakeet-mlx (default for both
|
|
6
|
+
finalize and live pass on macOS; Canary-1B-v2 was dropped — no Apple Silicon
|
|
7
|
+
runtime with word timestamps)
|
|
8
|
+
- ``parakeet-onnx`` — the same model, fp32 ONNX via onnx-asr on CPU (the
|
|
9
|
+
cross-platform default off macOS; Phase 5)
|
|
10
|
+
- planned: ``voxtral_mlx`` — Voxtral Small 24B via mlx-voxtral (opt-in max
|
|
11
|
+
accuracy)
|
|
12
|
+
|
|
13
|
+
Word-level timestamps are mandatory: speaker assignment intersects them with
|
|
14
|
+
diarization turns.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from abc import ABC, abstractmethod
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
from stenograf.config import Language
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class Word:
|
|
29
|
+
text: str
|
|
30
|
+
start: float
|
|
31
|
+
end: float
|
|
32
|
+
confidence: float | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class Segment:
|
|
37
|
+
text: str
|
|
38
|
+
start: float
|
|
39
|
+
end: float
|
|
40
|
+
words: tuple[Word, ...] = field(default=())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ASRBackend(ABC):
|
|
44
|
+
"""Transcribes mono int16/float32 PCM at 16 kHz into timestamped segments."""
|
|
45
|
+
|
|
46
|
+
name: str
|
|
47
|
+
|
|
48
|
+
model_id: str | None = None
|
|
49
|
+
"""Model identifier for display and provenance (HF repo / onnx-asr id);
|
|
50
|
+
``None`` when the backend has no meaningful id."""
|
|
51
|
+
|
|
52
|
+
provider: str | None = None
|
|
53
|
+
"""Requested execution provider (``cpu``/``dml``/``cuda``/``auto``) —
|
|
54
|
+
the ``[asr] provider`` / ``STENOGRAF_ASR_PROVIDER`` value, set by the
|
|
55
|
+
loader before :meth:`load`. ``None`` (the default) declares that this
|
|
56
|
+
backend manages its own runtime (MLX) and providers do not apply; an
|
|
57
|
+
ORT-backed backend initializes it to ``"cpu"``."""
|
|
58
|
+
|
|
59
|
+
active_provider: str | None = None
|
|
60
|
+
"""The provider actually running after :meth:`load` (post-fallback)."""
|
|
61
|
+
|
|
62
|
+
provider_fallback: str | None = None
|
|
63
|
+
"""Why an accelerated provider was abandoned for CPU, or ``None``."""
|
|
64
|
+
|
|
65
|
+
@abstractmethod
|
|
66
|
+
def load(self) -> None:
|
|
67
|
+
"""Load model weights (downloads to the local cache on first use)."""
|
|
68
|
+
|
|
69
|
+
@abstractmethod
|
|
70
|
+
def transcribe(self, samples: np.ndarray, language: Language | None) -> list[Segment]:
|
|
71
|
+
"""Transcribe a complete buffer.
|
|
72
|
+
|
|
73
|
+
``language`` is the resolved meeting language, or ``None`` when it is
|
|
74
|
+
not (yet) known: the default backend (Parakeet) is multilingual and
|
|
75
|
+
ignores it, and Phase-1 language detection runs over the finalized text
|
|
76
|
+
(see ``stenograf.lid``). A language-*requiring* backend must handle
|
|
77
|
+
``None`` itself (detect once, then lock) rather than assume a value."""
|
|
78
|
+
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def unload(self) -> None:
|
|
81
|
+
"""Release model memory."""
|