stenograf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stenograf/__init__.py +8 -0
- stenograf/aec.py +355 -0
- stenograf/archive.py +330 -0
- stenograf/asr/__init__.py +23 -0
- stenograf/asr/base.py +62 -0
- stenograf/asr/parakeet.py +94 -0
- stenograf/asr/registry.py +100 -0
- stenograf/audio.py +69 -0
- stenograf/capture/__init__.py +3 -0
- stenograf/capture/base.py +70 -0
- stenograf/capture/file.py +78 -0
- stenograf/capture/macos.py +164 -0
- stenograf/cli.py +1386 -0
- stenograf/config.py +130 -0
- stenograf/control.py +237 -0
- stenograf/diarization/__init__.py +3 -0
- stenograf/diarization/base.py +57 -0
- stenograf/diarization/sherpa.py +142 -0
- stenograf/doctor.py +151 -0
- stenograf/glossary.py +207 -0
- stenograf/lid.py +69 -0
- stenograf/live.py +324 -0
- stenograf/models.py +119 -0
- stenograf/pipeline.py +259 -0
- stenograf/profiles.py +287 -0
- stenograf/recording.py +210 -0
- stenograf/session.py +1152 -0
- stenograf/transcript.py +288 -0
- stenograf/tui.py +449 -0
- stenograf/vad.py +109 -0
- stenograf/view.py +185 -0
- stenograf-0.1.0.dist-info/METADATA +192 -0
- stenograf-0.1.0.dist-info/RECORD +36 -0
- stenograf-0.1.0.dist-info/WHEEL +4 -0
- stenograf-0.1.0.dist-info/entry_points.txt +3 -0
- stenograf-0.1.0.dist-info/licenses/LICENSE +21 -0
stenograf/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""stenograf — accuracy-first local meeting transcription. Audio never touches disk."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("stenograf")
|
|
7
|
+
except PackageNotFoundError: # running from a source tree without installation
|
|
8
|
+
__version__ = "0.0.0.dev0"
|
stenograf/aec.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Acoustic echo cancellation: subtract the speakers from the microphone.
|
|
2
|
+
|
|
3
|
+
The default way to sit in an online meeting is speakers + built-in mic. The
|
|
4
|
+
remote participants come out of the speakers, back into the mic, and land on the
|
|
5
|
+
mic channel — where the pipeline attributes them to *you*. Measured on this
|
|
6
|
+
machine, that echo sits ~24 dB above the mic's noise floor: loud enough to
|
|
7
|
+
transcribe, so it produces duplicate lines under a ``Local-N`` label.
|
|
8
|
+
|
|
9
|
+
We already capture the far end: the system channel *is* the audio the speakers
|
|
10
|
+
played. So this is a textbook echo-cancellation setup — near end (mic), far end
|
|
11
|
+
(system reference) — and we hand both to WebRTC's AEC3 via livekit's
|
|
12
|
+
``AudioProcessingModule``. The mic channel is replaced by its cleaned version;
|
|
13
|
+
the system channel passes through untouched (nothing echoes into the tap).
|
|
14
|
+
|
|
15
|
+
Why AEC3 and not Apple's Voice Processing IO, which is one flag away in the
|
|
16
|
+
capture helper: VPIO ducks other applications' audio, which would attenuate the
|
|
17
|
+
remote speech we are trying to transcribe (measured: −36 dB on the system
|
|
18
|
+
channel), and its AVAudioEngine binding delivered no mic frames at all on macOS
|
|
19
|
+
26. See native/README.md. Chrome ships both and defaults to this one.
|
|
20
|
+
|
|
21
|
+
Two properties of AEC3 shape the design:
|
|
22
|
+
|
|
23
|
+
- It consumes **exactly 10 ms frames** (160 samples at 16 kHz), far end first,
|
|
24
|
+
then near end, for the same instant. Hence the tick pump below.
|
|
25
|
+
- Its internal delay estimator does the real work; ``set_stream_delay_ms`` is a
|
|
26
|
+
hint. Feeding a deliberately wrong 500 ms hint measured the same 26 dB ERLE as
|
|
27
|
+
the correct 25 ms, so the constant below is a nicety, not a load-bearing value.
|
|
28
|
+
|
|
29
|
+
Measured across the PLAN-AEC.md scenario matrix (quiet/loud, batch/live,
|
|
30
|
+
built-in/Bluetooth, double-talk), a canceller with a live reference leaks
|
|
31
|
+
nothing the ASR decodes. What residual echo survives comes from *losing* the
|
|
32
|
+
reference — a stalled or mis-clocked tap — which ``far_end_missing_ticks``
|
|
33
|
+
counts; a cross-channel text backstop at merge time arms itself on that signal
|
|
34
|
+
(``session.MeetingRecorder``), and the CLI warns when it had to act.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from typing import TYPE_CHECKING
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
|
|
43
|
+
from stenograf.capture.base import (
|
|
44
|
+
ORDER_TOLERANCE_SAMPLES,
|
|
45
|
+
SAMPLE_RATE,
|
|
46
|
+
AudioFrame,
|
|
47
|
+
CaptureProvider,
|
|
48
|
+
Channel,
|
|
49
|
+
)
|
|
50
|
+
from stenograf.recording import WavTee
|
|
51
|
+
|
|
52
|
+
if TYPE_CHECKING:
|
|
53
|
+
from collections.abc import Iterator
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
|
|
56
|
+
TICK_SAMPLES = SAMPLE_RATE // 100
|
|
57
|
+
"""AEC3 processes exactly 10 ms at a time; neither channel may be fed anything else."""
|
|
58
|
+
|
|
59
|
+
DEFAULT_DELAY_MS = 25
|
|
60
|
+
"""Measured speaker → air → mic → tap round trip on a MacBook Pro (24.6 ms)."""
|
|
61
|
+
|
|
62
|
+
_FAR_HISTORY_S = 0.5
|
|
63
|
+
"""Reference kept behind the current tick, so a late mic frame can still pair up."""
|
|
64
|
+
|
|
65
|
+
_MAX_HOLD_S = 0.5
|
|
66
|
+
"""Mic backlog tolerated while waiting for the reference. Past this we cancel
|
|
67
|
+
against silence rather than stall the live captions: a stalled tap (device
|
|
68
|
+
change, sleep/wake) must degrade to "no cancellation", never to "no captions"."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class _Track:
|
|
72
|
+
"""A contiguous run of one channel's samples, and the timestamp of sample 0.
|
|
73
|
+
|
|
74
|
+
Providers deliver frames monotonically per channel, but a device that drops
|
|
75
|
+
buffers leaves a hole. Padding it with silence keeps the sample index and
|
|
76
|
+
the timeline in agreement, which is the only reason the far-end reference
|
|
77
|
+
can be looked up by timestamp at all.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
def __init__(self, channel: Channel) -> None:
|
|
81
|
+
self._channel = channel
|
|
82
|
+
self._buf = np.zeros(0, dtype=np.int16)
|
|
83
|
+
self.start_ts: float | None = None
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def available(self) -> int:
|
|
87
|
+
return self._buf.size
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def end_ts(self) -> float | None:
|
|
91
|
+
if self.start_ts is None:
|
|
92
|
+
return None
|
|
93
|
+
return self.start_ts + self._buf.size / SAMPLE_RATE
|
|
94
|
+
|
|
95
|
+
def append(self, frame: AudioFrame) -> None:
|
|
96
|
+
if self.start_ts is None:
|
|
97
|
+
self.start_ts = frame.timestamp
|
|
98
|
+
self._buf = frame.samples.astype(np.int16, copy=True)
|
|
99
|
+
return
|
|
100
|
+
assert self.end_ts is not None
|
|
101
|
+
gap = int(round((frame.timestamp - self.end_ts) * SAMPLE_RATE))
|
|
102
|
+
if gap < -ORDER_TOLERANCE_SAMPLES:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"{self._channel.value} frame went backwards by {-gap} samples; "
|
|
105
|
+
"the capture stream desynced"
|
|
106
|
+
)
|
|
107
|
+
pieces = [self._buf]
|
|
108
|
+
if gap > ORDER_TOLERANCE_SAMPLES:
|
|
109
|
+
pieces.append(np.zeros(gap, dtype=np.int16))
|
|
110
|
+
pieces.append(frame.samples)
|
|
111
|
+
self._buf = np.concatenate(pieces)
|
|
112
|
+
|
|
113
|
+
def take(self, count: int) -> np.ndarray:
|
|
114
|
+
"""Remove and return the first ``count`` samples, advancing the timeline."""
|
|
115
|
+
assert self.start_ts is not None
|
|
116
|
+
head, self._buf = self._buf[:count], self._buf[count:]
|
|
117
|
+
self.start_ts += head.size / SAMPLE_RATE
|
|
118
|
+
return head
|
|
119
|
+
|
|
120
|
+
def window(self, ts: float, count: int) -> np.ndarray | None:
|
|
121
|
+
"""``count`` samples starting at ``ts``, or None if they have not arrived.
|
|
122
|
+
|
|
123
|
+
A window starting before this track does is padded with silence: the mic
|
|
124
|
+
opens after the tap, so the reverse case (mic first) only happens when
|
|
125
|
+
the tap is late, and silence is the honest reference for audio nobody
|
|
126
|
+
has heard yet.
|
|
127
|
+
"""
|
|
128
|
+
if self.start_ts is None:
|
|
129
|
+
return None
|
|
130
|
+
index = int(round((ts - self.start_ts) * SAMPLE_RATE))
|
|
131
|
+
if index >= 0:
|
|
132
|
+
if index + count > self._buf.size:
|
|
133
|
+
return None
|
|
134
|
+
return self._buf[index : index + count]
|
|
135
|
+
pad = min(-index, count)
|
|
136
|
+
if pad == count:
|
|
137
|
+
return np.zeros(count, dtype=np.int16)
|
|
138
|
+
tail = self._buf[: count - pad]
|
|
139
|
+
if tail.size < count - pad:
|
|
140
|
+
return None
|
|
141
|
+
return np.concatenate([np.zeros(pad, dtype=np.int16), tail])
|
|
142
|
+
|
|
143
|
+
def trim_before(self, ts: float) -> None:
|
|
144
|
+
if self.start_ts is None or ts <= self.start_ts:
|
|
145
|
+
return
|
|
146
|
+
drop = min(int((ts - self.start_ts) * SAMPLE_RATE), self._buf.size)
|
|
147
|
+
if drop > 0:
|
|
148
|
+
self._buf = self._buf[drop:]
|
|
149
|
+
self.start_ts += drop / SAMPLE_RATE
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class EchoCanceller:
|
|
153
|
+
"""Replaces mic frames with echo-cancelled ones, using system as reference.
|
|
154
|
+
|
|
155
|
+
Feed it every frame in arrival order; it returns the frames to forward. The
|
|
156
|
+
system channel is returned immediately and unchanged, so remote transcription
|
|
157
|
+
is never delayed or altered. Mic frames are buffered until the reference
|
|
158
|
+
covers the same instant, then emitted 10 ms at a time (re-aggregated into one
|
|
159
|
+
frame per call, preserving the input's timestamps).
|
|
160
|
+
|
|
161
|
+
With no system channel there is nothing to cancel against, and every frame
|
|
162
|
+
passes straight through.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
def __init__(
|
|
166
|
+
self,
|
|
167
|
+
channels: set[Channel],
|
|
168
|
+
*,
|
|
169
|
+
delay_ms: int = DEFAULT_DELAY_MS,
|
|
170
|
+
noise_suppression: bool = False,
|
|
171
|
+
cancel: bool = True,
|
|
172
|
+
) -> None:
|
|
173
|
+
self.enabled = cancel and Channel.MIC in channels and Channel.SYSTEM in channels
|
|
174
|
+
self.far_end_missing_ticks = 0
|
|
175
|
+
self._delay_ms = delay_ms
|
|
176
|
+
self._near = _Track(Channel.MIC)
|
|
177
|
+
self._far = _Track(Channel.SYSTEM)
|
|
178
|
+
self._apm = None
|
|
179
|
+
if self.enabled:
|
|
180
|
+
# Imported here: the native lib is ~9 MB and no other path needs it.
|
|
181
|
+
from livekit import rtc
|
|
182
|
+
|
|
183
|
+
self._rtc = rtc
|
|
184
|
+
self._apm = rtc.AudioProcessingModule(
|
|
185
|
+
echo_cancellation=True,
|
|
186
|
+
# AGC pumps the gain and fights the ASR front end; NS is off by
|
|
187
|
+
# default because this is an accuracy-first transcriber and the
|
|
188
|
+
# suppressor colours speech. Neither is needed to cancel echo.
|
|
189
|
+
auto_gain_control=False,
|
|
190
|
+
noise_suppression=noise_suppression,
|
|
191
|
+
high_pass_filter=True,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
def process(self, frame: AudioFrame) -> list[AudioFrame]:
|
|
195
|
+
if not self.enabled:
|
|
196
|
+
return [frame]
|
|
197
|
+
if frame.channel is Channel.SYSTEM:
|
|
198
|
+
self._far.append(frame)
|
|
199
|
+
return [frame]
|
|
200
|
+
self._near.append(frame)
|
|
201
|
+
return self._drain()
|
|
202
|
+
|
|
203
|
+
def drain(self) -> list[AudioFrame]:
|
|
204
|
+
"""Flush the tail at end of stream, padding the last partial tick."""
|
|
205
|
+
if not self.enabled or self._near.available == 0:
|
|
206
|
+
return []
|
|
207
|
+
return self._drain(flush=True)
|
|
208
|
+
|
|
209
|
+
def _drain(self, *, flush: bool = False) -> list[AudioFrame]:
|
|
210
|
+
hold = int(_MAX_HOLD_S * SAMPLE_RATE)
|
|
211
|
+
ticks: list[np.ndarray] = []
|
|
212
|
+
first_ts: float | None = None
|
|
213
|
+
|
|
214
|
+
while self._near.available > 0:
|
|
215
|
+
partial = self._near.available < TICK_SAMPLES
|
|
216
|
+
if partial and not flush:
|
|
217
|
+
break
|
|
218
|
+
ts = self._near.start_ts
|
|
219
|
+
assert ts is not None
|
|
220
|
+
|
|
221
|
+
far = self._far.window(ts, TICK_SAMPLES)
|
|
222
|
+
if far is None:
|
|
223
|
+
# The reference has not caught up. Wait — unless the mic backlog
|
|
224
|
+
# says the tap has stopped, in which case forward uncancelled
|
|
225
|
+
# audio rather than freezing the captions.
|
|
226
|
+
if not flush and self._near.available < hold:
|
|
227
|
+
break
|
|
228
|
+
far = np.zeros(TICK_SAMPLES, dtype=np.int16)
|
|
229
|
+
if not flush:
|
|
230
|
+
# Only mid-capture counts as reference loss. At end-of-stream
|
|
231
|
+
# flush the far end has legitimately ended a hair before the
|
|
232
|
+
# mic tail; counting those ticks made every clean meeting
|
|
233
|
+
# report a phantom "ran without its reference for 0.0s" and
|
|
234
|
+
# spuriously armed the echo-text backstop.
|
|
235
|
+
self.far_end_missing_ticks += 1
|
|
236
|
+
|
|
237
|
+
count = min(TICK_SAMPLES, self._near.available)
|
|
238
|
+
near = self._near.take(count)
|
|
239
|
+
if count < TICK_SAMPLES:
|
|
240
|
+
near = np.concatenate([near, np.zeros(TICK_SAMPLES - count, dtype=np.int16)])
|
|
241
|
+
|
|
242
|
+
cleaned = self._tick(far, near)[:count]
|
|
243
|
+
if first_ts is None:
|
|
244
|
+
first_ts = ts
|
|
245
|
+
ticks.append(cleaned)
|
|
246
|
+
self._far.trim_before(ts - _FAR_HISTORY_S)
|
|
247
|
+
|
|
248
|
+
if first_ts is None:
|
|
249
|
+
return []
|
|
250
|
+
return [AudioFrame(channel=Channel.MIC, timestamp=first_ts, samples=np.concatenate(ticks))]
|
|
251
|
+
|
|
252
|
+
def _tick(self, far: np.ndarray, near: np.ndarray) -> np.ndarray:
|
|
253
|
+
"""One 10 ms step: reference first, then the mic, per AEC3's contract."""
|
|
254
|
+
assert self._apm is not None
|
|
255
|
+
reverse = self._rtc.AudioFrame(far.tobytes(), SAMPLE_RATE, 1, TICK_SAMPLES)
|
|
256
|
+
self._apm.process_reverse_stream(reverse)
|
|
257
|
+
self._apm.set_stream_delay_ms(self._delay_ms)
|
|
258
|
+
capture = self._rtc.AudioFrame(near.tobytes(), SAMPLE_RATE, 1, TICK_SAMPLES)
|
|
259
|
+
self._apm.process_stream(capture)
|
|
260
|
+
return np.frombuffer(bytes(capture.data), dtype=np.int16)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class AecDump:
|
|
264
|
+
"""The mic/lpb/enh WAV triple that ``eval/aec_score.py`` scores.
|
|
265
|
+
|
|
266
|
+
AECMOS naming: ``mic.wav`` is the near end as the device heard it,
|
|
267
|
+
``lpb.wav`` (loopback) is the far-end reference the canceller saw, and
|
|
268
|
+
``enh.wav`` is the mic as the ASR receives it. All three are mono 16 kHz
|
|
269
|
+
and share the capture clock's t=0 (``WavTee`` pads each file's head up to
|
|
270
|
+
its first frame's timestamp), so they are sample-aligned for scoring.
|
|
271
|
+
|
|
272
|
+
Opt-in via ``--aec-dump``: like ``--record-audio``, this writes meeting
|
|
273
|
+
audio to disk.
|
|
274
|
+
"""
|
|
275
|
+
|
|
276
|
+
def __init__(self, directory: Path) -> None:
|
|
277
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
278
|
+
self._mic = WavTee(directory / "mic.wav", {Channel.MIC})
|
|
279
|
+
self._lpb = WavTee(directory / "lpb.wav", {Channel.SYSTEM})
|
|
280
|
+
self._enh = WavTee(directory / "enh.wav", {Channel.MIC})
|
|
281
|
+
|
|
282
|
+
def add_input(self, frame: AudioFrame) -> None:
|
|
283
|
+
(self._mic if frame.channel is Channel.MIC else self._lpb).add(frame)
|
|
284
|
+
|
|
285
|
+
def add_output(self, frame: AudioFrame) -> None:
|
|
286
|
+
if frame.channel is Channel.MIC:
|
|
287
|
+
self._enh.add(frame)
|
|
288
|
+
|
|
289
|
+
def close(self) -> None:
|
|
290
|
+
self._mic.close()
|
|
291
|
+
self._lpb.close()
|
|
292
|
+
self._enh.close()
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class EchoCancellingProvider(CaptureProvider):
|
|
296
|
+
"""Wraps a provider, cancelling speaker bleed out of its mic channel.
|
|
297
|
+
|
|
298
|
+
``cancel=False`` keeps the wrapper as a pure pass-through — used with
|
|
299
|
+
``dump_dir`` to record the uncancelled baseline the eval rig compares
|
|
300
|
+
against (``--no-aec --aec-dump``).
|
|
301
|
+
"""
|
|
302
|
+
|
|
303
|
+
def __init__(
|
|
304
|
+
self,
|
|
305
|
+
inner: CaptureProvider,
|
|
306
|
+
*,
|
|
307
|
+
delay_ms: int = DEFAULT_DELAY_MS,
|
|
308
|
+
noise_suppression: bool = False,
|
|
309
|
+
cancel: bool = True,
|
|
310
|
+
dump_dir: Path | None = None,
|
|
311
|
+
) -> None:
|
|
312
|
+
self._inner = inner
|
|
313
|
+
self._delay_ms = delay_ms
|
|
314
|
+
self._noise_suppression = noise_suppression
|
|
315
|
+
self._cancel = cancel
|
|
316
|
+
self._dump_dir = dump_dir
|
|
317
|
+
self._dump: AecDump | None = None
|
|
318
|
+
self._canceller: EchoCanceller | None = None
|
|
319
|
+
|
|
320
|
+
@property
|
|
321
|
+
def canceller(self) -> EchoCanceller | None:
|
|
322
|
+
return self._canceller
|
|
323
|
+
|
|
324
|
+
def start(self, channels: set[Channel]) -> None:
|
|
325
|
+
self._canceller = EchoCanceller(
|
|
326
|
+
channels,
|
|
327
|
+
delay_ms=self._delay_ms,
|
|
328
|
+
noise_suppression=self._noise_suppression,
|
|
329
|
+
cancel=self._cancel,
|
|
330
|
+
)
|
|
331
|
+
if self._dump_dir is not None:
|
|
332
|
+
self._dump = AecDump(self._dump_dir)
|
|
333
|
+
self._inner.start(channels)
|
|
334
|
+
|
|
335
|
+
def frames(self) -> Iterator[AudioFrame]:
|
|
336
|
+
assert self._canceller is not None, "frames() called before start()"
|
|
337
|
+
try:
|
|
338
|
+
for frame in self._inner.frames():
|
|
339
|
+
if self._dump is not None:
|
|
340
|
+
self._dump.add_input(frame)
|
|
341
|
+
for produced in self._canceller.process(frame):
|
|
342
|
+
if self._dump is not None:
|
|
343
|
+
self._dump.add_output(produced)
|
|
344
|
+
yield produced
|
|
345
|
+
for produced in self._canceller.drain():
|
|
346
|
+
if self._dump is not None:
|
|
347
|
+
self._dump.add_output(produced)
|
|
348
|
+
yield produced
|
|
349
|
+
finally:
|
|
350
|
+
if self._dump is not None:
|
|
351
|
+
self._dump.close()
|
|
352
|
+
self._dump = None
|
|
353
|
+
|
|
354
|
+
def stop(self) -> None:
|
|
355
|
+
self._inner.stop()
|
stenograf/archive.py
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Meeting archive: a managed library of finalized transcripts.
|
|
2
|
+
|
|
3
|
+
Phase 4 turns ``steno start`` into a tool with memory. Every run writes its
|
|
4
|
+
transcript into a managed directory whose name *is* a stable meeting id
|
|
5
|
+
(``meeting-YYYYMMDD-HHMMSS`` + a collision suffix), and a maintained JSON index
|
|
6
|
+
records lightweight metadata for each — so the web UI's "meeting archive" view
|
|
7
|
+
lists meetings without re-scanning and re-parsing every transcript on disk
|
|
8
|
+
(PLAN.md §5 Stage B1).
|
|
9
|
+
|
|
10
|
+
Design mirrors :class:`~stenograf.profiles.ProfileStore`:
|
|
11
|
+
|
|
12
|
+
- The archive lives in the platform **data** dir (``data_dir()/meetings/``),
|
|
13
|
+
distinct from the re-downloadable model cache — a transcript library is
|
|
14
|
+
precious user data.
|
|
15
|
+
- The index (``meetings/index.json``) is written atomically (temp + replace), so
|
|
16
|
+
a crash mid-save never corrupts the library.
|
|
17
|
+
- The index is *maintained* (updated on every add/remove), not derived by
|
|
18
|
+
scanning — but :meth:`MeetingArchive.reconcile` self-heals it against the
|
|
19
|
+
actual directories (dropping records whose dir vanished, adopting orphan
|
|
20
|
+
meeting dirs written while the index was unavailable).
|
|
21
|
+
|
|
22
|
+
The **in-RAM-only privacy guarantee is preserved**: a record references audio
|
|
23
|
+
only when ``--record-audio`` actually wrote a WAV, which :meth:`MeetingRecord.
|
|
24
|
+
has_audio` gates (audio playback and archived re-diarize check it — PLAN.md §5
|
|
25
|
+
Stage B4). A record with no audio still supports text click-to-jump, because the
|
|
26
|
+
word timestamps live in the transcript JSON.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import json
|
|
32
|
+
import re
|
|
33
|
+
import tempfile
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from datetime import datetime
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
from stenograf.config import Language
|
|
39
|
+
from stenograf.profiles import data_dir
|
|
40
|
+
from stenograf.transcript import Transcript, UnsupportedTranscriptVersion
|
|
41
|
+
|
|
42
|
+
_INDEX_VERSION = 1
|
|
43
|
+
|
|
44
|
+
TRANSCRIPT_STEM = "transcript"
|
|
45
|
+
"""Basename (without extension) of the managed transcript files in a meeting dir:
|
|
46
|
+
``transcript.json`` / ``transcript.md`` / ``transcript.srt`` / ``transcript.vtt``."""
|
|
47
|
+
|
|
48
|
+
TRANSCRIPT_FORMATS = ("md", "json", "srt", "vtt")
|
|
49
|
+
"""Transcript file extensions the archive recognizes when summarizing a dir."""
|
|
50
|
+
|
|
51
|
+
_FORMAT_METHODS = {"md": "to_markdown", "json": "to_json", "srt": "to_srt", "vtt": "to_vtt"}
|
|
52
|
+
"""Transcript extension → the :class:`Transcript` method that renders it."""
|
|
53
|
+
|
|
54
|
+
AUDIO_NAME = "audio.wav"
|
|
55
|
+
"""Managed name of the opt-in ``--record-audio`` WAV inside a meeting dir."""
|
|
56
|
+
|
|
57
|
+
_ID_TIMESTAMP = re.compile(r"^meeting-(\d{8})-(\d{6})")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class MeetingRecord:
|
|
62
|
+
"""Lightweight index metadata for one archived meeting.
|
|
63
|
+
|
|
64
|
+
Denormalized into the index so the archive can be listed without opening each
|
|
65
|
+
transcript. ``dir`` is where the transcript files actually live — normally the
|
|
66
|
+
managed ``meetings/<id>/`` but an explicit ``--out`` can point elsewhere while
|
|
67
|
+
still registering here. Plain (unfrozen) dataclass: records are value-compared
|
|
68
|
+
for round-trip tests but deliberately unhashable (the ``speakers`` dict would
|
|
69
|
+
make a frozen hash raise — see :class:`~stenograf.profiles.SpeakerProfile`)."""
|
|
70
|
+
|
|
71
|
+
id: str
|
|
72
|
+
title: str | None
|
|
73
|
+
created_at: str
|
|
74
|
+
"""ISO-8601 creation timestamp (string, so the index is plain JSON)."""
|
|
75
|
+
duration_s: float
|
|
76
|
+
language: Language | None
|
|
77
|
+
speakers: dict[str, int | None] = field(default_factory=dict)
|
|
78
|
+
"""Per-channel resolved speaker count (``mic``/``system``, or ``audio`` for a
|
|
79
|
+
file transcribe); ``None`` for a channel whose count was left at a default."""
|
|
80
|
+
formats: tuple[str, ...] = ()
|
|
81
|
+
dir: Path = field(default=Path())
|
|
82
|
+
audio_path: Path | None = None
|
|
83
|
+
|
|
84
|
+
def has_audio(self) -> bool:
|
|
85
|
+
"""True only when a recorded WAV actually exists on disk.
|
|
86
|
+
|
|
87
|
+
The single predicate gating archived audio *playback* and archived
|
|
88
|
+
*re-diarize* (PLAN.md §5 Stage B4) — both contradict the in-memory-only
|
|
89
|
+
guarantee unless ``--record-audio`` was used, and a referenced file can
|
|
90
|
+
also have been deleted since."""
|
|
91
|
+
return self.audio_path is not None and self.audio_path.exists()
|
|
92
|
+
|
|
93
|
+
def _to_json(self) -> dict:
|
|
94
|
+
return {
|
|
95
|
+
"id": self.id,
|
|
96
|
+
"title": self.title,
|
|
97
|
+
"created_at": self.created_at,
|
|
98
|
+
"duration_s": self.duration_s,
|
|
99
|
+
"language": self.language.value if self.language else None,
|
|
100
|
+
"speakers": dict(self.speakers),
|
|
101
|
+
"formats": list(self.formats),
|
|
102
|
+
"dir": str(self.dir),
|
|
103
|
+
"audio_path": str(self.audio_path) if self.audio_path is not None else None,
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def _from_json(cls, data: dict) -> MeetingRecord:
|
|
108
|
+
audio = data.get("audio_path")
|
|
109
|
+
language = data.get("language")
|
|
110
|
+
return cls(
|
|
111
|
+
id=data["id"],
|
|
112
|
+
title=data.get("title"),
|
|
113
|
+
created_at=data.get("created_at", ""),
|
|
114
|
+
duration_s=data.get("duration_s", 0.0),
|
|
115
|
+
language=Language(language) if language is not None else None,
|
|
116
|
+
speakers=dict(data.get("speakers", {})),
|
|
117
|
+
formats=tuple(data.get("formats", ())),
|
|
118
|
+
dir=Path(data["dir"]),
|
|
119
|
+
audio_path=Path(audio) if audio is not None else None,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class MeetingArchive:
|
|
124
|
+
"""A managed, index-backed library of :class:`MeetingRecord` s.
|
|
125
|
+
|
|
126
|
+
Load with :meth:`load` (a missing index is an empty archive), mutate with
|
|
127
|
+
:meth:`add`/:meth:`remove` (each persists atomically), read a transcript back
|
|
128
|
+
through the A1 loader with :meth:`load_transcript`, and self-heal the index
|
|
129
|
+
against the directory tree with :meth:`reconcile`.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self, root: Path | None = None, records: list[MeetingRecord] | None = None
|
|
134
|
+
) -> None:
|
|
135
|
+
self.root = Path(root) if root is not None else meetings_dir()
|
|
136
|
+
# Keyed by id (== dir name) so get/remove are O(1) and ids stay unique;
|
|
137
|
+
# dict preserves insertion order for a stable listing.
|
|
138
|
+
self._records: dict[str, MeetingRecord] = {r.id: r for r in (records or [])}
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def index_path(self) -> Path:
|
|
142
|
+
return self.root / "index.json"
|
|
143
|
+
|
|
144
|
+
# ---- persistence ------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def load(cls, root: Path | None = None) -> MeetingArchive:
|
|
148
|
+
root = Path(root) if root is not None else meetings_dir()
|
|
149
|
+
index = root / "index.json"
|
|
150
|
+
if not index.exists():
|
|
151
|
+
return cls(root)
|
|
152
|
+
data = json.loads(index.read_text(encoding="utf-8"))
|
|
153
|
+
records = [MeetingRecord._from_json(r) for r in data.get("meetings", [])]
|
|
154
|
+
return cls(root, records)
|
|
155
|
+
|
|
156
|
+
def save(self) -> None:
|
|
157
|
+
"""Write the index to ``index_path`` atomically (temp file + replace)."""
|
|
158
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
payload = json.dumps(
|
|
160
|
+
{"version": _INDEX_VERSION, "meetings": [r._to_json() for r in self._records.values()]},
|
|
161
|
+
ensure_ascii=False,
|
|
162
|
+
indent=2,
|
|
163
|
+
)
|
|
164
|
+
with tempfile.NamedTemporaryFile(
|
|
165
|
+
"w", dir=self.root, suffix=".part", delete=False, encoding="utf-8"
|
|
166
|
+
) as tmp:
|
|
167
|
+
tmp.write(payload)
|
|
168
|
+
tmp_path = Path(tmp.name)
|
|
169
|
+
tmp_path.replace(self.index_path)
|
|
170
|
+
|
|
171
|
+
# ---- reads ------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
def records(self) -> list[MeetingRecord]:
|
|
174
|
+
"""Every record, in insertion order (callers sort as they like)."""
|
|
175
|
+
return list(self._records.values())
|
|
176
|
+
|
|
177
|
+
def get(self, meeting_id: str) -> MeetingRecord | None:
|
|
178
|
+
return self._records.get(meeting_id)
|
|
179
|
+
|
|
180
|
+
def meeting_dir(self, meeting_id: str) -> Path:
|
|
181
|
+
"""The managed directory for ``meeting_id`` (``root/<id>``); the default
|
|
182
|
+
``steno start`` output location for a new meeting."""
|
|
183
|
+
return self.root / meeting_id
|
|
184
|
+
|
|
185
|
+
def load_transcript(self, meeting_id: str) -> Transcript:
|
|
186
|
+
"""Read a meeting's transcript back through :meth:`Transcript.from_json`."""
|
|
187
|
+
record = self._records.get(meeting_id)
|
|
188
|
+
if record is None:
|
|
189
|
+
raise KeyError(meeting_id)
|
|
190
|
+
text = (record.dir / f"{TRANSCRIPT_STEM}.json").read_text(encoding="utf-8")
|
|
191
|
+
return Transcript.from_json(text)
|
|
192
|
+
|
|
193
|
+
# ---- writes -----------------------------------------------------------
|
|
194
|
+
|
|
195
|
+
def allocate_id(self, created_at: datetime) -> str:
|
|
196
|
+
"""Mint a stable, unique meeting id for ``created_at``.
|
|
197
|
+
|
|
198
|
+
Base form ``meeting-YYYYMMDD-HHMMSS``; on collision (same second, or a dir
|
|
199
|
+
already present) append ``-2``, ``-3``, … so the id — and therefore the
|
|
200
|
+
managed dir name — is always free."""
|
|
201
|
+
base = f"meeting-{created_at:%Y%m%d-%H%M%S}"
|
|
202
|
+
candidate = base
|
|
203
|
+
suffix = 2
|
|
204
|
+
while candidate in self._records or (self.root / candidate).exists():
|
|
205
|
+
candidate = f"{base}-{suffix}"
|
|
206
|
+
suffix += 1
|
|
207
|
+
return candidate
|
|
208
|
+
|
|
209
|
+
def add(self, record: MeetingRecord) -> None:
|
|
210
|
+
"""Insert or replace a record by id and persist the index."""
|
|
211
|
+
self._records[record.id] = record
|
|
212
|
+
self.save()
|
|
213
|
+
|
|
214
|
+
def rewrite(self, record: MeetingRecord, transcript: Transcript) -> MeetingRecord:
|
|
215
|
+
"""Rewrite a meeting's managed transcript files from ``transcript`` and
|
|
216
|
+
refresh its index metadata under the same id.
|
|
217
|
+
|
|
218
|
+
The persistence half of the reverse-control channel (PLAN.md §5 Stage B4):
|
|
219
|
+
after a rename or re-finalize produces a new transcript, re-render each of
|
|
220
|
+
the record's formats into ``<dir>/transcript.{fmt}`` (atomic temp+replace,
|
|
221
|
+
so a reader never sees a half-written file) and re-derive the metadata that
|
|
222
|
+
can have changed — title, language, per-channel speaker counts, duration —
|
|
223
|
+
while keeping the id, created_at, dir, formats, and audio reference. The
|
|
224
|
+
updated record is re-added (persisting the index) and returned."""
|
|
225
|
+
record.dir.mkdir(parents=True, exist_ok=True)
|
|
226
|
+
for fmt in record.formats:
|
|
227
|
+
method = _FORMAT_METHODS.get(fmt)
|
|
228
|
+
if method is None:
|
|
229
|
+
continue # an unknown extension in the record — nothing to render
|
|
230
|
+
path = record.dir / f"{TRANSCRIPT_STEM}.{fmt}"
|
|
231
|
+
_atomic_write_text(path, getattr(transcript, method)())
|
|
232
|
+
updated = MeetingRecord(
|
|
233
|
+
id=record.id,
|
|
234
|
+
title=transcript.profile.title,
|
|
235
|
+
created_at=record.created_at,
|
|
236
|
+
duration_s=max((e.end for e in transcript.entries), default=0.0),
|
|
237
|
+
language=transcript.language,
|
|
238
|
+
speakers=_speakers_from_transcript(transcript),
|
|
239
|
+
formats=record.formats,
|
|
240
|
+
dir=record.dir,
|
|
241
|
+
audio_path=record.audio_path,
|
|
242
|
+
)
|
|
243
|
+
self.add(updated)
|
|
244
|
+
return updated
|
|
245
|
+
|
|
246
|
+
def remove(self, meeting_id: str) -> bool:
|
|
247
|
+
"""Drop a record from the index (leaves its files alone). Persists if changed."""
|
|
248
|
+
if self._records.pop(meeting_id, None) is None:
|
|
249
|
+
return False
|
|
250
|
+
self.save()
|
|
251
|
+
return True
|
|
252
|
+
|
|
253
|
+
def reconcile(self) -> None:
|
|
254
|
+
"""Self-heal the index against the actual meeting directories.
|
|
255
|
+
|
|
256
|
+
Drops records whose ``dir`` has vanished, and adopts orphan managed dirs
|
|
257
|
+
(a ``meetings/<id>/`` holding a ``transcript.json`` that the index doesn't
|
|
258
|
+
know about — e.g. written while the index was unavailable). External
|
|
259
|
+
``--out`` dirs are never scanned for adoption; only the managed root is."""
|
|
260
|
+
for meeting_id, record in list(self._records.items()):
|
|
261
|
+
if not record.dir.exists():
|
|
262
|
+
del self._records[meeting_id]
|
|
263
|
+
if self.root.exists():
|
|
264
|
+
for child in sorted(self.root.iterdir()):
|
|
265
|
+
if not child.is_dir() or child.name in self._records:
|
|
266
|
+
continue
|
|
267
|
+
adopted = _record_from_dir(child)
|
|
268
|
+
if adopted is not None:
|
|
269
|
+
self._records[adopted.id] = adopted
|
|
270
|
+
self.save()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _record_from_dir(directory: Path) -> MeetingRecord | None:
|
|
274
|
+
"""Reconstruct a record from a managed meeting dir, or ``None`` if it holds no
|
|
275
|
+
readable transcript (a half-written or unrelated directory)."""
|
|
276
|
+
transcript_json = directory / f"{TRANSCRIPT_STEM}.json"
|
|
277
|
+
if not transcript_json.exists():
|
|
278
|
+
return None
|
|
279
|
+
try:
|
|
280
|
+
transcript = Transcript.from_json(transcript_json.read_text(encoding="utf-8"))
|
|
281
|
+
except (json.JSONDecodeError, UnsupportedTranscriptVersion, KeyError):
|
|
282
|
+
return None
|
|
283
|
+
audio = directory / AUDIO_NAME
|
|
284
|
+
formats = tuple(
|
|
285
|
+
ext for ext in TRANSCRIPT_FORMATS if (directory / f"{TRANSCRIPT_STEM}.{ext}").exists()
|
|
286
|
+
)
|
|
287
|
+
return MeetingRecord(
|
|
288
|
+
id=directory.name,
|
|
289
|
+
title=transcript.profile.title,
|
|
290
|
+
created_at=_created_at_from_id(directory.name),
|
|
291
|
+
duration_s=max((e.end for e in transcript.entries), default=0.0),
|
|
292
|
+
language=transcript.language,
|
|
293
|
+
speakers=_speakers_from_transcript(transcript),
|
|
294
|
+
formats=formats,
|
|
295
|
+
dir=directory,
|
|
296
|
+
audio_path=audio if audio.exists() else None,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _speakers_from_transcript(transcript: Transcript) -> dict[str, int | None]:
|
|
301
|
+
"""Per-channel resolved speaker counts off a transcript's parameters (empty
|
|
302
|
+
when it has none — e.g. a crash checkpoint predating the finalize)."""
|
|
303
|
+
if transcript.parameters is None:
|
|
304
|
+
return {}
|
|
305
|
+
return {ch: rv.value for ch, rv in transcript.parameters.speakers.items()} # type: ignore[misc]
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _atomic_write_text(path: Path, text: str) -> None:
|
|
309
|
+
"""Write ``text`` via a temp file + replace, so a reader never sees a partial
|
|
310
|
+
file (the same discipline :meth:`MeetingArchive.save` uses for the index)."""
|
|
311
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
312
|
+
tmp.write_text(text, encoding="utf-8")
|
|
313
|
+
tmp.replace(path)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _created_at_from_id(meeting_id: str) -> str:
|
|
317
|
+
"""Recover an ISO timestamp from a ``meeting-YYYYMMDD-HHMMSS`` id (the id
|
|
318
|
+
encodes it), or ``""`` for a non-standard dir name adopted during reconcile."""
|
|
319
|
+
match = _ID_TIMESTAMP.match(meeting_id)
|
|
320
|
+
if match:
|
|
321
|
+
try:
|
|
322
|
+
return datetime.strptime(match.group(1) + match.group(2), "%Y%m%d%H%M%S").isoformat()
|
|
323
|
+
except ValueError:
|
|
324
|
+
pass
|
|
325
|
+
return ""
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def meetings_dir() -> Path:
|
|
329
|
+
"""Managed archive root: ``data_dir()/meetings`` (honors ``$STENOGRAF_DATA``)."""
|
|
330
|
+
return data_dir() / "meetings"
|