sample-indexer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sample_indexer/__init__.py +6 -0
- sample_indexer/__main__.py +6 -0
- sample_indexer/audio.py +423 -0
- sample_indexer/audio_dsp.py +160 -0
- sample_indexer/cli.py +343 -0
- sample_indexer/config.py +43 -0
- sample_indexer/db.py +132 -0
- sample_indexer/hashing.py +18 -0
- sample_indexer/labels.py +443 -0
- sample_indexer/loops.py +574 -0
- sample_indexer/loops_dsp.py +503 -0
- sample_indexer/lowend.py +660 -0
- sample_indexer/lowend_dsp.py +459 -0
- sample_indexer/metadata.py +663 -0
- sample_indexer/midi.py +133 -0
- sample_indexer/midi_parser.py +284 -0
- sample_indexer/migrations.py +323 -0
- sample_indexer/reports.py +411 -0
- sample_indexer/scanner.py +440 -0
- sample_indexer-0.1.0.dist-info/METADATA +233 -0
- sample_indexer-0.1.0.dist-info/RECORD +25 -0
- sample_indexer-0.1.0.dist-info/WHEEL +5 -0
- sample_indexer-0.1.0.dist-info/entry_points.txt +2 -0
- sample_indexer-0.1.0.dist-info/licenses/LICENSE +21 -0
- sample_indexer-0.1.0.dist-info/top_level.txt +1 -0
sample_indexer/audio.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"""L3-6 streaming full-stream descriptors, isolated from earlier DSP measurements.
|
|
2
|
+
|
|
3
|
+
One ffmpeg process splits the first audio stream: unchanged native PCM feeds the
|
|
4
|
+
streaming analyzer; loudnorm's INPUT measurements feed loudness descriptors. Neither
|
|
5
|
+
normalized PCM nor output loudness is used. Only the main thread writes SQLite.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import shutil
|
|
15
|
+
import sqlite3
|
|
16
|
+
import subprocess
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from collections import defaultdict
|
|
20
|
+
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
from .db import utcnow
|
|
26
|
+
from .lowend import Item, VALID_SCOPE, classify_failure, _StratumView
|
|
27
|
+
from .metadata import ProcessRegistry, _strata_order
|
|
28
|
+
|
|
29
|
+
AUDIO_STAGE = "audio"
|
|
30
|
+
AUDIO_VERSION = 1
|
|
31
|
+
DEFAULT_AUDIO_WORKERS = 2
|
|
32
|
+
DEFAULT_AUDIO_BATCH = 50
|
|
33
|
+
DEFAULT_DECODE_TIMEOUT_S = 120.0
|
|
34
|
+
CHUNK_BYTES = 65536
|
|
35
|
+
STDERR_BYTES = 65536
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def settings() -> dict:
|
|
39
|
+
return {"stage_version": AUDIO_VERSION, "pcm": "native f32le first audio stream",
|
|
40
|
+
"meter": "loudnorm input_i/input_tp I=-23:TP=-2:LRA=7",
|
|
41
|
+
"dsp_version": getattr(_dsp(), "VERSION", 1)}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _dsp():
|
|
45
|
+
# Lazy import lets schema/CLI tooling load before the independent DSP module exists.
|
|
46
|
+
from . import audio_dsp
|
|
47
|
+
return audio_dsp
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def settings_sig() -> str:
|
|
51
|
+
return hashlib.sha256(json.dumps(settings(), sort_keys=True).encode()).hexdigest()[:32]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class Outcome:
|
|
56
|
+
status: str
|
|
57
|
+
error: str | None = None
|
|
58
|
+
frames: int = 0
|
|
59
|
+
analysis: dict | None = None
|
|
60
|
+
spawned: bool = False
|
|
61
|
+
full_decode: bool = False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def parse_loudness(stderr: str) -> tuple[dict, str]:
|
|
65
|
+
"""Remove only loudnorm's JSON, leaving decoder warnings for validity decisions."""
|
|
66
|
+
matches = list(re.finditer(r'\{[^{}]*"input_i"[^{}]*\}', stderr, re.S))
|
|
67
|
+
if not matches:
|
|
68
|
+
return {}, stderr
|
|
69
|
+
match = matches[-1]
|
|
70
|
+
try:
|
|
71
|
+
data = json.loads(match.group())
|
|
72
|
+
values = {}
|
|
73
|
+
for source, dest in (("input_i", "integrated_lufs"), ("input_tp", "true_peak_dbtp")):
|
|
74
|
+
value = float(data[source])
|
|
75
|
+
values[dest] = value if math.isfinite(value) else None
|
|
76
|
+
values["loudness_status"] = "ok" if values["integrated_lufs"] is not None else "below_gate_or_short"
|
|
77
|
+
except (ValueError, KeyError, TypeError):
|
|
78
|
+
return {}, stderr
|
|
79
|
+
# Level-prefixed info logging exposes the meter without confusing input paths
|
|
80
|
+
# or informational codec descriptions with decoder diagnostics.
|
|
81
|
+
return values, stderr[:match.start()] + stderr[match.end():]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _diagnostics(text: str) -> str:
|
|
85
|
+
"""ffmpeg info-level is required for loudnorm JSON; identify suspicious diagnostics."""
|
|
86
|
+
lines = []
|
|
87
|
+
for line in text.splitlines():
|
|
88
|
+
# WAV commonly omits a layout mask. Guessing mono/stereo from the native
|
|
89
|
+
# channel count does not imply a damaged or incomplete source decode.
|
|
90
|
+
if re.search(r"\[warning\] Guessed Channel Layout: (?:mono|stereo)$", line):
|
|
91
|
+
continue
|
|
92
|
+
if re.search(r"\[(?:warning|error|fatal|panic)\]", line, re.I):
|
|
93
|
+
lines.append(line.strip())
|
|
94
|
+
return " ".join(lines)[:1000]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def ffmpeg_decode(path: str, sr: int, channels: int, *, timeout: float = DEFAULT_DECODE_TIMEOUT_S,
|
|
98
|
+
ffmpeg_bin: str = "ffmpeg", registry: ProcessRegistry | None = None,
|
|
99
|
+
analyzer_factory=None) -> Outcome:
|
|
100
|
+
"""Read bounded frame-aligned blocks; drain bounded stderr concurrently; kill on timeout."""
|
|
101
|
+
if sr <= 0 or channels <= 0 or channels > 64:
|
|
102
|
+
return Outcome("unresolved_metadata", "unsupported sample rate/channel count")
|
|
103
|
+
try:
|
|
104
|
+
module = _dsp() if analyzer_factory is None else None
|
|
105
|
+
factory = analyzer_factory or getattr(module, "StreamingAnalyzer", None) or module.AudioAccumulator
|
|
106
|
+
analyzer = factory(sr, channels)
|
|
107
|
+
except Exception as exc:
|
|
108
|
+
return Outcome("analysis_error", f"{type(exc).__name__}: {exc}")
|
|
109
|
+
argv = [ffmpeg_bin, "-nostdin", "-hide_banner", "-nostats", "-v", "level+info", "-xerror",
|
|
110
|
+
"-threads", "1", "-protocol_whitelist", "file", "-i", "file:" + path,
|
|
111
|
+
"-filter_complex_threads", "1", "-filter_complex",
|
|
112
|
+
"[0:a:0]asplit=2[pcm][meter];[meter]loudnorm=I=-23:TP=-2:LRA=7:print_format=json[loud]",
|
|
113
|
+
"-map", "[pcm]", "-ar", str(sr), "-ac", str(channels), "-threads", "1",
|
|
114
|
+
"-c:a", "pcm_f32le", "-f", "f32le", "pipe:1",
|
|
115
|
+
"-map", "[loud]", "-threads", "1", "-f", "null", "-"]
|
|
116
|
+
try:
|
|
117
|
+
proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
118
|
+
except OSError as exc:
|
|
119
|
+
return Outcome("tool_error", str(exc))
|
|
120
|
+
if registry:
|
|
121
|
+
registry.add(proc)
|
|
122
|
+
tail = bytearray()
|
|
123
|
+
diagnostic_lines = []
|
|
124
|
+
|
|
125
|
+
def drain():
|
|
126
|
+
pending_line = b""
|
|
127
|
+
for chunk in iter(lambda: proc.stderr.read(4096), b""):
|
|
128
|
+
tail.extend(chunk)
|
|
129
|
+
if len(tail) > STDERR_BYTES:
|
|
130
|
+
del tail[:-STDERR_BYTES]
|
|
131
|
+
pending_line += chunk
|
|
132
|
+
lines = pending_line.split(b"\n")
|
|
133
|
+
pending_line = lines.pop()
|
|
134
|
+
for line in lines:
|
|
135
|
+
diagnostic = _diagnostics(line.decode("utf-8", "replace"))
|
|
136
|
+
if diagnostic and len(diagnostic_lines) < 8:
|
|
137
|
+
diagnostic_lines.append(diagnostic)
|
|
138
|
+
# Bound even a malformed diagnostic line without a newline.
|
|
139
|
+
if len(pending_line) > STDERR_BYTES:
|
|
140
|
+
diagnostic = _diagnostics(pending_line.decode("utf-8", "replace"))
|
|
141
|
+
if diagnostic and len(diagnostic_lines) < 8:
|
|
142
|
+
diagnostic_lines.append(diagnostic)
|
|
143
|
+
pending_line = pending_line[-4096:]
|
|
144
|
+
diagnostic = _diagnostics(pending_line.decode("utf-8", "replace"))
|
|
145
|
+
if diagnostic and len(diagnostic_lines) < 8:
|
|
146
|
+
diagnostic_lines.append(diagnostic)
|
|
147
|
+
|
|
148
|
+
thread = threading.Thread(target=drain, daemon=True)
|
|
149
|
+
thread.start()
|
|
150
|
+
expired = threading.Event()
|
|
151
|
+
|
|
152
|
+
def expire():
|
|
153
|
+
expired.set()
|
|
154
|
+
try:
|
|
155
|
+
proc.kill()
|
|
156
|
+
except OSError:
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
timer = threading.Timer(timeout, expire)
|
|
160
|
+
timer.start()
|
|
161
|
+
frames = 0
|
|
162
|
+
remainder = b""
|
|
163
|
+
analysis_error = None
|
|
164
|
+
try:
|
|
165
|
+
while True:
|
|
166
|
+
chunk = proc.stdout.read(CHUNK_BYTES)
|
|
167
|
+
if not chunk:
|
|
168
|
+
break
|
|
169
|
+
block = remainder + chunk
|
|
170
|
+
aligned = len(block) // (4 * channels) * (4 * channels)
|
|
171
|
+
if aligned:
|
|
172
|
+
samples = np.frombuffer(block[:aligned], dtype="<f4").reshape(-1, channels)
|
|
173
|
+
frames += len(samples)
|
|
174
|
+
if analysis_error is None:
|
|
175
|
+
try:
|
|
176
|
+
analyzer.update(samples)
|
|
177
|
+
except Exception as exc:
|
|
178
|
+
analysis_error = f"{type(exc).__name__}: {exc}"
|
|
179
|
+
remainder = block[aligned:]
|
|
180
|
+
proc.wait()
|
|
181
|
+
finally:
|
|
182
|
+
timer.cancel()
|
|
183
|
+
if proc.poll() is None:
|
|
184
|
+
proc.kill()
|
|
185
|
+
proc.wait()
|
|
186
|
+
thread.join()
|
|
187
|
+
proc.stdout.close()
|
|
188
|
+
proc.stderr.close()
|
|
189
|
+
if registry:
|
|
190
|
+
registry.discard(proc)
|
|
191
|
+
if expired.is_set():
|
|
192
|
+
return Outcome("timeout", f"ffmpeg exceeded {timeout:g}s", frames, spawned=True)
|
|
193
|
+
if registry and registry.stopping:
|
|
194
|
+
return Outcome("interrupted", "stage interrupted", frames, spawned=True)
|
|
195
|
+
stderr = tail.decode("utf-8", "replace")
|
|
196
|
+
if proc.returncode:
|
|
197
|
+
return Outcome(classify_failure(proc.returncode, stderr), _diagnostics(stderr) or stderr[-1000:],
|
|
198
|
+
frames, spawned=True)
|
|
199
|
+
loudness, remaining = parse_loudness(stderr)
|
|
200
|
+
warnings = " ".join(diagnostic_lines) or _diagnostics(remaining)
|
|
201
|
+
if warnings:
|
|
202
|
+
return Outcome("decode_warnings", warnings, frames, spawned=True)
|
|
203
|
+
if remainder or not frames:
|
|
204
|
+
return Outcome("empty_audio" if not frames else "tool_error", "empty or incomplete PCM frames",
|
|
205
|
+
frames, spawned=True)
|
|
206
|
+
if analysis_error:
|
|
207
|
+
return Outcome("analysis_error", analysis_error, frames, spawned=True, full_decode=True)
|
|
208
|
+
try:
|
|
209
|
+
analysis = analyzer.finish()
|
|
210
|
+
analysis.update(loudness)
|
|
211
|
+
reasons = analysis.setdefault("null_reasons", {})
|
|
212
|
+
for key in ("integrated_lufs", "true_peak_dbtp"):
|
|
213
|
+
if analysis.get(key) is None:
|
|
214
|
+
analysis[key] = None
|
|
215
|
+
reasons[key] = ("meter_unavailable" if not loudness else
|
|
216
|
+
"shorter_than_400ms_gate" if key == "integrated_lufs" and frames / sr < 0.4 else
|
|
217
|
+
"silent_or_below_absolute_gate" if key == "integrated_lufs" else "silent")
|
|
218
|
+
if not loudness:
|
|
219
|
+
analysis["loudness_status"] = "unavailable"
|
|
220
|
+
# Reject non-JSON descriptors before crossing back to the SQLite thread.
|
|
221
|
+
json.dumps(analysis, allow_nan=False)
|
|
222
|
+
except Exception as exc:
|
|
223
|
+
return Outcome("analysis_error", f"{type(exc).__name__}: {exc}", frames,
|
|
224
|
+
spawned=True, full_decode=True)
|
|
225
|
+
return Outcome("ok", frames=frames, analysis=analysis, spawned=True, full_decode=True)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass
|
|
229
|
+
class AudioReport:
|
|
230
|
+
status: str = "running"
|
|
231
|
+
stop_reason: str | None = None
|
|
232
|
+
selected: int = 0
|
|
233
|
+
already_done: int = 0
|
|
234
|
+
audio_with_path: int = 0
|
|
235
|
+
decoded: int = 0
|
|
236
|
+
ok: int = 0
|
|
237
|
+
valid1: int = 0
|
|
238
|
+
valid0: int = 0
|
|
239
|
+
pending_after: int = 0
|
|
240
|
+
elapsed_s: float = 0.0
|
|
241
|
+
statuses: dict = field(default_factory=lambda: defaultdict(int))
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def processed(self):
|
|
245
|
+
return sum(self.statuses.values())
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def unresolved(self):
|
|
249
|
+
return sum(n for status, n in self.statuses.items() if status not in ("ok", "decode_failed"))
|
|
250
|
+
|
|
251
|
+
def format(self):
|
|
252
|
+
return (f"audio {self.status}" + (f" ({self.stop_reason})" if self.stop_reason else "") +
|
|
253
|
+
f"\n selected {self.selected}; already complete {self.already_done}"
|
|
254
|
+
f"\n processed {self.processed}; ok {self.ok}; unresolved {self.unresolved}"
|
|
255
|
+
f"\n ffmpeg subprocesses {self.decoded}; valid=1 / valid=0 {self.valid1} / {self.valid0}"
|
|
256
|
+
f"\n statuses {dict(self.statuses)}"
|
|
257
|
+
f"\n still pending {self.pending_after}; elapsed {self.elapsed_s:.2f}s")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class AudioStage:
|
|
261
|
+
def __init__(self, conn, *, workers=DEFAULT_AUDIO_WORKERS, limit=None, force=False,
|
|
262
|
+
retry_errors=False, stratify=False, seed=0, timeout=DEFAULT_DECODE_TIMEOUT_S,
|
|
263
|
+
batch_size=DEFAULT_AUDIO_BATCH, max_seconds=None, ffmpeg_bin="ffmpeg",
|
|
264
|
+
decode_func=None, progress=None):
|
|
265
|
+
if workers < 1 or batch_size < 1 or timeout <= 0 or (limit is not None and limit < 0):
|
|
266
|
+
raise ValueError("invalid worker, batch, timeout or limit setting")
|
|
267
|
+
if max_seconds is not None and max_seconds <= 0:
|
|
268
|
+
raise ValueError("max_seconds must be positive")
|
|
269
|
+
self.conn, self.workers, self.limit = conn, workers, limit
|
|
270
|
+
self.force, self.retry_errors = force, retry_errors
|
|
271
|
+
self.stratify, self.seed = stratify, seed
|
|
272
|
+
self.timeout, self.batch_size, self.max_seconds = timeout, batch_size, max_seconds
|
|
273
|
+
self.ffmpeg_bin, self.decode_func = ffmpeg_bin, decode_func
|
|
274
|
+
self.progress = progress or (lambda msg: None)
|
|
275
|
+
self.registry = ProcessRegistry()
|
|
276
|
+
self.report = AudioReport()
|
|
277
|
+
self.sig = settings_sig()
|
|
278
|
+
|
|
279
|
+
def select(self):
|
|
280
|
+
paths = defaultdict(list)
|
|
281
|
+
for r in self.conn.execute("SELECT resource_id, root, rel_path, size, mtime_ns FROM path"
|
|
282
|
+
" WHERE missing=0 ORDER BY rel_path"):
|
|
283
|
+
paths[r["resource_id"]].append((r["root"], r["rel_path"], r["size"], r["mtime_ns"]))
|
|
284
|
+
stages = {r["resource_id"]: r for r in self.conn.execute(
|
|
285
|
+
"SELECT resource_id, version, input_sig, error FROM resource_stage WHERE stage='audio'")}
|
|
286
|
+
items = []
|
|
287
|
+
for r in self.conn.execute("SELECT id,sr,channels,duration_s,probe_status FROM resource"
|
|
288
|
+
" WHERE lower(ext) NOT IN ('.mid','.midi') ORDER BY id"):
|
|
289
|
+
if r["id"] not in paths:
|
|
290
|
+
continue
|
|
291
|
+
self.report.audio_with_path += 1
|
|
292
|
+
old = stages.get(r["id"])
|
|
293
|
+
current = old is not None and old["version"] == AUDIO_VERSION and old["input_sig"] == self.sig
|
|
294
|
+
need = self.force or (current and old["error"] is not None if self.retry_errors else not current)
|
|
295
|
+
if need:
|
|
296
|
+
items.append(Item(r["id"], r["sr"], r["channels"], r["duration_s"], r["probe_status"],
|
|
297
|
+
{}, paths[r["id"]]))
|
|
298
|
+
else:
|
|
299
|
+
self.report.already_done += 1
|
|
300
|
+
if self.stratify:
|
|
301
|
+
items = [v.item for v in _strata_order([_StratumView(i) for i in items], self.seed)]
|
|
302
|
+
if self.limit is not None:
|
|
303
|
+
items = items[:self.limit]
|
|
304
|
+
self.report.selected = len(items)
|
|
305
|
+
return items
|
|
306
|
+
|
|
307
|
+
def _worker(self, item):
|
|
308
|
+
if not item.sr or not item.channels or item.probe_status != "ok":
|
|
309
|
+
return Outcome("unresolved_metadata", "native sample rate/channels unavailable; run metadata")
|
|
310
|
+
last = Outcome("unreadable", "no readable present path")
|
|
311
|
+
for root, rel, size, mtime in item.paths:
|
|
312
|
+
path = os.path.join(root, *rel.split("/"))
|
|
313
|
+
try:
|
|
314
|
+
st = os.stat(path)
|
|
315
|
+
except OSError as exc:
|
|
316
|
+
last = Outcome("unreadable", str(exc))
|
|
317
|
+
continue
|
|
318
|
+
if (st.st_size, st.st_mtime_ns) != (size, mtime):
|
|
319
|
+
last = Outcome("stale", "file changed since scan; rescan")
|
|
320
|
+
continue
|
|
321
|
+
if self.decode_func:
|
|
322
|
+
out = self.decode_func(path, item.sr, item.channels)
|
|
323
|
+
else:
|
|
324
|
+
out = ffmpeg_decode(path, item.sr, item.channels, timeout=self.timeout,
|
|
325
|
+
ffmpeg_bin=self.ffmpeg_bin, registry=self.registry)
|
|
326
|
+
try:
|
|
327
|
+
after = os.stat(path)
|
|
328
|
+
unchanged = (after.st_size, after.st_mtime_ns) == (size, mtime)
|
|
329
|
+
except OSError:
|
|
330
|
+
unchanged = False
|
|
331
|
+
if not unchanged:
|
|
332
|
+
return Outcome("stale", "file changed during decode; rescan", spawned=out.spawned)
|
|
333
|
+
return out
|
|
334
|
+
return last
|
|
335
|
+
|
|
336
|
+
def _store(self, item, out):
|
|
337
|
+
now = utcnow()
|
|
338
|
+
descriptor = json.dumps({"stage_version": AUDIO_VERSION, "settings": settings(),
|
|
339
|
+
"decoded_frames": out.frames, "analysis": out.analysis},
|
|
340
|
+
sort_keys=True, allow_nan=False)
|
|
341
|
+
error = None if out.status == "ok" else f"{out.status}: {out.error}"
|
|
342
|
+
self.conn.execute("INSERT OR REPLACE INTO audio_measurement"
|
|
343
|
+
"(resource_id,status,error,measured_at,descriptor) VALUES (?,?,?,?,?)",
|
|
344
|
+
(item.rid, out.status, error, now, descriptor))
|
|
345
|
+
self.conn.execute("INSERT OR REPLACE INTO resource_stage"
|
|
346
|
+
"(resource_id,stage,version,done_at,error,input_sig) VALUES (?,?,?,?,?,?)",
|
|
347
|
+
(item.rid, AUDIO_STAGE, AUDIO_VERSION, now, error, self.sig))
|
|
348
|
+
# Inconclusive attempts MUST preserve previous validity, scope and frame count.
|
|
349
|
+
if out.full_decode or out.status == "decode_failed":
|
|
350
|
+
valid = int(out.full_decode)
|
|
351
|
+
self.conn.execute("UPDATE resource SET valid=?,valid_error=?,valid_scope=?,decoded_frames=? WHERE id=?",
|
|
352
|
+
(valid, None if valid else out.error, VALID_SCOPE, out.frames, item.rid))
|
|
353
|
+
an = out.analysis or {}
|
|
354
|
+
try:
|
|
355
|
+
self.conn.execute("UPDATE resource SET centroid=?,rolloff=?,flatness=?,lufs=?,true_peak=? WHERE id=?",
|
|
356
|
+
(an.get("spectral_centroid_hz"), an.get("spectral_rolloff85_hz"),
|
|
357
|
+
an.get("spectral_flatness"), an.get("integrated_lufs"),
|
|
358
|
+
an.get("true_peak_dbtp"), item.rid))
|
|
359
|
+
except sqlite3.OperationalError:
|
|
360
|
+
pass
|
|
361
|
+
if valid:
|
|
362
|
+
self.report.valid1 += 1
|
|
363
|
+
else:
|
|
364
|
+
self.report.valid0 += 1
|
|
365
|
+
self.report.statuses[out.status] += 1
|
|
366
|
+
self.report.decoded += int(out.spawned)
|
|
367
|
+
self.report.ok += int(out.status == "ok")
|
|
368
|
+
|
|
369
|
+
def run(self):
|
|
370
|
+
if self.decode_func is None and shutil.which(self.ffmpeg_bin) is None:
|
|
371
|
+
raise FileNotFoundError(f"{self.ffmpeg_bin} not found on PATH")
|
|
372
|
+
started = time.monotonic()
|
|
373
|
+
items = iter(self.select())
|
|
374
|
+
deadline = started + self.max_seconds if self.max_seconds is not None else None
|
|
375
|
+
pending = {}
|
|
376
|
+
executor = ThreadPoolExecutor(max_workers=self.workers, thread_name_prefix="audio")
|
|
377
|
+
writes = 0
|
|
378
|
+
try:
|
|
379
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
380
|
+
exhausted = False
|
|
381
|
+
while pending or not exhausted:
|
|
382
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
383
|
+
self.report.stop_reason = "max-seconds reached"
|
|
384
|
+
self.registry.kill_all()
|
|
385
|
+
break
|
|
386
|
+
while not exhausted and len(pending) < self.workers * 2:
|
|
387
|
+
item = next(items, None)
|
|
388
|
+
if item is None:
|
|
389
|
+
exhausted = True
|
|
390
|
+
break
|
|
391
|
+
pending[executor.submit(self._worker, item)] = item
|
|
392
|
+
if not pending:
|
|
393
|
+
break
|
|
394
|
+
done, _ = wait(pending, timeout=0.05, return_when=FIRST_COMPLETED)
|
|
395
|
+
for future in done:
|
|
396
|
+
self._store(pending.pop(future), future.result())
|
|
397
|
+
writes += 1
|
|
398
|
+
if writes >= self.batch_size:
|
|
399
|
+
self.conn.execute("COMMIT")
|
|
400
|
+
self.progress(f"audio checkpoint: {self.report.processed}/{self.report.selected} processed; "
|
|
401
|
+
f"decoded={self.report.decoded} ok={self.report.ok} "
|
|
402
|
+
f"statuses={dict(self.report.statuses)} elapsed={time.monotonic()-started:.1f}s")
|
|
403
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
404
|
+
writes = 0
|
|
405
|
+
except KeyboardInterrupt:
|
|
406
|
+
self.report.stop_reason = "keyboard interrupt"
|
|
407
|
+
self.registry.kill_all()
|
|
408
|
+
except BaseException:
|
|
409
|
+
self.registry.kill_all()
|
|
410
|
+
if self.conn.in_transaction:
|
|
411
|
+
self.conn.execute("ROLLBACK")
|
|
412
|
+
raise
|
|
413
|
+
finally:
|
|
414
|
+
for future in pending:
|
|
415
|
+
future.cancel()
|
|
416
|
+
executor.shutdown(wait=True, cancel_futures=True)
|
|
417
|
+
if self.conn.in_transaction:
|
|
418
|
+
self.conn.execute("COMMIT")
|
|
419
|
+
self.report.status = "interrupted" if self.report.stop_reason else "complete"
|
|
420
|
+
self.report.elapsed_s = time.monotonic() - started
|
|
421
|
+
probe = AudioStage(self.conn, decode_func=self.decode_func)
|
|
422
|
+
self.report.pending_after = len(probe.select())
|
|
423
|
+
return self.report
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""L3-6 bounded, native-channel streaming measurements (no I/O).
|
|
2
|
+
|
|
3
|
+
Feed finite float arrays shaped (frames, channels) to AudioAccumulator.update;
|
|
4
|
+
finish once. Working memory is O(nfft * channels), independent of duration and
|
|
5
|
+
caller chunk size. Spectra use periodic Hann windows, 75% overlap and zero
|
|
6
|
+
padding at both ends: integrated power is whole-stream mean square (Parseval).
|
|
7
|
+
Centroid/rolloff/flatness describe the channel-mean POWER spectrum, including DC.
|
|
8
|
+
Clipping counts samples in runs of at least two, independently per channel.
|
|
9
|
+
"""
|
|
10
|
+
import math
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from .lowend_dsp import BANDS, STEREO_BAND
|
|
15
|
+
|
|
16
|
+
VERSION = 1
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AudioAccumulator:
|
|
20
|
+
"""Streaming DSP; first two channels define stereo, all channels define power."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, sr, channels, *, nfft=32768, clip_threshold=0.999):
|
|
23
|
+
if isinstance(sr, bool) or not isinstance(sr, (int, float)) or not math.isfinite(sr) or sr < 2000:
|
|
24
|
+
raise ValueError("sr must be finite and >= 2000")
|
|
25
|
+
if isinstance(channels, bool) or not isinstance(channels, int) or not 1 <= channels <= 256:
|
|
26
|
+
raise ValueError("channels must be an integer in [1, 256]")
|
|
27
|
+
if isinstance(nfft, bool) or not isinstance(nfft, int) or not 16 <= nfft <= 131072 or nfft & (nfft - 1):
|
|
28
|
+
raise ValueError("nfft must be a power of two in [16, 131072]")
|
|
29
|
+
if not math.isfinite(clip_threshold) or clip_threshold <= 0:
|
|
30
|
+
raise ValueError("clip_threshold must be finite and positive")
|
|
31
|
+
self.sr, self.channels, self.nfft = float(sr), channels, nfft
|
|
32
|
+
self.clip_threshold = float(clip_threshold)
|
|
33
|
+
self.hop = nfft // 4
|
|
34
|
+
self.window = np.hanning(nfft + 1)[:-1]
|
|
35
|
+
self.buffer = np.zeros((nfft, channels))
|
|
36
|
+
self.used = nfft - self.hop
|
|
37
|
+
self.frames = self.clipped = 0
|
|
38
|
+
self.peak = self.cross = 0.0
|
|
39
|
+
self.sums = np.zeros(channels)
|
|
40
|
+
self.squares = np.zeros(channels)
|
|
41
|
+
self.last_clip = np.zeros(channels, dtype=bool)
|
|
42
|
+
self.pending = np.zeros(channels, dtype=bool)
|
|
43
|
+
self.freqs = np.fft.rfftfreq(nfft, 1 / self.sr)
|
|
44
|
+
self.power = np.zeros(len(self.freqs))
|
|
45
|
+
self.low = np.zeros(3)
|
|
46
|
+
self.closed = False
|
|
47
|
+
|
|
48
|
+
def _fft(self):
|
|
49
|
+
z = np.fft.rfft(self.buffer * self.window[:, None], axis=0)
|
|
50
|
+
scale = np.full(len(z), 2.0)
|
|
51
|
+
scale[[0, -1]] = 1.0
|
|
52
|
+
p = (z.real ** 2 + z.imag ** 2) * scale[:, None]
|
|
53
|
+
self.power += p.mean(axis=1)
|
|
54
|
+
if self.channels >= 2:
|
|
55
|
+
mask = (self.freqs >= STEREO_BAND[0]) & (self.freqs < STEREO_BAND[1])
|
|
56
|
+
self.low += [float(p[mask, 0].sum()), float(p[mask, 1].sum()),
|
|
57
|
+
float(((z[mask, 0] * z[mask, 1].conj()).real * scale[mask]).sum())]
|
|
58
|
+
self.buffer[:-self.hop] = self.buffer[self.hop:]
|
|
59
|
+
self.used -= self.hop
|
|
60
|
+
|
|
61
|
+
def update(self, chunk):
|
|
62
|
+
"""Consume without retaining or modifying caller memory; empty chunks are OK."""
|
|
63
|
+
if self.closed:
|
|
64
|
+
raise ValueError("accumulator is finished")
|
|
65
|
+
if not isinstance(chunk, np.ndarray) or chunk.ndim != 2 or chunk.shape[1] != self.channels or chunk.dtype.kind != 'f':
|
|
66
|
+
raise ValueError("chunk must be a floating array (frames, channels)")
|
|
67
|
+
# Validate the entire update before mutating state, with bounded temporaries.
|
|
68
|
+
for start in range(0, len(chunk), self.nfft):
|
|
69
|
+
c = chunk[start:start + self.nfft]
|
|
70
|
+
if not np.isfinite(c).all() or (np.abs(c.astype(np.float64, copy=False)) > 1e100).any():
|
|
71
|
+
raise ValueError("samples must be finite and bounded by 1e100")
|
|
72
|
+
for start in range(0, len(chunk), self.hop):
|
|
73
|
+
c = np.asarray(chunk[start:start + self.hop], dtype=np.float64)
|
|
74
|
+
self.frames += len(c)
|
|
75
|
+
self.peak = max(self.peak, float(np.abs(c).max()))
|
|
76
|
+
self.sums += c.sum(axis=0)
|
|
77
|
+
self.squares += np.einsum('ij,ij->j', c, c)
|
|
78
|
+
if self.channels >= 2:
|
|
79
|
+
self.cross += float(np.dot(c[:, 0], c[:, 1]))
|
|
80
|
+
m = np.abs(c) >= self.clip_threshold
|
|
81
|
+
prev = np.vstack((self.last_clip, m[:-1]))
|
|
82
|
+
nxt = np.vstack((m[1:], np.zeros((1, self.channels), dtype=bool)))
|
|
83
|
+
self.clipped += int((m & (prev | nxt)).sum()) + int((self.pending & m[0]).sum())
|
|
84
|
+
self.pending = m[-1] & ~prev[-1]
|
|
85
|
+
self.last_clip = m[-1].copy()
|
|
86
|
+
pos = 0
|
|
87
|
+
while pos < len(c):
|
|
88
|
+
take = min(self.nfft - self.used, len(c) - pos)
|
|
89
|
+
self.buffer[self.used:self.used + take] = c[pos:pos + take]
|
|
90
|
+
self.used += take
|
|
91
|
+
pos += take
|
|
92
|
+
if self.used == self.nfft:
|
|
93
|
+
self._fft()
|
|
94
|
+
return self
|
|
95
|
+
|
|
96
|
+
def finish(self):
|
|
97
|
+
"""Finalize and return JSON-finite metrics, provenance and per-field null reasons."""
|
|
98
|
+
if self.closed or not self.frames:
|
|
99
|
+
raise ValueError("accumulator finished or empty")
|
|
100
|
+
self.closed = True
|
|
101
|
+
# Flush every frame overlapping real data, including the trailing Hann skirt.
|
|
102
|
+
remaining = self.used
|
|
103
|
+
while remaining > 0:
|
|
104
|
+
self.buffer[self.used:] = 0
|
|
105
|
+
self._fft()
|
|
106
|
+
remaining -= self.hop
|
|
107
|
+
norm = self.hop / (self.nfft * float(np.dot(self.window, self.window)) * self.frames)
|
|
108
|
+
p = self.power * norm
|
|
109
|
+
total = float(p.sum())
|
|
110
|
+
ms = float(self.squares.mean() / self.frames)
|
|
111
|
+
rms = math.sqrt(ms)
|
|
112
|
+
means = self.sums / self.frames
|
|
113
|
+
out = dict(sample_peak=self.peak, rms=rms,
|
|
114
|
+
crest_db=20 * math.log10(self.peak / rms) if rms else None,
|
|
115
|
+
dc_offset=float(means[np.argmax(np.abs(means))]),
|
|
116
|
+
dc_per_channel=means.tolist(), clipped_samples=self.clipped,
|
|
117
|
+
clipping_fraction=self.clipped / (self.frames * self.channels))
|
|
118
|
+
reasons = {}
|
|
119
|
+
if not rms:
|
|
120
|
+
reasons['crest_db'] = 'silent'
|
|
121
|
+
for name, lo, hi in BANDS:
|
|
122
|
+
mask = (self.freqs >= lo) & (self.freqs < hi if hi is not None else self.freqs <= self.sr / 2)
|
|
123
|
+
out[name] = float(p[mask].sum()) if mask.any() else None
|
|
124
|
+
if out[name] is None:
|
|
125
|
+
reasons[name] = 'no_bins_at_sample_rate_and_resolution'
|
|
126
|
+
out['spectral_centroid_hz'] = float(np.dot(self.freqs, p) / total) if total else None
|
|
127
|
+
out['spectral_rolloff85_hz'] = float(self.freqs[min(int(np.searchsorted(np.cumsum(p), .85 * total)), len(p) - 1)]) if total else None
|
|
128
|
+
out['spectral_flatness'] = (float(math.exp(float(np.log(p).mean())) / p.mean()) if np.all(p > 0) else 0.0) if total else None
|
|
129
|
+
if not total:
|
|
130
|
+
for key in ('spectral_centroid_hz', 'spectral_rolloff85_hz', 'spectral_flatness'):
|
|
131
|
+
reasons[key] = 'silent'
|
|
132
|
+
for key, energy, cross in (
|
|
133
|
+
('stereo_corr', self.squares[:2] / self.frames, self.cross / self.frames),
|
|
134
|
+
('stereo_corr_low', self.low[:2] * norm, self.low[2] * norm)):
|
|
135
|
+
reason = 'mono' if self.channels == 1 else None
|
|
136
|
+
if reason is None and min(energy) <= 1e-12:
|
|
137
|
+
reason = 'no_low_band_energy_in_channel' if key.endswith('_low') else 'no_energy_in_channel'
|
|
138
|
+
out[key] = None if reason else max(-1.0, min(1.0, float(cross / math.sqrt(float(energy[0]) * float(energy[1])))))
|
|
139
|
+
if reason:
|
|
140
|
+
reasons[key] = reason
|
|
141
|
+
out['null_reasons'] = reasons
|
|
142
|
+
out['provenance'] = dict(module=__name__, version=VERSION, numpy=np.__version__,
|
|
143
|
+
scope='complete_native_channel_stream', frames=self.frames,
|
|
144
|
+
channels=self.channels, sr=self.sr, nfft=self.nfft, hop=self.hop,
|
|
145
|
+
window='periodic_hann', spectrum='channel_mean_power_including_dc',
|
|
146
|
+
stereo='first_two_uncentered_normalized_cross_power',
|
|
147
|
+
clipping_threshold=self.clip_threshold, clipping_min_run=2,
|
|
148
|
+
bands_hz=[list(b) for b in BANDS], low_band_hz=list(STEREO_BAND))
|
|
149
|
+
return out
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Public runner-facing name; retain the descriptive accumulator name as well.
|
|
153
|
+
StreamingAnalyzer = AudioAccumulator
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def analyze(samples, sr, **kwargs):
|
|
157
|
+
"""Convenience array API; streaming callers should use AudioAccumulator directly."""
|
|
158
|
+
if not isinstance(samples, np.ndarray) or samples.ndim != 2:
|
|
159
|
+
raise ValueError("samples must be a 2-D numpy array")
|
|
160
|
+
return AudioAccumulator(sr, samples.shape[1], **kwargs).update(samples).finish()
|