goesb-runner 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- goesb_runner-0.0.1/.gitignore +48 -0
- goesb_runner-0.0.1/PKG-INFO +61 -0
- goesb_runner-0.0.1/README.md +35 -0
- goesb_runner-0.0.1/pyproject.toml +37 -0
- goesb_runner-0.0.1/src/oesb_runner/__init__.py +2 -0
- goesb_runner-0.0.1/src/oesb_runner/adapters/__init__.py +50 -0
- goesb_runner-0.0.1/src/oesb_runner/adapters/faster_whisper.py +169 -0
- goesb_runner-0.0.1/src/oesb_runner/adapters/vosk.py +98 -0
- goesb_runner-0.0.1/src/oesb_runner/adapters/whisper_cpp.py +79 -0
- goesb_runner-0.0.1/src/oesb_runner/audio.py +104 -0
- goesb_runner-0.0.1/src/oesb_runner/cli.py +589 -0
- goesb_runner-0.0.1/src/oesb_runner/energy.py +75 -0
- goesb_runner-0.0.1/src/oesb_runner/environment.py +134 -0
- goesb_runner-0.0.1/src/oesb_runner/hashing.py +75 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/_align.py +21 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/cer.py +22 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/cpu_ram.py +49 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/end_of_speech_latency.py +21 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/energy.py +23 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/first_final_latency.py +24 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/first_partial_latency.py +24 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/partial_stability.py +35 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/rtf.py +12 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/streaming_responsiveness.py +24 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/temperature.py +14 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/update_frequency.py +28 -0
- goesb_runner-0.0.1/src/oesb_runner/metrics/wer.py +32 -0
- goesb_runner-0.0.1/src/oesb_runner/normalization/__init__.py +44 -0
- goesb_runner-0.0.1/src/oesb_runner/normalization/oesb_en_v1.py +75 -0
- goesb_runner-0.0.1/src/oesb_runner/normalization/oesb_nl_v1.py +88 -0
- goesb_runner-0.0.1/src/oesb_runner/pack.py +102 -0
- goesb_runner-0.0.1/src/oesb_runner/remote.py +74 -0
- goesb_runner-0.0.1/src/oesb_runner/schema_validation.py +25 -0
- goesb_runner-0.0.1/src/oesb_runner/schemas/benchmark-pack.schema.json +56 -0
- goesb_runner-0.0.1/src/oesb_runner/schemas/benchmark-profile.schema.json +80 -0
- goesb_runner-0.0.1/src/oesb_runner/schemas/benchmark-result.schema.json +149 -0
- goesb_runner-0.0.1/src/oesb_runner/signing.py +170 -0
- goesb_runner-0.0.1/src/oesb_runner/stats.py +38 -0
- goesb_runner-0.0.1/src/oesb_runner/streaming.py +37 -0
- goesb_runner-0.0.1/tests/test_adapter_faster_whisper.py +87 -0
- goesb_runner-0.0.1/tests/test_adapter_vosk.py +50 -0
- goesb_runner-0.0.1/tests/test_adapter_whisper_cpp.py +50 -0
- goesb_runner-0.0.1/tests/test_adapters.py +28 -0
- goesb_runner-0.0.1/tests/test_cli.py +41 -0
- goesb_runner-0.0.1/tests/test_energy.py +73 -0
- goesb_runner-0.0.1/tests/test_environment.py +32 -0
- goesb_runner-0.0.1/tests/test_faster_whisper_model_id.py +26 -0
- goesb_runner-0.0.1/tests/test_hashing.py +40 -0
- goesb_runner-0.0.1/tests/test_metrics.py +97 -0
- goesb_runner-0.0.1/tests/test_normalization.py +56 -0
- goesb_runner-0.0.1/tests/test_normalization_en.py +37 -0
- goesb_runner-0.0.1/tests/test_overhead.py +73 -0
- goesb_runner-0.0.1/tests/test_pack.py +27 -0
- goesb_runner-0.0.1/tests/test_run_end_to_end.py +66 -0
- goesb_runner-0.0.1/tests/test_run_streaming_end_to_end.py +65 -0
- goesb_runner-0.0.1/tests/test_run_vosk_end_to_end.py +51 -0
- goesb_runner-0.0.1/tests/test_run_whispercpp_end_to_end.py +53 -0
- goesb_runner-0.0.1/tests/test_schema_validation.py +26 -0
- goesb_runner-0.0.1/tests/test_signing.py +75 -0
- goesb_runner-0.0.1/tests/test_stats.py +35 -0
- goesb_runner-0.0.1/tests/test_streaming_metrics.py +109 -0
- goesb_runner-0.0.1/tests/test_whisper_cpp_model_id.py +19 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
.venv/
|
|
7
|
+
venv/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
dist/
|
|
12
|
+
build/
|
|
13
|
+
*.spec
|
|
14
|
+
|
|
15
|
+
# Generated fresh at each frozen-binary build (scripts/generate_frozen_adapter_hashes.py)
|
|
16
|
+
runner/src/oesb_runner/_frozen_adapter_hashes.json
|
|
17
|
+
|
|
18
|
+
# Node / Next.js
|
|
19
|
+
node_modules/
|
|
20
|
+
.next/
|
|
21
|
+
out/
|
|
22
|
+
*.tsbuildinfo
|
|
23
|
+
.env*.local
|
|
24
|
+
|
|
25
|
+
# GOESB local run artifacts
|
|
26
|
+
/runs/
|
|
27
|
+
/.goesb-cache/
|
|
28
|
+
*.local.yaml
|
|
29
|
+
|
|
30
|
+
# Audio & datasets are never committed (privacy-first)
|
|
31
|
+
*.wav
|
|
32
|
+
*.flac
|
|
33
|
+
*.mp3
|
|
34
|
+
*.ogg
|
|
35
|
+
*.m4a
|
|
36
|
+
/packs/**/audio/
|
|
37
|
+
|
|
38
|
+
# OS / editor
|
|
39
|
+
.DS_Store
|
|
40
|
+
.idea/
|
|
41
|
+
.vscode/
|
|
42
|
+
*.log
|
|
43
|
+
|
|
44
|
+
# transfer artifact (safe to delete)
|
|
45
|
+
_to_delete/
|
|
46
|
+
|
|
47
|
+
# Local agent/editor settings
|
|
48
|
+
.claude/settings.local.json
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: goesb-runner
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Official Open Edge Speech Benchmark (GOESB) runner
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: cryptography>=42.0
|
|
7
|
+
Requires-Dist: jsonschema>=4.21
|
|
8
|
+
Requires-Dist: packaging>=23.0
|
|
9
|
+
Requires-Dist: psutil>=5.9
|
|
10
|
+
Requires-Dist: pydantic>=2.6
|
|
11
|
+
Requires-Dist: pyyaml>=6.0
|
|
12
|
+
Requires-Dist: typer>=0.12
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: mypy>=1.9; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff~=0.16.0; extra == 'dev'
|
|
17
|
+
Provides-Extra: faster-whisper
|
|
18
|
+
Requires-Dist: faster-whisper>=1.0.0; extra == 'faster-whisper'
|
|
19
|
+
Provides-Extra: vosk
|
|
20
|
+
Requires-Dist: soundfile>=0.12; extra == 'vosk'
|
|
21
|
+
Requires-Dist: vosk>=0.3.44; extra == 'vosk'
|
|
22
|
+
Provides-Extra: whisper-cpp
|
|
23
|
+
Requires-Dist: pywhispercpp>=1.5.0; extra == 'whisper-cpp'
|
|
24
|
+
Requires-Dist: soundfile>=0.12; extra == 'whisper-cpp'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# GOESB Runner
|
|
28
|
+
|
|
29
|
+
The official, reproducible benchmark runner. It loads a signed **Benchmark
|
|
30
|
+
Profile** and **Benchmark Pack**, captures the full hardware/software/model
|
|
31
|
+
environment, runs the benchmark (batch / streaming / conversation) against a
|
|
32
|
+
pluggable **runtime adapter**, computes metrics, and emits a signed,
|
|
33
|
+
hash-verifiable result.
|
|
34
|
+
|
|
35
|
+
The runner **never executes arbitrary user code** (see the security model).
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
```bash
|
|
39
|
+
pip install -e ".[dev,faster-whisper]"
|
|
40
|
+
goesb --help
|
|
41
|
+
goesb env # print captured environment fingerprint
|
|
42
|
+
goesb validate profiles/whisper-medium-en-batch/profile.yaml
|
|
43
|
+
goesb validate packs/example-librispeech-en-batch/pack.yaml
|
|
44
|
+
|
|
45
|
+
# Fetch the example pack's audio (not committed to git — FR-3.5) before
|
|
46
|
+
# running against it. Reruns are safe with --skip-download.
|
|
47
|
+
python ../scripts/fetch_librispeech_subset.py
|
|
48
|
+
|
|
49
|
+
# Run a batch benchmark end-to-end: environment capture, transcription,
|
|
50
|
+
# normalization + WER/CER/RTF/CPU/RAM, a signed+hashed result on disk.
|
|
51
|
+
goesb run whisper-medium-en-batch example-librispeech-en-batch --repeats 2
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Pass `--model-override tiny` (or `base`) to `goesb run` for a fast local smoke
|
|
55
|
+
test instead of downloading the full `whisper-medium` the profile specifies —
|
|
56
|
+
results from an override are for pipeline verification only, never comparable
|
|
57
|
+
to the official profile's numbers.
|
|
58
|
+
|
|
59
|
+
Schema validation currently requires running from within this monorepo
|
|
60
|
+
checkout (see `oesb_runner.schema_validation`); standalone packaging of the
|
|
61
|
+
schemas is tracked for M2.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# GOESB Runner
|
|
2
|
+
|
|
3
|
+
The official, reproducible benchmark runner. It loads a signed **Benchmark
|
|
4
|
+
Profile** and **Benchmark Pack**, captures the full hardware/software/model
|
|
5
|
+
environment, runs the benchmark (batch / streaming / conversation) against a
|
|
6
|
+
pluggable **runtime adapter**, computes metrics, and emits a signed,
|
|
7
|
+
hash-verifiable result.
|
|
8
|
+
|
|
9
|
+
The runner **never executes arbitrary user code** (see the security model).
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
```bash
|
|
13
|
+
pip install -e ".[dev,faster-whisper]"
|
|
14
|
+
goesb --help
|
|
15
|
+
goesb env # print captured environment fingerprint
|
|
16
|
+
goesb validate profiles/whisper-medium-en-batch/profile.yaml
|
|
17
|
+
goesb validate packs/example-librispeech-en-batch/pack.yaml
|
|
18
|
+
|
|
19
|
+
# Fetch the example pack's audio (not committed to git — FR-3.5) before
|
|
20
|
+
# running against it. Reruns are safe with --skip-download.
|
|
21
|
+
python ../scripts/fetch_librispeech_subset.py
|
|
22
|
+
|
|
23
|
+
# Run a batch benchmark end-to-end: environment capture, transcription,
|
|
24
|
+
# normalization + WER/CER/RTF/CPU/RAM, a signed+hashed result on disk.
|
|
25
|
+
goesb run whisper-medium-en-batch example-librispeech-en-batch --repeats 2
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Pass `--model-override tiny` (or `base`) to `goesb run` for a fast local smoke
|
|
29
|
+
test instead of downloading the full `whisper-medium` the profile specifies —
|
|
30
|
+
results from an override are for pipeline verification only, never comparable
|
|
31
|
+
to the official profile's numbers.
|
|
32
|
+
|
|
33
|
+
Schema validation currently requires running from within this monorepo
|
|
34
|
+
checkout (see `oesb_runner.schema_validation`); standalone packaging of the
|
|
35
|
+
schemas is tracked for M2.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "goesb-runner"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "Official Open Edge Speech Benchmark (GOESB) runner"
|
|
5
|
+
requires-python = ">=3.11"
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"pydantic>=2.6",
|
|
9
|
+
"typer>=0.12",
|
|
10
|
+
"pyyaml>=6.0",
|
|
11
|
+
"psutil>=5.9",
|
|
12
|
+
"cryptography>=42.0",
|
|
13
|
+
"jsonschema>=4.21",
|
|
14
|
+
"packaging>=23.0",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
# ruff pinned to a minor version, not left floating: an unpinned >=0.4 let
|
|
19
|
+
# CI silently pick up a newer ruff whose default rule set changed, breaking
|
|
20
|
+
# a previously-green build with no repo change at all.
|
|
21
|
+
dev = ["pytest>=8.0", "ruff~=0.16.0", "mypy>=1.9"]
|
|
22
|
+
faster-whisper = ["faster-whisper>=1.0.0"]
|
|
23
|
+
vosk = ["vosk>=0.3.44", "soundfile>=0.12"]
|
|
24
|
+
whisper-cpp = ["pywhispercpp>=1.5.0", "soundfile>=0.12"]
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
goesb = "oesb_runner.cli:app"
|
|
28
|
+
|
|
29
|
+
[tool.pytest.ini_options]
|
|
30
|
+
markers = ["slow: exercises a real model/network (e.g. faster-whisper inference)"]
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["hatchling"]
|
|
34
|
+
build-backend = "hatchling.build"
|
|
35
|
+
|
|
36
|
+
[tool.hatch.build.targets.wheel]
|
|
37
|
+
packages = ["src/oesb_runner"]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Runtime adapter registry (FR-11.1): how a model is driven for a benchmark
|
|
2
|
+
type. Each adapter is reviewed, in-tree code (ADR-0004) — never a plugin
|
|
3
|
+
supplied at run time. Core dispatch here is runtime-agnostic; adapter modules
|
|
4
|
+
lazy-import their actual ML dependency so importing this package never
|
|
5
|
+
requires it to be installed.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Transcription:
|
|
15
|
+
"""One utterance's batch-adapter output — shared shape every batch
|
|
16
|
+
adapter (faster-whisper, vosk, whisper.cpp, ...) returns, so the CLI's
|
|
17
|
+
batch run loop stays adapter-agnostic."""
|
|
18
|
+
utterance_id: str
|
|
19
|
+
hypothesis_text: str
|
|
20
|
+
processing_time_s: float
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Keyed by (runtime_name, benchmark_type) — a runtime can implement more than
|
|
24
|
+
# one benchmark type (e.g. faster-whisper's "batch" and "streaming" loops are
|
|
25
|
+
# different callables, same underlying runtime).
|
|
26
|
+
_ADAPTERS: dict[tuple[str, str], Callable] = {}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def register(runtime_name: str, benchmark_type: str = "batch") -> Callable[[Callable], Callable]:
|
|
30
|
+
def decorator(fn: Callable) -> Callable:
|
|
31
|
+
key = (runtime_name, benchmark_type)
|
|
32
|
+
if key in _ADAPTERS:
|
|
33
|
+
raise ValueError(f"runtime adapter already registered: {key!r}")
|
|
34
|
+
_ADAPTERS[key] = fn
|
|
35
|
+
return fn
|
|
36
|
+
|
|
37
|
+
return decorator
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_adapter(runtime_name: str, benchmark_type: str = "batch") -> Callable:
|
|
41
|
+
try:
|
|
42
|
+
return _ADAPTERS[(runtime_name, benchmark_type)]
|
|
43
|
+
except KeyError:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"unknown runtime adapter: {runtime_name!r} for benchmark_type {benchmark_type!r}"
|
|
46
|
+
) from None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Built-in adapters register themselves on import.
|
|
50
|
+
from . import faster_whisper, vosk, whisper_cpp # noqa: F401
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""`faster-whisper` batch runtime adapter (docs/02-architecture.md §4).
|
|
2
|
+
|
|
3
|
+
Optional dependency (`pip install goesb-runner[faster-whisper]`) — the actual
|
|
4
|
+
`faster_whisper` package is only imported inside `run_batch`, so importing
|
|
5
|
+
`oesb_runner.adapters` never requires it, matching the normalization plugin
|
|
6
|
+
pattern.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from ..pack import Utterance
|
|
14
|
+
from ..streaming import PartialUpdate, StreamTrace
|
|
15
|
+
from . import Transcription, register
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _resolve_model_id(model_name: str) -> str:
|
|
19
|
+
"""Translate GOESB's runtime-agnostic model name (profiles say
|
|
20
|
+
'whisper-medium') into the identifier faster-whisper's own API expects
|
|
21
|
+
('medium') — this translation belongs in the adapter, not the profile,
|
|
22
|
+
so profiles stay independent of any one runtime's naming convention."""
|
|
23
|
+
prefix = "whisper-"
|
|
24
|
+
return model_name.removeprefix(prefix)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@register("faster-whisper", benchmark_type="batch")
|
|
28
|
+
def run_batch(
|
|
29
|
+
model_name: str,
|
|
30
|
+
utterances: list[Utterance],
|
|
31
|
+
*,
|
|
32
|
+
quantization: str = "int8",
|
|
33
|
+
beam_size: int = 5,
|
|
34
|
+
temperature: float = 0.0,
|
|
35
|
+
vad: bool = True,
|
|
36
|
+
threads: int = 4,
|
|
37
|
+
download_root: str | Path | None = None,
|
|
38
|
+
) -> list[Transcription]:
|
|
39
|
+
"""Transcribe every utterance once, batch-style, and time each call.
|
|
40
|
+
|
|
41
|
+
Model load time is deliberately excluded from per-utterance timing (it is
|
|
42
|
+
a one-off cost, not part of what RTF measures) but the loaded model is
|
|
43
|
+
reused across all utterances, matching how a real deployment would run.
|
|
44
|
+
|
|
45
|
+
`download_root`, when given, pins exactly where the model snapshot is
|
|
46
|
+
cached — the caller (the CLI) hashes that directory as `model.sha256`, so
|
|
47
|
+
it must actually be where the weights land, not faster-whisper's default
|
|
48
|
+
shared HF cache.
|
|
49
|
+
"""
|
|
50
|
+
try:
|
|
51
|
+
from faster_whisper import WhisperModel
|
|
52
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
53
|
+
raise RuntimeError(
|
|
54
|
+
"faster-whisper is not installed; run "
|
|
55
|
+
"`pip install goesb-runner[faster-whisper]`"
|
|
56
|
+
) from exc
|
|
57
|
+
|
|
58
|
+
model = WhisperModel(
|
|
59
|
+
_resolve_model_id(model_name),
|
|
60
|
+
compute_type=quantization,
|
|
61
|
+
cpu_threads=threads,
|
|
62
|
+
download_root=str(download_root) if download_root is not None else None,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
results: list[Transcription] = []
|
|
66
|
+
for utterance in utterances:
|
|
67
|
+
start = time.perf_counter()
|
|
68
|
+
segments, _info = model.transcribe(
|
|
69
|
+
str(utterance.audio_path),
|
|
70
|
+
beam_size=beam_size,
|
|
71
|
+
temperature=temperature,
|
|
72
|
+
vad_filter=vad,
|
|
73
|
+
)
|
|
74
|
+
hypothesis_text = " ".join(segment.text.strip() for segment in segments).strip()
|
|
75
|
+
elapsed = time.perf_counter() - start
|
|
76
|
+
results.append(Transcription(
|
|
77
|
+
utterance_id=utterance.utterance_id,
|
|
78
|
+
hypothesis_text=hypothesis_text,
|
|
79
|
+
processing_time_s=elapsed,
|
|
80
|
+
))
|
|
81
|
+
return results
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@register("faster-whisper", benchmark_type="streaming")
|
|
85
|
+
def run_streaming(
|
|
86
|
+
model_name: str,
|
|
87
|
+
utterances: list[Utterance],
|
|
88
|
+
*,
|
|
89
|
+
chunk_ms: int = 1000,
|
|
90
|
+
quantization: str = "int8",
|
|
91
|
+
beam_size: int = 5,
|
|
92
|
+
temperature: float = 0.0,
|
|
93
|
+
vad: bool = True,
|
|
94
|
+
threads: int = 4,
|
|
95
|
+
download_root: str | Path | None = None,
|
|
96
|
+
) -> list[StreamTrace]:
|
|
97
|
+
"""Feed each utterance to faster-whisper in `chunk_ms` chunks, re-decoding
|
|
98
|
+
the growing buffer after every chunk (faster-whisper has no incremental
|
|
99
|
+
decoder state to resume, so "streaming" here means repeated whole-buffer
|
|
100
|
+
re-transcription — the same "local agreement" pattern used by e.g.
|
|
101
|
+
whisper_streaming). One `StreamTrace` per utterance records the resulting
|
|
102
|
+
hypothesis timeline for the streaming metric plugins to score.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
from faster_whisper import WhisperModel
|
|
106
|
+
from faster_whisper.audio import decode_audio
|
|
107
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
108
|
+
raise RuntimeError(
|
|
109
|
+
"faster-whisper is not installed; run "
|
|
110
|
+
"`pip install goesb-runner[faster-whisper]`"
|
|
111
|
+
) from exc
|
|
112
|
+
|
|
113
|
+
sample_rate = 16000
|
|
114
|
+
chunk_samples = max(1, int(chunk_ms / 1000 * sample_rate))
|
|
115
|
+
|
|
116
|
+
model = WhisperModel(
|
|
117
|
+
_resolve_model_id(model_name),
|
|
118
|
+
compute_type=quantization,
|
|
119
|
+
cpu_threads=threads,
|
|
120
|
+
download_root=str(download_root) if download_root is not None else None,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
traces: list[StreamTrace] = []
|
|
124
|
+
for utterance in utterances:
|
|
125
|
+
samples = decode_audio(str(utterance.audio_path), sampling_rate=sample_rate)
|
|
126
|
+
total_samples = len(samples)
|
|
127
|
+
audio_duration_s = total_samples / sample_rate
|
|
128
|
+
|
|
129
|
+
updates: list[PartialUpdate] = []
|
|
130
|
+
processing_time_s = 0.0
|
|
131
|
+
end = 0
|
|
132
|
+
while end < total_samples:
|
|
133
|
+
end = min(end + chunk_samples, total_samples)
|
|
134
|
+
is_last_chunk = end >= total_samples
|
|
135
|
+
chunk_end_s = end / sample_rate
|
|
136
|
+
|
|
137
|
+
start = time.perf_counter()
|
|
138
|
+
segments, _info = model.transcribe(
|
|
139
|
+
samples[:end],
|
|
140
|
+
beam_size=beam_size,
|
|
141
|
+
temperature=temperature,
|
|
142
|
+
vad_filter=vad,
|
|
143
|
+
)
|
|
144
|
+
segments = list(segments)
|
|
145
|
+
decode_wall_s = time.perf_counter() - start
|
|
146
|
+
processing_time_s += decode_wall_s
|
|
147
|
+
|
|
148
|
+
text = " ".join(segment.text.strip() for segment in segments).strip()
|
|
149
|
+
if is_last_chunk:
|
|
150
|
+
committed_word_count = len(text.split())
|
|
151
|
+
else:
|
|
152
|
+
committed_text = " ".join(s.text.strip() for s in segments[:-1]).strip()
|
|
153
|
+
committed_word_count = len(committed_text.split())
|
|
154
|
+
|
|
155
|
+
updates.append(PartialUpdate(
|
|
156
|
+
chunk_end_s=chunk_end_s,
|
|
157
|
+
emit_time_s=chunk_end_s + decode_wall_s,
|
|
158
|
+
text=text,
|
|
159
|
+
committed_word_count=committed_word_count,
|
|
160
|
+
))
|
|
161
|
+
|
|
162
|
+
traces.append(StreamTrace(
|
|
163
|
+
utterance_id=utterance.utterance_id,
|
|
164
|
+
audio_duration_s=audio_duration_s,
|
|
165
|
+
processing_time_s=processing_time_s,
|
|
166
|
+
updates=updates,
|
|
167
|
+
final_text=updates[-1].text if updates else "",
|
|
168
|
+
))
|
|
169
|
+
return traces
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""`vosk` batch runtime adapter (docs/02-architecture.md §4).
|
|
2
|
+
|
|
3
|
+
Optional dependency (`pip install goesb-runner[vosk]`) — `vosk`/`soundfile`
|
|
4
|
+
are only imported inside `run_batch`, matching the lazy-import pattern used
|
|
5
|
+
by every other adapter.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
11
|
+
import urllib.request
|
|
12
|
+
import zipfile
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from ..audio import decode_pcm
|
|
16
|
+
from ..pack import Utterance
|
|
17
|
+
from . import Transcription, register
|
|
18
|
+
|
|
19
|
+
_MODEL_URLS = {
|
|
20
|
+
"vosk-model-small-en-us-0.15":
|
|
21
|
+
"https://alphacephei.com/vosk/models/vosk-model-small-en-us-0.15.zip",
|
|
22
|
+
}
|
|
23
|
+
_SAMPLE_RATE = 16000
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _resolve_model_dir(model_name: str, download_root: Path) -> Path:
|
|
27
|
+
"""Download + unzip the pinned vosk model into `download_root` if not
|
|
28
|
+
already there. `vosk.Model(model_path=...)` needs an already-extracted
|
|
29
|
+
local directory — unlike faster-whisper's HF snapshot download, vosk has
|
|
30
|
+
no built-in fetch-by-pinned-version — so this does it explicitly: a
|
|
31
|
+
declarative fetch of a named, version-pinned asset, not arbitrary code
|
|
32
|
+
(ADR-0004)."""
|
|
33
|
+
model_dir = download_root / model_name
|
|
34
|
+
if model_dir.exists():
|
|
35
|
+
return model_dir
|
|
36
|
+
try:
|
|
37
|
+
url = _MODEL_URLS[model_name]
|
|
38
|
+
except KeyError:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"unknown vosk model {model_name!r}; known models: {sorted(_MODEL_URLS)}"
|
|
41
|
+
) from None
|
|
42
|
+
|
|
43
|
+
download_root.mkdir(parents=True, exist_ok=True)
|
|
44
|
+
zip_path = download_root / f"{model_name}.zip"
|
|
45
|
+
urllib.request.urlretrieve(url, zip_path)
|
|
46
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
47
|
+
zf.extractall(download_root)
|
|
48
|
+
zip_path.unlink()
|
|
49
|
+
return model_dir
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@register("vosk", benchmark_type="batch")
|
|
53
|
+
def run_batch(
|
|
54
|
+
model_name: str,
|
|
55
|
+
utterances: list[Utterance],
|
|
56
|
+
*,
|
|
57
|
+
quantization: str = "int8",
|
|
58
|
+
beam_size: int = 5,
|
|
59
|
+
temperature: float = 0.0,
|
|
60
|
+
vad: bool = True,
|
|
61
|
+
threads: int = 4,
|
|
62
|
+
download_root: str | Path | None = None,
|
|
63
|
+
) -> list[Transcription]:
|
|
64
|
+
"""Transcribe every utterance once, batch-style, and time each call.
|
|
65
|
+
|
|
66
|
+
`quantization`/`beam_size`/`temperature`/`vad`/`threads` are accepted for
|
|
67
|
+
call-shape parity with the other batch adapters (docs/03-roadmap.md M2
|
|
68
|
+
exit criterion: adapters swap without core changes) but unused — vosk's
|
|
69
|
+
Kaldi decoder has no equivalent tunables exposed through its Python API.
|
|
70
|
+
"""
|
|
71
|
+
try:
|
|
72
|
+
import vosk
|
|
73
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
74
|
+
raise RuntimeError(
|
|
75
|
+
"vosk is not installed; run `pip install goesb-runner[vosk]`"
|
|
76
|
+
) from exc
|
|
77
|
+
vosk.SetLogLevel(-1) # silence vosk's own stderr logging
|
|
78
|
+
|
|
79
|
+
root = Path(download_root) if download_root is not None else Path.home() / ".goesb" / "models" / model_name
|
|
80
|
+
model_dir = _resolve_model_dir(model_name, root)
|
|
81
|
+
model = vosk.Model(model_path=str(model_dir))
|
|
82
|
+
|
|
83
|
+
results: list[Transcription] = []
|
|
84
|
+
for utterance in utterances:
|
|
85
|
+
samples = decode_pcm(utterance.audio_path, dtype="int16")
|
|
86
|
+
recognizer = vosk.KaldiRecognizer(model, _SAMPLE_RATE)
|
|
87
|
+
|
|
88
|
+
start = time.perf_counter()
|
|
89
|
+
recognizer.AcceptWaveform(samples.tobytes())
|
|
90
|
+
hypothesis_text = json.loads(recognizer.FinalResult()).get("text", "")
|
|
91
|
+
elapsed = time.perf_counter() - start
|
|
92
|
+
|
|
93
|
+
results.append(Transcription(
|
|
94
|
+
utterance_id=utterance.utterance_id,
|
|
95
|
+
hypothesis_text=hypothesis_text,
|
|
96
|
+
processing_time_s=elapsed,
|
|
97
|
+
))
|
|
98
|
+
return results
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""`whisper.cpp` (via `pywhispercpp`) batch runtime adapter
|
|
2
|
+
(docs/02-architecture.md §4).
|
|
3
|
+
|
|
4
|
+
Optional dependency (`pip install goesb-runner[whisper-cpp]`) —
|
|
5
|
+
`pywhispercpp`/`soundfile` are only imported inside `run_batch`, matching the
|
|
6
|
+
lazy-import pattern used by every other adapter.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from ..audio import decode_pcm
|
|
14
|
+
from ..pack import Utterance
|
|
15
|
+
from . import Transcription, register
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _resolve_model_id(model_name: str) -> str:
|
|
19
|
+
"""Translate GOESB's runtime-agnostic model name ('whisper-base.en') into
|
|
20
|
+
the ggml model id pywhispercpp/whisper.cpp expects ('base.en') — same
|
|
21
|
+
translation role as faster_whisper._resolve_model_id; each adapter
|
|
22
|
+
carries its own runtime's naming convention rather than the profile."""
|
|
23
|
+
prefix = "whisper-"
|
|
24
|
+
return model_name.removeprefix(prefix)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@register("whisper-cpp", benchmark_type="batch")
|
|
28
|
+
def run_batch(
|
|
29
|
+
model_name: str,
|
|
30
|
+
utterances: list[Utterance],
|
|
31
|
+
*,
|
|
32
|
+
quantization: str = "int8",
|
|
33
|
+
beam_size: int = 5,
|
|
34
|
+
temperature: float = 0.0,
|
|
35
|
+
vad: bool = True,
|
|
36
|
+
threads: int = 4,
|
|
37
|
+
download_root: str | Path | None = None,
|
|
38
|
+
) -> list[Transcription]:
|
|
39
|
+
"""Transcribe every utterance once, batch-style, and time each call.
|
|
40
|
+
|
|
41
|
+
`quantization`/`beam_size`/`vad` are accepted for call-shape parity with
|
|
42
|
+
the other batch adapters (docs/03-roadmap.md M2 exit criterion: adapters
|
|
43
|
+
swap without core changes) but unused here — whisper.cpp's ggml models
|
|
44
|
+
are pre-quantized by model-file choice rather than a runtime flag,
|
|
45
|
+
`beam_size` lives under a nested `beam_search` param pywhispercpp's flat
|
|
46
|
+
`**params` doesn't set (defaults to whisper.cpp's own greedy strategy),
|
|
47
|
+
and this batch adapter doesn't chunk audio (that's M5's streaming
|
|
48
|
+
concern).
|
|
49
|
+
"""
|
|
50
|
+
try:
|
|
51
|
+
from pywhispercpp.model import Model
|
|
52
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
53
|
+
raise RuntimeError(
|
|
54
|
+
"pywhispercpp is not installed; run "
|
|
55
|
+
"`pip install goesb-runner[whisper-cpp]`"
|
|
56
|
+
) from exc
|
|
57
|
+
|
|
58
|
+
model = Model(
|
|
59
|
+
_resolve_model_id(model_name),
|
|
60
|
+
models_dir=str(download_root) if download_root is not None else None,
|
|
61
|
+
n_threads=threads,
|
|
62
|
+
temperature=temperature,
|
|
63
|
+
print_realtime=False,
|
|
64
|
+
print_progress=False,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
results: list[Transcription] = []
|
|
68
|
+
for utterance in utterances:
|
|
69
|
+
samples = decode_pcm(utterance.audio_path, dtype="float32")
|
|
70
|
+
start = time.perf_counter()
|
|
71
|
+
segments = model.transcribe(samples)
|
|
72
|
+
hypothesis_text = " ".join(segment.text.strip() for segment in segments).strip()
|
|
73
|
+
elapsed = time.perf_counter() - start
|
|
74
|
+
results.append(Transcription(
|
|
75
|
+
utterance_id=utterance.utterance_id,
|
|
76
|
+
hypothesis_text=hypothesis_text,
|
|
77
|
+
processing_time_s=elapsed,
|
|
78
|
+
))
|
|
79
|
+
return results
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Minimal audio introspection and decoding.
|
|
2
|
+
|
|
3
|
+
`flac_duration_s` is dependency-free by design: FLAC duration, parsed straight
|
|
4
|
+
from the STREAMINFO metadata block (no ffmpeg/libsndfile required just to know
|
|
5
|
+
how long a pack's clips are). `decode_pcm` is the one exception — adapters
|
|
6
|
+
that need actual PCM samples (vosk, whisper.cpp) lazy-import `soundfile` for
|
|
7
|
+
it, same optional-dependency pattern as the runtime adapters themselves, so
|
|
8
|
+
importing this module never requires it.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import struct
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def wav_duration_s(path: str | Path) -> float:
|
|
21
|
+
"""Duration in seconds, read straight from a WAV file's RIFF chunks —
|
|
22
|
+
no extra dependency, same spirit as `flac_duration_s` below. Parsed by
|
|
23
|
+
hand rather than via the stdlib `wave` module: `wave` only understands
|
|
24
|
+
integer-PCM format tag 1 and raises on anything else, but datasets
|
|
25
|
+
published as 32-bit float WAV (format tag 3 — e.g. FLEURS) are common
|
|
26
|
+
and don't need decoding just to know their length."""
|
|
27
|
+
with open(path, "rb") as f:
|
|
28
|
+
if f.read(4) != b"RIFF":
|
|
29
|
+
raise ValueError(f"not a RIFF/WAV file: {path}")
|
|
30
|
+
f.read(4) # overall chunk size, unused
|
|
31
|
+
if f.read(4) != b"WAVE":
|
|
32
|
+
raise ValueError(f"not a WAVE file: {path}")
|
|
33
|
+
|
|
34
|
+
sample_rate: int | None = None
|
|
35
|
+
block_align: int | None = None
|
|
36
|
+
while True:
|
|
37
|
+
header = f.read(8)
|
|
38
|
+
if len(header) < 8:
|
|
39
|
+
break
|
|
40
|
+
chunk_id, chunk_size = header[:4], int.from_bytes(header[4:8], "little")
|
|
41
|
+
if chunk_id == b"fmt ":
|
|
42
|
+
fmt = f.read(chunk_size)
|
|
43
|
+
# channels(2)@2, sample_rate(4)@4, block_align(2)@12
|
|
44
|
+
sample_rate = struct.unpack("<I", fmt[4:8])[0]
|
|
45
|
+
block_align = struct.unpack("<H", fmt[12:14])[0]
|
|
46
|
+
elif chunk_id == b"data":
|
|
47
|
+
if sample_rate is None or block_align is None:
|
|
48
|
+
raise ValueError(f"WAV data chunk came before fmt chunk: {path}")
|
|
49
|
+
frames = chunk_size / block_align
|
|
50
|
+
return frames / sample_rate
|
|
51
|
+
else:
|
|
52
|
+
f.seek(chunk_size, 1)
|
|
53
|
+
if chunk_size % 2:
|
|
54
|
+
f.seek(1, 1) # RIFF chunks are word-aligned; skip the pad byte
|
|
55
|
+
raise ValueError(f"WAV file has no data chunk: {path}")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def flac_duration_s(path: str | Path) -> float:
|
|
59
|
+
"""Duration in seconds, read from a FLAC file's STREAMINFO block."""
|
|
60
|
+
with open(path, "rb") as f:
|
|
61
|
+
if f.read(4) != b"fLaC":
|
|
62
|
+
raise ValueError(f"not a FLAC file: {path}")
|
|
63
|
+
while True:
|
|
64
|
+
header = f.read(4)
|
|
65
|
+
if len(header) < 4:
|
|
66
|
+
raise ValueError(f"FLAC file has no STREAMINFO block: {path}")
|
|
67
|
+
is_last = header[0] & 0x80
|
|
68
|
+
block_type = header[0] & 0x7F
|
|
69
|
+
length = int.from_bytes(header[1:4], "big")
|
|
70
|
+
body = f.read(length)
|
|
71
|
+
if block_type == 0: # STREAMINFO
|
|
72
|
+
# bytes 10..18 of STREAMINFO pack sample_rate(20)/channels(3)/
|
|
73
|
+
# bits_per_sample(5)/total_samples(36) into 64 bits.
|
|
74
|
+
packed = struct.unpack(">Q", body[10:18])[0]
|
|
75
|
+
sample_rate = packed >> 44
|
|
76
|
+
total_samples = packed & 0xF_FFFF_FFFF
|
|
77
|
+
if sample_rate == 0:
|
|
78
|
+
raise ValueError(f"FLAC STREAMINFO has zero sample rate: {path}")
|
|
79
|
+
return total_samples / sample_rate
|
|
80
|
+
if is_last:
|
|
81
|
+
raise ValueError(f"FLAC file has no STREAMINFO block: {path}")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def decode_pcm(path: str | Path, dtype: str = "int16") -> np.ndarray:
|
|
85
|
+
"""Decode an audio file to a 1-D mono PCM array at its native sample rate.
|
|
86
|
+
|
|
87
|
+
`dtype` is `"int16"` (what vosk's `AcceptWaveform` expects as raw bytes)
|
|
88
|
+
or `"float32"` (what pywhispercpp's `transcribe()` accepts directly).
|
|
89
|
+
Every GOESB pack shipped so far is already mono at the profile's target
|
|
90
|
+
rate (see each pack's `pack.yaml` `audio.sample_rate_hz`), so this does
|
|
91
|
+
not resample — a documented assumption, not a silent one.
|
|
92
|
+
"""
|
|
93
|
+
try:
|
|
94
|
+
import soundfile as sf
|
|
95
|
+
except ImportError as exc: # pragma: no cover - exercised only without an extra
|
|
96
|
+
raise RuntimeError(
|
|
97
|
+
"soundfile is not installed; run `pip install goesb-runner[vosk]` "
|
|
98
|
+
"or `pip install goesb-runner[whisper-cpp]`"
|
|
99
|
+
) from exc
|
|
100
|
+
|
|
101
|
+
samples, _sample_rate = sf.read(str(path), dtype=dtype, always_2d=False)
|
|
102
|
+
if samples.ndim > 1: # collapse stereo to mono by averaging channels
|
|
103
|
+
samples = samples.mean(axis=1).astype(samples.dtype)
|
|
104
|
+
return samples
|