fa-bench 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fa_bench-2.1.0/.fabench.env.example +153 -0
- fa_bench-2.1.0/CHANGELOG.md +407 -0
- fa_bench-2.1.0/CITATION.cff +53 -0
- fa_bench-2.1.0/CONTRIBUTING.md +89 -0
- fa_bench-2.1.0/LICENSE +202 -0
- fa_bench-2.1.0/MANIFEST.in +25 -0
- fa_bench-2.1.0/NOTICE +26 -0
- fa_bench-2.1.0/PKG-INFO +430 -0
- fa_bench-2.1.0/README.md +396 -0
- fa_bench-2.1.0/bin/fabench +38 -0
- fa_bench-2.1.0/datasets/README.md +72 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/buckeye.md +27 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/config.yaml +24 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/speakers.tsv +50 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/split/dev.list +4468 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/split/test.list +4525 -0
- fa_bench-2.1.0/datasets/languages/en/buckeye/split/train.list +13485 -0
- fa_bench-2.1.0/datasets/languages/en/config.yaml +24 -0
- fa_bench-2.1.0/datasets/languages/en/timit/config.yaml +33 -0
- fa_bench-2.1.0/datasets/languages/en/timit/split/README.md +129 -0
- fa_bench-2.1.0/datasets/languages/en/timit/split/core_test.list +196 -0
- fa_bench-2.1.0/datasets/languages/en/timit/split/dev.list +406 -0
- fa_bench-2.1.0/datasets/languages/en/timit/split/train.list +3699 -0
- fa_bench-2.1.0/datasets/languages/en/timit/timit.md +35 -0
- fa_bench-2.1.0/datasets/prep/README.md +236 -0
- fa_bench-2.1.0/datasets/prep/augment.sh +92 -0
- fa_bench-2.1.0/datasets/prep/compare_reference.py +37 -0
- fa_bench-2.1.0/datasets/prep/ingest.sh +41 -0
- fa_bench-2.1.0/datasets/prep/reference_audio_202610.sha256 +2764 -0
- fa_bench-2.1.0/evals/README.md +348 -0
- fa_bench-2.1.0/evals/__init__.py +2 -0
- fa_bench-2.1.0/evals/aligners/bfa/config.yaml +14 -0
- fa_bench-2.1.0/evals/aligners/bfa/download_and_install.sh +34 -0
- fa_bench-2.1.0/evals/aligners/bfa/requirements.lock +92 -0
- fa_bench-2.1.0/evals/aligners/bfa/worker.py +91 -0
- fa_bench-2.1.0/evals/aligners/charsiu/config.yaml +16 -0
- fa_bench-2.1.0/evals/aligners/charsiu/download_and_install.sh +59 -0
- fa_bench-2.1.0/evals/aligners/charsiu/requirements.lock +98 -0
- fa_bench-2.1.0/evals/aligners/charsiu/worker.py +47 -0
- fa_bench-2.1.0/evals/aligners/crisperwhisper_fa/config.yaml +26 -0
- fa_bench-2.1.0/evals/aligners/elevenlabs_fa/config.yaml +35 -0
- fa_bench-2.1.0/evals/aligners/falcon/config.yaml +20 -0
- fa_bench-2.1.0/evals/aligners/falcon/install.sh +30 -0
- fa_bench-2.1.0/evals/aligners/falcon/worker.py +94 -0
- fa_bench-2.1.0/evals/aligners/maps/config.yaml +28 -0
- fa_bench-2.1.0/evals/aligners/maps/download_and_install.sh +63 -0
- fa_bench-2.1.0/evals/aligners/maps/requirements.lock +48 -0
- fa_bench-2.1.0/evals/aligners/mfa/config.yaml +13 -0
- fa_bench-2.1.0/evals/aligners/mfa/download_and_install.sh +40 -0
- fa_bench-2.1.0/evals/aligners/mfa/exps/arpa_wide/config.yaml +18 -0
- fa_bench-2.1.0/evals/aligners/mfa/exps/english_mfa/config.yaml +16 -0
- fa_bench-2.1.0/evals/aligners/mfa/exps/english_mfa_wide/config.yaml +18 -0
- fa_bench-2.1.0/evals/aligners/mfa/v2.0.6/config.yaml +37 -0
- fa_bench-2.1.0/evals/aligners/mfa/v2.0.6/download_and_install.sh +77 -0
- fa_bench-2.1.0/evals/aligners/mms_fa/config.yaml +17 -0
- fa_bench-2.1.0/evals/aligners/nemo_fa/config.yaml +60 -0
- fa_bench-2.1.0/evals/aligners/nemo_fa/exps/conformer/config.yaml +44 -0
- fa_bench-2.1.0/evals/aligners/nemo_fa/worker.py +186 -0
- fa_bench-2.1.0/evals/aligners/neufa/config.yaml +46 -0
- fa_bench-2.1.0/evals/aligners/neufa/download_and_install.sh +35 -0
- fa_bench-2.1.0/evals/aligners/neufa/worker.py +147 -0
- fa_bench-2.1.0/evals/aligners/nyra_fa/config.yaml +19 -0
- fa_bench-2.1.0/evals/aligners/nyra_fa/download_and_install.sh +25 -0
- fa_bench-2.1.0/evals/aligners/nyra_fa/worker.py +48 -0
- fa_bench-2.1.0/evals/aligners/olign/config.yaml +16 -0
- fa_bench-2.1.0/evals/aligners/olign_v1/config.yaml +34 -0
- fa_bench-2.1.0/evals/aligners/pocketsphinx/config.yaml +23 -0
- fa_bench-2.1.0/evals/aligners/pocketsphinx/download_and_install.sh +50 -0
- fa_bench-2.1.0/evals/aligners/pocketsphinx/requirements.lock +20 -0
- fa_bench-2.1.0/evals/aligners/pocketsphinx/worker.py +133 -0
- fa_bench-2.1.0/evals/aligners/qwen3_fa/config.yaml +34 -0
- fa_bench-2.1.0/evals/aligners/qwen3_fa/download_and_install.sh +39 -0
- fa_bench-2.1.0/evals/aligners/qwen3_fa/requirements.lock +108 -0
- fa_bench-2.1.0/evals/aligners/qwen3_fa/worker.py +72 -0
- fa_bench-2.1.0/evals/aligners/stable_ts/config.yaml +27 -0
- fa_bench-2.1.0/evals/aligners/stable_ts/download_and_install.sh +55 -0
- fa_bench-2.1.0/evals/aligners/stable_ts/requirements.lock +40 -0
- fa_bench-2.1.0/evals/aligners/stable_ts/worker.py +73 -0
- fa_bench-2.1.0/evals/aligners/torchaudio_fa/config.yaml +37 -0
- fa_bench-2.1.0/evals/aligners/torchaudio_fa/download_and_install.sh +48 -0
- fa_bench-2.1.0/evals/aligners/torchaudio_fa/worker.py +218 -0
- fa_bench-2.1.0/evals/aligners/unity2/config.yaml +16 -0
- fa_bench-2.1.0/evals/aligners/unity2/worker.py +82 -0
- fa_bench-2.1.0/evals/aligners/whisperx/config.yaml +20 -0
- fa_bench-2.1.0/evals/aligners/whisperx/download_and_install.sh +34 -0
- fa_bench-2.1.0/evals/aligners/whisperx/requirements.lock +116 -0
- fa_bench-2.1.0/evals/aligners/whisperx/worker.py +106 -0
- fa_bench-2.1.0/evals/audit_cascades.py +168 -0
- fa_bench-2.1.0/evals/config.yaml +35 -0
- fa_bench-2.1.0/evals/env.sh +84 -0
- fa_bench-2.1.0/evals/gen_cascade_configs.py +145 -0
- fa_bench-2.1.0/evals/gen_config.py +230 -0
- fa_bench-2.1.0/evals/gen_noisy_configs.py +162 -0
- fa_bench-2.1.0/evals/launch_cascades.sh +42 -0
- fa_bench-2.1.0/evals/overnight.sh +77 -0
- fa_bench-2.1.0/evals/parse_options.sh +68 -0
- fa_bench-2.1.0/evals/run_cascade_cells.sh +63 -0
- fa_bench-2.1.0/evals/run_cpu_cells.sh +27 -0
- fa_bench-2.1.0/evals/run_evals.sh +193 -0
- fa_bench-2.1.0/evals/run_evals_parallel.sh +57 -0
- fa_bench-2.1.0/evals/run_noisy_evals.sh +157 -0
- fa_bench-2.1.0/evals/systems.py +383 -0
- fa_bench-2.1.0/evals/timestamp_asrs/assemblyai/config.yaml +51 -0
- fa_bench-2.1.0/evals/timestamp_asrs/aws_transcribe/config.yaml +128 -0
- fa_bench-2.1.0/evals/timestamp_asrs/azure_speech/config.yaml +108 -0
- fa_bench-2.1.0/evals/timestamp_asrs/bfa_on_qwen3asr/config.yaml +22 -0
- fa_bench-2.1.0/evals/timestamp_asrs/charsiu_on_qwen3asr/config.yaml +24 -0
- fa_bench-2.1.0/evals/timestamp_asrs/cloud_check.py +231 -0
- fa_bench-2.1.0/evals/timestamp_asrs/cloud_cost.py +313 -0
- fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/config.yaml +27 -0
- fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/download_and_install.sh +47 -0
- fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/requirements.lock +75 -0
- fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/worker.py +99 -0
- fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper_fa_on_qwen3asr/config.yaml +25 -0
- fa_bench-2.1.0/evals/timestamp_asrs/deepgram/config.yaml +45 -0
- fa_bench-2.1.0/evals/timestamp_asrs/deepgram/exps/nova2/config.yaml +44 -0
- fa_bench-2.1.0/evals/timestamp_asrs/elevenlabs/config.yaml +39 -0
- fa_bench-2.1.0/evals/timestamp_asrs/google_stt/config.yaml +41 -0
- fa_bench-2.1.0/evals/timestamp_asrs/google_stt/exps/chirp2/config.yaml +101 -0
- fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/config.yaml +29 -0
- fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/download_and_install.sh +32 -0
- fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/requirements.observed +59 -0
- fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/worker.py +106 -0
- fa_bench-2.1.0/evals/timestamp_asrs/ibm/config.yaml +63 -0
- fa_bench-2.1.0/evals/timestamp_asrs/maps_on_qwen3asr/config.yaml +22 -0
- fa_bench-2.1.0/evals/timestamp_asrs/mfa2_on_qwen3asr/config.yaml +27 -0
- fa_bench-2.1.0/evals/timestamp_asrs/mfa3_on_parakeettdt/config.yaml +26 -0
- fa_bench-2.1.0/evals/timestamp_asrs/mfa3_on_qwen3asr/config.yaml +24 -0
- fa_bench-2.1.0/evals/timestamp_asrs/mms_fa_on_qwen3asr/config.yaml +27 -0
- fa_bench-2.1.0/evals/timestamp_asrs/nemo_fa_conformer_on_qwen3asr/config.yaml +32 -0
- fa_bench-2.1.0/evals/timestamp_asrs/nemo_fa_on_qwen3asr/config.yaml +31 -0
- fa_bench-2.1.0/evals/timestamp_asrs/olign_on_chirp2/config.yaml +26 -0
- fa_bench-2.1.0/evals/timestamp_asrs/olign_on_parakeettdt/config.yaml +26 -0
- fa_bench-2.1.0/evals/timestamp_asrs/olign_on_qwen3asr/config.yaml +24 -0
- fa_bench-2.1.0/evals/timestamp_asrs/olign_v1_asr/config.yaml +32 -0
- fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/config.yaml +39 -0
- fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/download_and_install.sh +52 -0
- fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/requirements.observed +204 -0
- fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/worker.py +56 -0
- fa_bench-2.1.0/evals/timestamp_asrs/pocketsphinx_on_qwen3asr/config.yaml +24 -0
- fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/config.yaml +19 -0
- fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/download_and_install.sh +30 -0
- fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/worker.py +141 -0
- fa_bench-2.1.0/evals/timestamp_asrs/speechmatics/config.yaml +42 -0
- fa_bench-2.1.0/evals/timestamp_asrs/stable_ts_on_qwen3asr/config.yaml +26 -0
- fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/config.yaml +15 -0
- fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/download_and_install.sh +13 -0
- fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/worker.py +94 -0
- fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_fa_on_qwen3asr/config.yaml +29 -0
- fa_bench-2.1.0/evals/timestamp_asrs/unity2_on_qwen3asr/config.yaml +27 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisper3/config.yaml +17 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisper3/download_and_install.sh +17 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisper3/worker.py +62 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisper_ts/config.yaml +17 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisper_ts/worker.py +49 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/config.yaml +27 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/download_and_install.sh +21 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/worker.py +101 -0
- fa_bench-2.1.0/evals/timestamp_asrs/whisperx_on_qwen3asr/config.yaml +25 -0
- fa_bench-2.1.0/fa_bench.egg-info/PKG-INFO +430 -0
- fa_bench-2.1.0/fa_bench.egg-info/SOURCES.txt +310 -0
- fa_bench-2.1.0/fa_bench.egg-info/dependency_links.txt +1 -0
- fa_bench-2.1.0/fa_bench.egg-info/entry_points.txt +2 -0
- fa_bench-2.1.0/fa_bench.egg-info/requires.txt +15 -0
- fa_bench-2.1.0/fa_bench.egg-info/top_level.txt +1 -0
- fa_bench-2.1.0/fabench/__init__.py +11 -0
- fa_bench-2.1.0/fabench/__main__.py +9 -0
- fa_bench-2.1.0/fabench/aligners/__init__.py +96 -0
- fa_bench-2.1.0/fabench/aligners/base.py +110 -0
- fa_bench-2.1.0/fabench/aligners/bfa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/bfa/adapter.py +32 -0
- fa_bench-2.1.0/fabench/aligners/charsiu/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/charsiu/adapter.py +30 -0
- fa_bench-2.1.0/fabench/aligners/crisperwhisper_fa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/crisperwhisper_fa/adapter.py +38 -0
- fa_bench-2.1.0/fabench/aligners/elevenlabs_fa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/elevenlabs_fa/adapter.py +29 -0
- fa_bench-2.1.0/fabench/aligners/falcon/__init__.py +6 -0
- fa_bench-2.1.0/fabench/aligners/falcon/adapter.py +79 -0
- fa_bench-2.1.0/fabench/aligners/maps/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/maps/adapter.py +171 -0
- fa_bench-2.1.0/fabench/aligners/mfa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/mfa/adapter.py +191 -0
- fa_bench-2.1.0/fabench/aligners/nemo_fa/__init__.py +6 -0
- fa_bench-2.1.0/fabench/aligners/nemo_fa/adapter.py +47 -0
- fa_bench-2.1.0/fabench/aligners/neufa/__init__.py +6 -0
- fa_bench-2.1.0/fabench/aligners/neufa/adapter.py +37 -0
- fa_bench-2.1.0/fabench/aligners/nyra_fa/__init__.py +6 -0
- fa_bench-2.1.0/fabench/aligners/nyra_fa/adapter.py +27 -0
- fa_bench-2.1.0/fabench/aligners/olign/__init__.py +20 -0
- fa_bench-2.1.0/fabench/aligners/olign/adapter.py +236 -0
- fa_bench-2.1.0/fabench/aligners/olign_v1/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/olign_v1/adapter.py +27 -0
- fa_bench-2.1.0/fabench/aligners/pocketsphinx/__init__.py +8 -0
- fa_bench-2.1.0/fabench/aligners/pocketsphinx/adapter.py +24 -0
- fa_bench-2.1.0/fabench/aligners/qwen3_fa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/qwen3_fa/adapter.py +160 -0
- fa_bench-2.1.0/fabench/aligners/relabel.py +122 -0
- fa_bench-2.1.0/fabench/aligners/runner.py +327 -0
- fa_bench-2.1.0/fabench/aligners/stable_ts/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/stable_ts/adapter.py +41 -0
- fa_bench-2.1.0/fabench/aligners/subprocess_aligner.py +245 -0
- fa_bench-2.1.0/fabench/aligners/torchaudio_fa/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/torchaudio_fa/adapter.py +64 -0
- fa_bench-2.1.0/fabench/aligners/unity2/__init__.py +6 -0
- fa_bench-2.1.0/fabench/aligners/unity2/adapter.py +32 -0
- fa_bench-2.1.0/fabench/aligners/whisperx/__init__.py +7 -0
- fa_bench-2.1.0/fabench/aligners/whisperx/adapter.py +31 -0
- fa_bench-2.1.0/fabench/audio.py +148 -0
- fa_bench-2.1.0/fabench/cli.py +275 -0
- fa_bench-2.1.0/fabench/config.py +618 -0
- fa_bench-2.1.0/fabench/config.yaml +58 -0
- fa_bench-2.1.0/fabench/dataprep/__init__.py +18 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/__init__.py +171 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/acquire.py +131 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/__init__.py +11 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/__init__.py +17 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/download.py +55 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/processor.py +588 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/__init__.py +11 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/download.py +44 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/processor.py +84 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/__init__.py +17 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/download.py +51 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/processor.py +241 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/export_refs.py +138 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/plausibility.py +84 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/runner.py +88 -0
- fa_bench-2.1.0/fabench/dataprep/datasets/textgrid.py +44 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/__init__.py +10 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/datadirs.py +114 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/COPYING +364 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/PROVENANCE.md +76 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/augment_data_dir.py +303 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/data_dir_manipulation_lib.py +17 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/make_musan.py +178 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/make_musan.sh +71 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/reverberate_data_dir.py +687 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/libs/__init__.py +11 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/libs/common.py +498 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/make_noisy.py +346 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/make_noisy.sh +167 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/manifest.py +102 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/materialise.py +73 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/prepare.py +237 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/shadow_root.py +251 -0
- fa_bench-2.1.0/fabench/dataprep/noisemix/utt2dur.py +60 -0
- fa_bench-2.1.0/fabench/envfile.py +102 -0
- fa_bench-2.1.0/fabench/gates.py +261 -0
- fa_bench-2.1.0/fabench/init.py +318 -0
- fa_bench-2.1.0/fabench/noise/__init__.py +2 -0
- fa_bench-2.1.0/fabench/noise/config.yaml +34 -0
- fa_bench-2.1.0/fabench/noise/manifest.py +112 -0
- fa_bench-2.1.0/fabench/noise/mixer.py +149 -0
- fa_bench-2.1.0/fabench/noise/musan.py +165 -0
- fa_bench-2.1.0/fabench/noise/provider.py +63 -0
- fa_bench-2.1.0/fabench/noise/runner.py +90 -0
- fa_bench-2.1.0/fabench/noise/synth.py +44 -0
- fa_bench-2.1.0/fabench/normalize/__init__.py +93 -0
- fa_bench-2.1.0/fabench/normalize/check.py +52 -0
- fa_bench-2.1.0/fabench/normalize/en/__init__.py +91 -0
- fa_bench-2.1.0/fabench/normalize/en/canonical.py +89 -0
- fa_bench-2.1.0/fabench/normalize/en/config.yaml +12 -0
- fa_bench-2.1.0/fabench/normalize/en/maps.py +208 -0
- fa_bench-2.1.0/fabench/paths.py +211 -0
- fa_bench-2.1.0/fabench/pipeline.py +121 -0
- fa_bench-2.1.0/fabench/report/__init__.py +2 -0
- fa_bench-2.1.0/fabench/report/runner.py +76 -0
- fa_bench-2.1.0/fabench/schema.py +314 -0
- fa_bench-2.1.0/fabench/score/__init__.py +2 -0
- fa_bench-2.1.0/fabench/score/aggregate.py +159 -0
- fa_bench-2.1.0/fabench/score/boundary.py +125 -0
- fa_bench-2.1.0/fabench/score/config.yaml +29 -0
- fa_bench-2.1.0/fabench/score/core.py +363 -0
- fa_bench-2.1.0/fabench/score/matched.py +206 -0
- fa_bench-2.1.0/fabench/score/runner.py +250 -0
- fa_bench-2.1.0/fabench/score/segmentation.py +205 -0
- fa_bench-2.1.0/fabench/score/word.py +129 -0
- fa_bench-2.1.0/fabench/selftest.py +231 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/__init__.py +41 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/base.py +82 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/cloud/__init__.py +37 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/cloud/adapter.py +746 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/cloud/awssig.py +159 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/cloud/http.py +232 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/cloud/providers.py +1264 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/crisperwhisper/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/crisperwhisper/adapter.py +27 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/granite_speech_plus/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/granite_speech_plus/adapter.py +28 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/parakeet_tdt/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/parakeet_tdt/adapter.py +25 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/qwen3_asr/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/qwen3_asr/adapter.py +30 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/subprocess_asr.py +45 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/torchaudio_asr/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/torchaudio_asr/adapter.py +25 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisper3/__init__.py +7 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisper3/adapter.py +26 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisper_ts/__init__.py +6 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisper_ts/adapter.py +27 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisperx_asr/__init__.py +6 -0
- fa_bench-2.1.0/fabench/timestamp_asrs/whisperx_asr/adapter.py +28 -0
- fa_bench-2.1.0/pyproject.toml +156 -0
- fa_bench-2.1.0/run_all.sh +163 -0
- fa_bench-2.1.0/setup.cfg +4 -0
- fa_bench-2.1.0/tests/repro/202610/headline_sha256.json +1208 -0
- fa_bench-2.1.0/tests/repro/202610/inputs.json +4658 -0
- fa_bench-2.1.0/tests/repro/test_records_202610.py +345 -0
- fa_bench-2.1.0/tools/build_wav_reverberate.sh +113 -0
- fa_bench-2.1.0/tools/ci.sh +92 -0
- fa_bench-2.1.0/tools/verify_binary.sh +55 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# FA-Bench — this machine's settings. Copy to `.fabench.env` (gitignored):
|
|
2
|
+
#
|
|
3
|
+
# cp .fabench.env.example .fabench.env
|
|
4
|
+
#
|
|
5
|
+
# Both worlds read that file: the `fabench` CLI applies it before loading a
|
|
6
|
+
# config, and `evals/env.sh` sources it for the shell drivers. KEY=value lines
|
|
7
|
+
# are valid shell, which is why one file can serve both.
|
|
8
|
+
#
|
|
9
|
+
# An already-exported variable ALWAYS wins over this file, so a one-off
|
|
10
|
+
# FABENCH_THREADS=8 ./evals/run_evals.sh mfa
|
|
11
|
+
# overrides without editing anything.
|
|
12
|
+
#
|
|
13
|
+
# WHAT BELONGS HERE: what the MACHINE is. What the RUN is — which corpora, on
|
|
14
|
+
# which subset, which systems, how scored — needs no file at all: it composes
|
|
15
|
+
# from the tree that owns it (datasets/<lang>/config.yaml, evals/config.yaml,
|
|
16
|
+
# fabench/*/config.yaml). This is the one file `fabench init` writes.
|
|
17
|
+
|
|
18
|
+
# --- staged corpora -------------------------------------------------------
|
|
19
|
+
# Licensed and never downloaded: stage them yourself, then say where they are.
|
|
20
|
+
# `fabench init` fills these in; a run config naming a root would win over them.
|
|
21
|
+
#FABENCH_TIMIT_ROOT=/data/speech/TIMIT
|
|
22
|
+
#FABENCH_BUCKEYE_ROOT=/data/speech/Buckeye
|
|
23
|
+
|
|
24
|
+
# --- where bulk output goes -----------------------------------------------
|
|
25
|
+
# work_dir holds mixes and caches and gets large; results holds leaderboards.
|
|
26
|
+
#FABENCH_WORK_DIR=/fast/scratch/fabench
|
|
27
|
+
# Defaults to summary/local -- YOUR runs, gitignored. summary/ is the sweep's
|
|
28
|
+
# script output (also gitignored). Published numbers live on the leaderboard,
|
|
29
|
+
# which a run never writes.
|
|
30
|
+
#FABENCH_RESULTS_DIR=/data/fabench-results
|
|
31
|
+
|
|
32
|
+
# --- commercial ASR credentials (track 2, one step) -----------------------
|
|
33
|
+
# Only needed to run deepgram, assemblyai, elevenlabs or google_stt. Nothing
|
|
34
|
+
# reaches a paid endpoint without one of these set, and the adapters read the
|
|
35
|
+
# environment only -- never a tracked config -- so a key cannot reach the
|
|
36
|
+
# history. Check one with:
|
|
37
|
+
# evals/timestamp_asrs/cloud_check.py deepgram
|
|
38
|
+
# Responses are cached under evals/timestamp_asrs/<tool>/cache/, so a rescore
|
|
39
|
+
# or a restart costs nothing.
|
|
40
|
+
#DEEPGRAM_STT_API_KEY=
|
|
41
|
+
#ASSEMBLYAI_STT_API_KEY=
|
|
42
|
+
#ELEVENLABS_STT_API_KEY=
|
|
43
|
+
# Google takes either an API key (v1, the default path) or an OAuth bearer
|
|
44
|
+
# token (required if the recipe sets project_id, i.e. v2). With neither set the
|
|
45
|
+
# adapter runs `gcloud auth print-access-token`, so an authenticated machine
|
|
46
|
+
# needs nothing here.
|
|
47
|
+
#GOOGLE_STT_API_KEY=
|
|
48
|
+
#GOOGLE_STT_ACCESS_TOKEN=
|
|
49
|
+
# Google Cloud project for the Chirp 2 recipe (the v2 API); your own project.
|
|
50
|
+
#GOOGLE_STT_PROJECT_ID=
|
|
51
|
+
#SPEECHMATICS_STT_API_KEY=
|
|
52
|
+
# IBM Watson needs a key AND the endpoint for your own service instance, which
|
|
53
|
+
# is account-specific and so does not belong in a tracked recipe. The bare
|
|
54
|
+
# regional URL also authenticates with an IAM key; naming the instance says
|
|
55
|
+
# which service the numbers were billed to.
|
|
56
|
+
#IBM_STT_API_KEY=
|
|
57
|
+
#IBM_STT_URL=https://api.us-south.speech-to-text.watson.cloud.ibm.com/instances/<instance-id>
|
|
58
|
+
|
|
59
|
+
# Amazon Transcribe. FOUR values, not one: the signature covers the whole
|
|
60
|
+
# request, so the key id and the secret are both part of it, and batch
|
|
61
|
+
# Transcribe reads its audio from S3 rather than taking it in the request. The
|
|
62
|
+
# bucket is staging only and each object is deleted as soon as its transcript
|
|
63
|
+
# returns. It MUST be in the same region as the endpoint.
|
|
64
|
+
#AWS_ACCESS_KEY_ID=
|
|
65
|
+
#AWS_SECRET_ACCESS_KEY=
|
|
66
|
+
#AWS_SESSION_TOKEN= # only for temporary credentials
|
|
67
|
+
#AWS_REGION=us-west-1
|
|
68
|
+
#AWS_S3_BUCKET=
|
|
69
|
+
|
|
70
|
+
# Azure AI Speech, fast transcription. The portal issues TWO interchangeable
|
|
71
|
+
# keys per resource so one can rotate while the other serves; either name
|
|
72
|
+
# works and the first one set wins. The endpoint is on the same Keys and
|
|
73
|
+
# Endpoint page and is NOT optional: a multi-service resource refuses the
|
|
74
|
+
# regional gateway and answers only on its own custom subdomain.
|
|
75
|
+
#AZURE_STT_API_KEY1=
|
|
76
|
+
#AZURE_STT_API_KEY2=
|
|
77
|
+
#AZURE_SPEECH_ENDPOINT=https://<resource>.cognitiveservices.azure.com
|
|
78
|
+
#AZURE_SPEECH_REGION= # instead of the endpoint, for a single-service resource
|
|
79
|
+
|
|
80
|
+
# --- repairing a damaged sweep --------------------------------------------
|
|
81
|
+
# A noisy cell holding fewer records than its split is re-run rather than
|
|
82
|
+
# skipped, but only below this fraction. Short is not always damaged: a cascade
|
|
83
|
+
# drops an utterance its upstream ASR heard as nothing, and NeMo-FA 80 ms drops
|
|
84
|
+
# 32 short Buckeye utterances by design, both landing at 93-100%. A cell cut off
|
|
85
|
+
# by an expired token sat at 19%. Default 0.90.
|
|
86
|
+
#FABENCH_REDO_BELOW=0.90
|
|
87
|
+
|
|
88
|
+
# --- how the box is used (read by evals/env.sh) ---------------------------
|
|
89
|
+
# Which GPUs a sweep may use. UNSET means all of them, which is correct on
|
|
90
|
+
# every machine; set it only to pin a sweep to specific cards. Set it EMPTY
|
|
91
|
+
# (`FABENCH_CUDA_DEVICES=`) to expose none and run on CPU.
|
|
92
|
+
#FABENCH_CUDA_DEVICES=0,1
|
|
93
|
+
# BLAS/OMP threads per aligner process. 1 keeps a multi-tool sweep from
|
|
94
|
+
# thrashing (measured 40x throughput loss unpinned); raise it when running a
|
|
95
|
+
# single tool.
|
|
96
|
+
#FABENCH_THREADS=1
|
|
97
|
+
# The Python every driver runs (run_evals.sh, augment.sh and the rest).
|
|
98
|
+
# Unset means the project's .venv/bin/python. The drivers read it before this
|
|
99
|
+
# file is loaded, so export it in the shell instead of setting it here.
|
|
100
|
+
#FABENCH_PY=/path/to/venv/bin/python
|
|
101
|
+
|
|
102
|
+
# --- external tool locations ----------------------------------------------
|
|
103
|
+
# MFA and the kalpy scoring bridge shell into a micromamba env.
|
|
104
|
+
#FABENCH_MICROMAMBA=$HOME/micromamba/bin/micromamba
|
|
105
|
+
#FABENCH_MAMBA_ROOT=$HOME/micromamba
|
|
106
|
+
# Research checkouts that are not pip-installable.
|
|
107
|
+
#FABENCH_MAPS_REPO=tools/MAPS
|
|
108
|
+
#FABENCH_MAPS_PYTHON=tools/maps_env/bin/python
|
|
109
|
+
#FABENCH_NEUFA_REPO=tools/NeuFA
|
|
110
|
+
|
|
111
|
+
# --- noise augmentation (fabench prepare, datasets/prep/augment.sh) --------
|
|
112
|
+
# The PARENT of the per-corpus trees, each holding noisy/<type>/<utt>.wav --
|
|
113
|
+
# i.e. <root>/TIMIT/noisy/babble/... So this is usually the directory the
|
|
114
|
+
# staged corpora themselves sit in, not a separate one.
|
|
115
|
+
#FABENCH_NOISY_ROOT=/data/speech/english
|
|
116
|
+
# Where the per-condition shadow roots are built (one dir per corpus x type).
|
|
117
|
+
# These are symlink trees, so they are small; the audio they point at is not.
|
|
118
|
+
# `fabench prepare` builds them under <out>/shadow.
|
|
119
|
+
#FABENCH_SHADOW=data/shadow
|
|
120
|
+
# A dir holding data/golden/<split>/wav.scp for make_noisy.py, when it runs on
|
|
121
|
+
# its own. `fabench prepare --out <dir>` writes one at <dir>/ref.
|
|
122
|
+
#REF=/path/to/ref
|
|
123
|
+
# MUSAN and the RIR corpus, read by fabench prepare and make_noisy.py.
|
|
124
|
+
#MUSAN=/data/speech/audio/musan
|
|
125
|
+
#RIRS=/data/speech/audio/RIRS_NOISES
|
|
126
|
+
# Kaldi's wav-reverberate, with wav-to-duration beside it, for fabench prepare.
|
|
127
|
+
# tools/build_wav_reverberate.sh builds both. Unset means the one on PATH.
|
|
128
|
+
#FABENCH_WAV_REVERBERATE=tools/kaldi/src/featbin/wav-reverberate
|
|
129
|
+
|
|
130
|
+
# --- optional -------------------------------------------------------------
|
|
131
|
+
# A run config to use when --config is omitted. There is no magic filename:
|
|
132
|
+
# without this (or --config) a run uses the composed defaults.
|
|
133
|
+
#FABENCH_CONFIG=/path/to/my.yaml
|
|
134
|
+
# Fixtures for tests that otherwise self-skip.
|
|
135
|
+
#FABENCH_TEST_LJ_WAV=/data/LJSpeech-1.1/wavs/LJ001-0002.wav
|
|
136
|
+
# The frozen inputs behind the October 2026 snapshot, for tests/repro (the .tar.zst or the
|
|
137
|
+
# directory it was extracted into). Local only and never uploaded, because the
|
|
138
|
+
# hyps carry the licensed transcripts. The test reads the environment, so export
|
|
139
|
+
# it in the shell. FABENCH_REPRO_JOBS sets how many cells score at once.
|
|
140
|
+
#FABENCH_REPRO_INPUTS=/path/to/inputs_202610.tar.zst
|
|
141
|
+
#FABENCH_REPRO_JOBS=16
|
|
142
|
+
# The map from the recipe names those frozen outputs carry to the published ones,
|
|
143
|
+
# a JSON object kept with them. The repro's cell checks skip without it.
|
|
144
|
+
#FABENCH_PUBLIC_NAMES=/path/to/public_names.json
|
|
145
|
+
|
|
146
|
+
# Restrict a sweep to specific cells, ';'-separated as "<corpus> <subset>".
|
|
147
|
+
# Unset means the full grid. Set it to split one sweep across several GPUs by
|
|
148
|
+
# launching one run_evals.sh per cell:
|
|
149
|
+
# FABENCH_CELLS="buckeye test" FABENCH_DEVICE=cuda:3 ./evals/run_evals.sh qwen3_asr
|
|
150
|
+
# FABENCH_CELLS="timit core_test;buckeye test"
|
|
151
|
+
|
|
152
|
+
# Where HuggingFace models are cached for the tool workers (default ~/.cache/huggingface).
|
|
153
|
+
#HF_HOME=
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Every release of FA-Bench, newest first. A benchmark's numbers move when its
|
|
4
|
+
scoring does, so each release lists first the changes that move published
|
|
5
|
+
numbers, and names the snapshot it was scored with. Two snapshots can
|
|
6
|
+
be compared only when no such change lies between them.
|
|
7
|
+
|
|
8
|
+
## [2.1.0] - 2026-11-10
|
|
9
|
+
|
|
10
|
+
No change moves a headline number of the October 2026 snapshot, and
|
|
11
|
+
`tests/repro/` still rebuilds every one of them. The records snapshot is still
|
|
12
|
+
October 2026, and the two new rows were measured in November 2026 and are on
|
|
13
|
+
the leaderboard. This release adds two systems and the options to run MFA with
|
|
14
|
+
either of its English models.
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- Granite Speech 4.1 Plus as a Track 2 row, contributed by IBM. The open model
|
|
19
|
+
`ibm-granite/granite-speech-4.1-2b-plus` returns word times with its
|
|
20
|
+
transcript, so it is scored as shipped. Recipe
|
|
21
|
+
`evals/timestamp_asrs/granite_speech_plus/`. FA-Bench ran all 20 cells again
|
|
22
|
+
on its own hardware, and each came within 0.012 F1 of IBM's run.
|
|
23
|
+
- The nyra forced aligner from nyra labs as a Track 1 row, on the word tier.
|
|
24
|
+
It puts WavLM features under a triphone GMM-HMM and reports a confidence per
|
|
25
|
+
word. Recipe `evals/aligners/nyra_fa/`. The package is MIT, and the model
|
|
26
|
+
`nyralabs/nyra_forced_aligner_en` is under the nyra health Non-Commercial
|
|
27
|
+
Research License, which the licence table in `evals/README.md` records.
|
|
28
|
+
- The MFA adapter takes `beam` and `retry_beam` in a recipe and passes them to
|
|
29
|
+
`mfa align`. With neither set MFA keeps its defaults of 10 and 40, so the
|
|
30
|
+
`mfa` recipe and its numbers are unchanged.
|
|
31
|
+
- Three MFA variant recipes under `evals/aligners/mfa/exps/`. They run
|
|
32
|
+
`english_us_arpa` at a beam of 400 and 4000, and `english_mfa` at the
|
|
33
|
+
default beam and at the wide one. The leaderboard row stays the `mfa`
|
|
34
|
+
recipe, `english_us_arpa` at the default beam, which is the stronger model
|
|
35
|
+
on TIMIT and Buckeye. The `english_mfa` recipes score the word tier only,
|
|
36
|
+
since that model uses MFA's own phone set.
|
|
37
|
+
- `evals/aligners/mfa/download_and_install.sh` also fetches `english_mfa`
|
|
38
|
+
v3.1.0.
|
|
39
|
+
- `fabench/aligners/mfa/README.md` has a Models and beams section. It says
|
|
40
|
+
which model the leaderboard runs and what the other settings change.
|
|
41
|
+
|
|
42
|
+
### Changed
|
|
43
|
+
|
|
44
|
+
- Every CI job runs with a read-only token, so a pull request from a fork can
|
|
45
|
+
read the repository and nothing more.
|
|
46
|
+
- The README lists Granite Speech 4.1 Plus and the nyra forced aligner among
|
|
47
|
+
the systems scored, which now number 33, and `evals/README.md` carries their
|
|
48
|
+
licence rows.
|
|
49
|
+
- The README names the nyra speech timing benchmark under Related benchmarks
|
|
50
|
+
and says how its Buckeye protocol differs.
|
|
51
|
+
|
|
52
|
+
## [2.0.0] - 2026-10-28
|
|
53
|
+
|
|
54
|
+
FA-Bench is licensed under the Apache License 2.0 from this release. Releases
|
|
55
|
+
1.0.0 to 1.1.3 were published under PolyForm Noncommercial 1.0.0 and are no
|
|
56
|
+
longer distributed, and 2.0.0 reproduces every headline number of the October
|
|
57
|
+
2026 snapshot. The major number marks that boundary, and this release also
|
|
58
|
+
removes public code, which no minor release would do. FA-Bench now ships the
|
|
59
|
+
scorer, the system adapters and the data loaders. The analysis metrics, the
|
|
60
|
+
full report, the result tables and the code that builds them leave with
|
|
61
|
+
`fabench.analyze`, and the numbers are published on the leaderboard. No change
|
|
62
|
+
moves a headline number of the October 2026 snapshot, and `tests/repro/`
|
|
63
|
+
rebuilds every one of them from the frozen October system outputs.
|
|
64
|
+
|
|
65
|
+
### Licence
|
|
66
|
+
|
|
67
|
+
- `LICENSE` is the Apache License 2.0. `NOTICE` names Olewave and the
|
|
68
|
+
third-party material in the tree, and `CITATION.cff` cites the paper,
|
|
69
|
+
arXiv 2609.32396. Every source file carries a two-line SPDX header.
|
|
70
|
+
- `CONTRIBUTING.md` takes contributions under the project licence.
|
|
71
|
+
- The distribution is named `fa-bench`. The import name stays `fabench`. The
|
|
72
|
+
wheel and the sdist carry every licence and notice file.
|
|
73
|
+
|
|
74
|
+
### Added
|
|
75
|
+
|
|
76
|
+
- Word timestamp columns, `wts_*`. Every reference word owns a start and an
|
|
77
|
+
end. A timestamp is a hit when the system returned the same word, matched by
|
|
78
|
+
label, and placed that timestamp within the tolerance. Recall divides by two
|
|
79
|
+
per reference word and precision by two per system word, so a recognition
|
|
80
|
+
error costs only the word it hit. `wts_f1` leads the word tier from this
|
|
81
|
+
release, and `wts_start_mae_ms` and `wts_end_mae_ms` report the two edges
|
|
82
|
+
apart. Every column this release keeps is unchanged.
|
|
83
|
+
- `fabench prepare` builds the four degraded conditions from your staged
|
|
84
|
+
corpora. It writes the Kaldi data directories the noise recipe reads from
|
|
85
|
+
the tracked split lists, so the recipe no longer needs files from outside
|
|
86
|
+
the repository, and it records a sha256 manifest of every input and output.
|
|
87
|
+
Each Buckeye recording is one line of those directories, since the noise is
|
|
88
|
+
drawn and mixed per recording. Its noise draws match the published
|
|
89
|
+
conditions line for line. See `datasets/prep/README.md` for how close the
|
|
90
|
+
audio comes.
|
|
91
|
+
- `datasets/prep/reference_audio_202610.sha256` holds the sha256 of every
|
|
92
|
+
degraded audio file of the published conditions, 2,764 in all, and
|
|
93
|
+
`datasets/prep/compare_reference.py` checks a local build against it.
|
|
94
|
+
`datasets/prep/README.md` says what an SNR means in these conditions.
|
|
95
|
+
- A cloud recipe can set `data_opt_out`. Deepgram then sends `mip_opt_out`
|
|
96
|
+
and IBM its learning opt-out header, and both recipes turn it on, since the
|
|
97
|
+
corpora's audio is licensed. The option stays outside the response cache
|
|
98
|
+
key, so every cached response and every published number stays as it was.
|
|
99
|
+
- `tests/repro/test_records_202610.py` rebuilds the 40 cells of the October
|
|
100
|
+
2026 snapshot from frozen system outputs and compares every headline row
|
|
101
|
+
with its sha256 in `tests/repro/202610/headline_sha256.json`, so the check
|
|
102
|
+
is exact to the last digit and the numbers stay off the repository. It runs only when
|
|
103
|
+
`FABENCH_REPRO_INPUTS` names those inputs, which stay on the machine that
|
|
104
|
+
holds them.
|
|
105
|
+
- `score_pair` and `aggregate` take an `extras` list, which FA-Bench itself
|
|
106
|
+
never fills. Each extra sees the alignments a pair was scored on, or a
|
|
107
|
+
finished row with the scores behind it, so a further metric can be computed
|
|
108
|
+
without aligning twice. `runner.score_utts` returns the scores of a run.
|
|
109
|
+
- `evals/systems.py` holds the display name, family, pipeline and caveat flag
|
|
110
|
+
of every system, for the leaderboard and the repro test alike.
|
|
111
|
+
- `FABENCH_PY` names the Python every driver runs.
|
|
112
|
+
|
|
113
|
+
### Changed
|
|
114
|
+
|
|
115
|
+
- FA-Bench needs Python 3.11 or newer. The GitHub pipeline runs the tests and
|
|
116
|
+
the self-test on 3.11 and 3.12.
|
|
117
|
+
- Each leaderboard row holds the headline columns at 20 ms and the utterance
|
|
118
|
+
counts `n_utts`, `n_absent` and `n_empty`. On the word tier these are
|
|
119
|
+
`wts_f1`, `wts_precision`, `wts_recall`, `wts_mae_ms`, `wts_start_mae_ms`
|
|
120
|
+
and `wts_end_mae_ms` with their counts, `wbe_ms` with `n_word_boundaries`,
|
|
121
|
+
`wbnd_f1_all_20ms` with its denominators, and `wer`. On the phone tier they
|
|
122
|
+
are `mae_ms` with `n_boundaries`, `bnd_f1_all_20ms` with its denominators,
|
|
123
|
+
and `per`. Each is computed as before, and every one the October records
|
|
124
|
+
hold equals its October value.
|
|
125
|
+
- `fabench report` writes a short `report.md` of those columns, and
|
|
126
|
+
`per_type.parquet` is no longer written.
|
|
127
|
+
- The self-test and gate checks that read an analysis metric read a headline
|
|
128
|
+
one now, each with an answer fixed in advance.
|
|
129
|
+
- `run_all.sh` scores each tool into its own cell and no longer pools a
|
|
130
|
+
rescore into `summary/`.
|
|
131
|
+
- `fabench run` fetches MUSAN only when a requested condition draws from it.
|
|
132
|
+
A clean run no longer downloads 11 GB.
|
|
133
|
+
- The MUSAN download and unpacking are rewritten. The tarball still lands only
|
|
134
|
+
when its size matches, and the unpacked tree still appears only when it is
|
|
135
|
+
complete, now from a staging directory that is always removed.
|
|
136
|
+
- `tools/build_wav_reverberate.sh` builds from a fresh Kaldi checkout again,
|
|
137
|
+
and builds `wav-to-duration` too.
|
|
138
|
+
- The shadow step writes its WAV files with ffmpeg's bit-exact flags, so their
|
|
139
|
+
headers carry no ffmpeg version and two builds from the same audio compare
|
|
140
|
+
byte for byte.
|
|
141
|
+
- The README banner carries no Exif or C2PA metadata. Its image data is
|
|
142
|
+
unchanged.
|
|
143
|
+
- The paper is linked on arXiv, and its PDFs are no longer in the repository.
|
|
144
|
+
- Examples and comments no longer quote corpus text.
|
|
145
|
+
- The README's leaderboard badge opens the live leaderboard at
|
|
146
|
+
https://olewave.com/en/leaderboard/fa-bench in place of the records page.
|
|
147
|
+
|
|
148
|
+
- Olign 1.0 given audio alone is classed one-step, as every commercial ASR
|
|
149
|
+
with timestamps is, since its words and times come back from one request.
|
|
150
|
+
|
|
151
|
+
### Removed
|
|
152
|
+
|
|
153
|
+
- The cross-vendor analysis tools, `fabench/analyze/`, the eight `evals/`
|
|
154
|
+
scripts that read every system's per-utterance output side by side, and
|
|
155
|
+
`evals/gen_paper_tables.py`. They are Olewave's own work and are no longer
|
|
156
|
+
part of this repository.
|
|
157
|
+
- The analysis metrics, now part of Olewave's analysis work with the two
|
|
158
|
+
items after this one. They are every column at 10, 25, 50 and 100 ms, the
|
|
159
|
+
time-only boundary detection scores with over-segmentation and R-value, and
|
|
160
|
+
every boundary class other than `all`. Tolerance accuracy, median and
|
|
161
|
+
signed error and the bootstrap intervals go too. So do the S, D and I
|
|
162
|
+
percentages, onset and offset MAE, ARR and insertion rate, the error tail
|
|
163
|
+
share and the common-matched MAE. The wrong-hit share, the label variants of
|
|
164
|
+
the boundary F1, calibration, latency, RTF, coverage, the speaker count and
|
|
165
|
+
the underpowered flag complete the set. `fabench/metrics/` and
|
|
166
|
+
`fabench/score/{calibration,latency,recall}.py` go with them.
|
|
167
|
+
- The full report. `fabench/report/tables.py` and `curves.py` built its
|
|
168
|
+
per-type panel, degradation curves and calibration panel, and
|
|
169
|
+
`per_type.parquet` fed it.
|
|
170
|
+
- The code that builds the records pages, `evals/update_public_tables.py`,
|
|
171
|
+
`publish_records.py`, `report_parse.py`, `gen_tex_tables.py`,
|
|
172
|
+
`gen_provenance.py` with its cache, and `rescore_all.sh`.
|
|
173
|
+
- The result tables of every snapshot under `records/`, August, September
|
|
174
|
+
and October 2026, with their `Details.md` pages. They are part of Olewave's
|
|
175
|
+
analysis work, and the numbers are on the leaderboard.
|
|
176
|
+
`records/README.md` lists the snapshots and describes the corpora and the
|
|
177
|
+
four conditions.
|
|
178
|
+
- `docs/metrics/word_timestamp.{EN,CN}.md`, which compared named systems
|
|
179
|
+
under the word timestamp view. The README's Metrics section defines every
|
|
180
|
+
`wts_*` column.
|
|
181
|
+
- The MFA-paper scoring protocol, `scoring.protocol: mfa_paper`, with
|
|
182
|
+
`fabench/score/mfa_paper/` and its vendored mapping files. FA-Bench scores
|
|
183
|
+
under its own protocol only, so the `scoring_protocol` column goes too.
|
|
184
|
+
- The Olign test that needed a live service. The Olign adapter speaks REST
|
|
185
|
+
to the hosted endpoint, and a `transport` other than `rest` is an error.
|
|
186
|
+
- Two Olign 0.9 adapter options that no recipe set.
|
|
187
|
+
- Kaldi's unused `steps/libs/nnet3/` scripts.
|
|
188
|
+
- The GitHub workflows other than `ci.yml`. CI on GitHub runs `ci.yml` alone,
|
|
189
|
+
with lint, the tests and the self-test.
|
|
190
|
+
|
|
191
|
+
### Fixed
|
|
192
|
+
|
|
193
|
+
- The cloud batch loop flags an exhausted quota on the worker thread that
|
|
194
|
+
hit it, so no further call leaves before the main thread notices. The CI
|
|
195
|
+
run on the 1.1.3 merge failed on the test for this, four calls past the
|
|
196
|
+
wall on a loaded runner, and the test now expects exactly one.
|
|
197
|
+
- The noise recipe's manifest reads the corpus splits only. It used to read
|
|
198
|
+
the noise corpora's own data directories as one more split.
|
|
199
|
+
|
|
200
|
+
## [1.1.3] - 2026-10-12
|
|
201
|
+
|
|
202
|
+
Records snapshot `records/202610/`, scored with this release. No change
|
|
203
|
+
moves a number published in `records/202609/`, which stands as scored for
|
|
204
|
+
1.1.0. The October snapshot adds PocketSphinx, Olign 1.0 and one cascade,
|
|
205
|
+
and its Olign rows are described below.
|
|
206
|
+
|
|
207
|
+
### Added
|
|
208
|
+
|
|
209
|
+
- Olign 1.0 as its own row beside Olign 0.9, on both tracks. On Track 1 it
|
|
210
|
+
is given the reference transcript. On Track 2 it recognizes the words
|
|
211
|
+
itself and then aligns them, so the tables call it two-step, as they do
|
|
212
|
+
Qwen3-ASR. It is reached through Olewave's job API at
|
|
213
|
+
`https://api.olewave.com/olign/v1`, with the key in `OLIGN_API_KEY`. The
|
|
214
|
+
call lives with the cloud providers, which cache every response, retry
|
|
215
|
+
and run requests concurrently. Recipes `evals/aligners/olign_v1/` and
|
|
216
|
+
`evals/timestamp_asrs/olign_v1_asr/`.
|
|
217
|
+
- PocketSphinx as a Track 1 row, the HMM family's one member not built on
|
|
218
|
+
Kaldi. It aligns the way `pocketsphinx align` does, with the bundled
|
|
219
|
+
en-us model and CMUdict, and g2p_en for words outside the dictionary. It
|
|
220
|
+
is built from a pinned main-branch commit rather than the PyPI wheel,
|
|
221
|
+
because the alignment fixes that matter here landed after v5.1.1. An
|
|
222
|
+
utterance that fails on the default beams is aligned again with the beams
|
|
223
|
+
open. The utterances that still fail also fail upstream's own CLI, and
|
|
224
|
+
they count against the row as every system's failures do. See
|
|
225
|
+
`fabench/aligners/pocketsphinx/README.md`.
|
|
226
|
+
- Qwen3-ASR → PocketSphinx as a Track 2 cascade, generated the way the
|
|
227
|
+
other cascades are.
|
|
228
|
+
- The records snapshot `records/202610/`, with 19 forced aligners on
|
|
229
|
+
Track 1 and 34 systems on Track 2. `records/latest` and the README point
|
|
230
|
+
at it. ElevenLabs Scribe v2 has 12 of its 20 cells, since the account ran
|
|
231
|
+
out of credit before the dev split under the four degradations, and
|
|
232
|
+
ElevenLabs forced alignment is still unpublished for the same reason.
|
|
233
|
+
- Each Olign 0.9 hyp record keeps the `versions` object of the response
|
|
234
|
+
that made it, for responses fetched since 2026-10-11.
|
|
235
|
+
- A leaderboard badge at the top of the README, ahead of the paper badge,
|
|
236
|
+
that opens the current snapshot's results page.
|
|
237
|
+
|
|
238
|
+
### Changed
|
|
239
|
+
|
|
240
|
+
- The three cascades into Olign 0.9 on Track 2 were rerun against the
|
|
241
|
+
Olign 0.9 service that produced its Track 1 row, so every Olign 0.9 row
|
|
242
|
+
comes from the same service. Under noise their word MAE improves by 0.8
|
|
243
|
+
to 1.6 ms and their F1 at 20 ms by 0.02 to 0.05. Clean cells move by
|
|
244
|
+
0.3 ms or less.
|
|
245
|
+
- A cloud recipe's `base_url` is part of its response cache key, so a
|
|
246
|
+
recipe aimed at one endpoint cannot replay the cached answers of another.
|
|
247
|
+
No tracked recipe sets `base_url`, so no cached response is orphaned.
|
|
248
|
+
- The Olign live test runs only when it is configured.
|
|
249
|
+
|
|
250
|
+
### Fixed
|
|
251
|
+
|
|
252
|
+
- The cloud adapter drops a phone whose end is not after its start, as the
|
|
253
|
+
Olign 0.9 adapter has since it was written. Olign 1.0 is the only cloud
|
|
254
|
+
row with a phone tier and it had kept them, so the two versions of one
|
|
255
|
+
system were scored under different rules. On Buckeye dev under additive
|
|
256
|
+
noise the difference was close to 10 ms of phone MAE. No published number
|
|
257
|
+
moves, since Olign 1.0 first appears in `records/202610/`.
|
|
258
|
+
- `evals/gen_cascade_configs.py` writes a base tool's relative paths
|
|
259
|
+
relative to the cascade's directory, and decides what is a path by the
|
|
260
|
+
key, as the composer does. A bare `venv: venv` was left pointing at
|
|
261
|
+
nothing.
|
|
262
|
+
- `evals/run_cpu_cells.sh` printed an empty tool name in every status line.
|
|
263
|
+
|
|
264
|
+
## [1.1.2] - 2026-10-08
|
|
265
|
+
|
|
266
|
+
No change moves a published number. The records snapshot is still
|
|
267
|
+
`records/202609/`, as scored for 1.1.0.
|
|
268
|
+
|
|
269
|
+
### Added
|
|
270
|
+
|
|
271
|
+
- ElevenLabs forced alignment as a Track 1 row, `elevenlabs_fa`. Each
|
|
272
|
+
utterance goes to ElevenLabs' `/v1/forced-alignment` endpoint with its
|
|
273
|
+
reference transcript, and the row is scored on word boundaries beside MFA
|
|
274
|
+
and Olign. It ships with `enabled: false` and no published numbers, because
|
|
275
|
+
every request bills the account. See
|
|
276
|
+
`fabench/aligners/elevenlabs_fa/README.md`.
|
|
277
|
+
- `cloud_check.py` takes `--text`, the words spoken in `--audio`, so it can
|
|
278
|
+
check a forced aligner. `cloud_cost.py` prices the new row and counts its
|
|
279
|
+
cache in `--actual`.
|
|
280
|
+
- A one-column, double-spaced copy of the paper,
|
|
281
|
+
`docs/paper_for_reviewer.pdf`, which the README's paper links now
|
|
282
|
+
open. Its tables are set larger and write every number with its leading
|
|
283
|
+
zero, 0.40 where the paper has .40, and its Fig. 1 is larger.
|
|
284
|
+
`evals/gen_paper_tables.py` writes its tables in the same run as the
|
|
285
|
+
paper's, so the two cannot disagree. The two-column `docs/paper.pdf`
|
|
286
|
+
stays, and a section at the end of the README links both and the paper
|
|
287
|
+
on arXiv.
|
|
288
|
+
|
|
289
|
+
### Changed
|
|
290
|
+
|
|
291
|
+
- The commercial cloud adapter can send the reference transcript with the
|
|
292
|
+
audio. When it does, the transcript is part of the response cache key. The
|
|
293
|
+
Track 2 rows send none, so their keys and cached responses are unchanged.
|
|
294
|
+
|
|
295
|
+
### Fixed
|
|
296
|
+
|
|
297
|
+
- Olign is named Olign 0.9, the version that produced its numbers. The records
|
|
298
|
+
in both snapshots, the Olign adapter's docs and the caption of the paper's
|
|
299
|
+
word table had called it Olign 1.0. Only the name changes.
|
|
300
|
+
- `docs/paper.pdf` is the revised paper. Its title is now "FA-Bench: A
|
|
301
|
+
Benchmark for Phone- and Word-Level Timestamp Accuracy in Forced Alignment
|
|
302
|
+
and ASR on Clean and Noisy Speech", its word table names Olign 0.9, and the
|
|
303
|
+
abstract and the introduction open with a new first sentence. The text
|
|
304
|
+
gives Qwen3-FA's word F1 at 20 ms as 0.40, the value in its table, where
|
|
305
|
+
it said 0.39. The tables are unchanged.
|
|
306
|
+
|
|
307
|
+
## [1.1.1] - 2026-09-29
|
|
308
|
+
|
|
309
|
+
No change moves a published number. The records snapshot is still
|
|
310
|
+
`records/202609/`, as scored for 1.1.0.
|
|
311
|
+
|
|
312
|
+
### Added
|
|
313
|
+
|
|
314
|
+
- The paper as `docs/paper.pdf`, linked from the top of the README by a PDF
|
|
315
|
+
icon beside the title, a badge and a sentence in the introduction.
|
|
316
|
+
- `tools/ci.sh` runs the three CI jobs on a clean export of HEAD, in a
|
|
317
|
+
Python 3.12 environment holding only what CI installs. GitHub and GitLab
|
|
318
|
+
call the same script, and ruff is pinned to 0.16.9.
|
|
319
|
+
|
|
320
|
+
### Fixed
|
|
321
|
+
|
|
322
|
+
- The Olign adapter reaches the hosted endpoint. The endpoint rejects
|
|
323
|
+
Python's default User-Agent, so the adapter names itself, and it sends the
|
|
324
|
+
service token from `.fabench.env` when one is set. The default endpoint is `https://api.olewave.com/olign/v0.9`.
|
|
325
|
+
- Two workers kept an import they no longer use, which failed the lint job.
|
|
326
|
+
- `evals/README.md` again says the top-level `run_all.sh` uses the same
|
|
327
|
+
option parser. The house-style rewrite had dropped it.
|
|
328
|
+
|
|
329
|
+
## [1.1.0] - 2026-09-26
|
|
330
|
+
|
|
331
|
+
Records snapshot `records/202609/`. This release was first tagged on
|
|
332
|
+
2026-09-25 and re-cut on 2026-09-26 with further scorer fixes, so numbers
|
|
333
|
+
taken from the first tag are superseded.
|
|
334
|
+
|
|
335
|
+
### Changes that move published numbers
|
|
336
|
+
|
|
337
|
+
- **Word MAE counts each boundary once.** A boundary shared by two words was
|
|
338
|
+
counted twice, which weighted the easy interior double. MAE rose by 3 to
|
|
339
|
+
5 ms.
|
|
340
|
+
- **Word F1 is label-checked.** A word boundary counts only when the words on
|
|
341
|
+
both sides match the reference and its time is within the tolerance, the
|
|
342
|
+
two utterance edges included. It paired boundaries by time alone.
|
|
343
|
+
- **Phone boundary F1 checks the phone labels on both sides**, as the word
|
|
344
|
+
tier does.
|
|
345
|
+
- **An utterance a system returns nothing for is charged.** Whether it wrote
|
|
346
|
+
no record or an empty one, every boundary of that utterance counts against
|
|
347
|
+
F1 recall and every word and phone in it as a deletion in WER and PER. MAE
|
|
348
|
+
is still over what came back. Before, such utterances left the evaluation,
|
|
349
|
+
so a system that failed on hard material scored as if it had not. MFA, for
|
|
350
|
+
one, returns nothing for up to 354 of Buckeye test's 4,513 utterances under
|
|
351
|
+
noise.
|
|
352
|
+
- **A one-step recognizer is scored on the words it wrote.** The eight
|
|
353
|
+
commercial endpoints had their words rewritten onto the reference spelling
|
|
354
|
+
when saved (`boats` became the reference's `boat's`), and the open
|
|
355
|
+
recognizers never had.
|
|
356
|
+
- **Word MAE matches words without regard to case**, as F1 and WER already
|
|
357
|
+
did. It moved by at most 0.14 ms.
|
|
358
|
+
- **TIMIT's degraded audio is cut to the clean length.** Each file carried
|
|
359
|
+
about 475 ms of noise after its last word, which charged a system that
|
|
360
|
+
timed words into it. The TIMIT noisy cells were re-run, apart from
|
|
361
|
+
ElevenLabs, which a spot check found unchanged, and NeuFA, whose output is
|
|
362
|
+
cut at the clean length.
|
|
363
|
+
- **TIMIT's SA sentences are excluded, and Buckeye is re-split** 24/8/8 by
|
|
364
|
+
speaker, balanced for sex and age.
|
|
365
|
+
- **The records' Noisy column averages unrounded values**, so it matches the
|
|
366
|
+
paper to the last digit.
|
|
367
|
+
|
|
368
|
+
### Added
|
|
369
|
+
|
|
370
|
+
- Track 2, timestamped ASR. One-step recognizers, including eight commercial
|
|
371
|
+
endpoints, and cascades that re-align a recognizer's transcript with a
|
|
372
|
+
Track 1 aligner.
|
|
373
|
+
- NeuFA, scored from output its authors produced with a checkpoint they have
|
|
374
|
+
not released.
|
|
375
|
+
- Boundary classes by position and by how many adjacent labels match.
|
|
376
|
+
- Leaderboard columns `n_absent` and `n_empty`, and an Utterance completion
|
|
377
|
+
note on every records page that says who returned nothing for how much.
|
|
378
|
+
- Records gain the version and checkpoint of every system, the commercial
|
|
379
|
+
endpoints with the model requested and the days of the calls, training data
|
|
380
|
+
and overlap for every system, and the output quirks the scores keep.
|
|
381
|
+
- `evals/publish_records.py --release <tag>`, so a snapshot names the public
|
|
382
|
+
release it was scored with.
|
|
383
|
+
|
|
384
|
+
### Changed
|
|
385
|
+
|
|
386
|
+
- Every README and the records prose are rewritten in one house style, with
|
|
387
|
+
stale facts corrected along the way. There are eight commercial endpoints,
|
|
388
|
+
eight Track 1 systems emit phones, ElevenLabs has twelve of its twenty
|
|
389
|
+
cells, and NeuFA's rows come from its authors' checkpoint.
|
|
390
|
+
|
|
391
|
+
### Fixed
|
|
392
|
+
|
|
393
|
+
- The MFA adapter finds the micromamba its install script puts under
|
|
394
|
+
`mamba_root`, and `fabench run` reads a recipe's relative paths against the
|
|
395
|
+
recipe folder.
|
|
396
|
+
- `max_calls: 0` means no API calls, a cache-only replay. It read as no
|
|
397
|
+
limit.
|
|
398
|
+
- The noise scripts exit non-zero when nothing was built or found.
|
|
399
|
+
- Audio files are written atomically.
|
|
400
|
+
- Machine-specific paths and account ids left the tracked recipes. They come
|
|
401
|
+
from `.fabench.env`.
|
|
402
|
+
|
|
403
|
+
## [1.0.0] - 2026-08-17
|
|
404
|
+
|
|
405
|
+
First public release. Track 1 forced alignment on TIMIT and Buckeye, word
|
|
406
|
+
and phone tiers, clean audio and four degradations. Records snapshot
|
|
407
|
+
`records/202608/`.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you publish results produced with this software, please cite the FA-Bench paper."
|
|
3
|
+
title: "FA-Bench"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- name: "Olewave, LLC"
|
|
7
|
+
website: "https://olewave.com"
|
|
8
|
+
version: "2.1.0"
|
|
9
|
+
license: Apache-2.0
|
|
10
|
+
repository-code: "https://github.com/olewave/fa-bench"
|
|
11
|
+
url: "https://olewave.com/en/leaderboard/fa-bench"
|
|
12
|
+
keywords:
|
|
13
|
+
- forced alignment
|
|
14
|
+
- speech recognition
|
|
15
|
+
- word timestamps
|
|
16
|
+
- phone boundaries
|
|
17
|
+
- benchmark
|
|
18
|
+
preferred-citation:
|
|
19
|
+
type: article
|
|
20
|
+
title: "FA-Bench: A Benchmark for Phone- and Word-Level Timestamp Accuracy in Forced Alignment and ASR on Clean and Noisy Speech"
|
|
21
|
+
authors:
|
|
22
|
+
- given-names: Wei
|
|
23
|
+
family-names: Chu
|
|
24
|
+
- given-names: Yuanzhe
|
|
25
|
+
family-names: Dong
|
|
26
|
+
- given-names: Ke
|
|
27
|
+
family-names: Tan
|
|
28
|
+
- given-names: Dong
|
|
29
|
+
family-names: Han
|
|
30
|
+
- given-names: Yichao
|
|
31
|
+
family-names: Zhou
|
|
32
|
+
- given-names: Ruchao
|
|
33
|
+
family-names: Fan
|
|
34
|
+
- given-names: Bingshen
|
|
35
|
+
family-names: Mu
|
|
36
|
+
- given-names: Jingbei
|
|
37
|
+
family-names: Li
|
|
38
|
+
- given-names: Vishwas
|
|
39
|
+
family-names: Shetty
|
|
40
|
+
- given-names: Sarthak
|
|
41
|
+
family-names: Bisht
|
|
42
|
+
- given-names: Ziyue
|
|
43
|
+
family-names: Qiu
|
|
44
|
+
- given-names: Massa
|
|
45
|
+
family-names: Baali
|
|
46
|
+
- given-names: Rita
|
|
47
|
+
family-names: Singh
|
|
48
|
+
- given-names: Bhiksha
|
|
49
|
+
family-names: Raj
|
|
50
|
+
year: 2026
|
|
51
|
+
journal: "arXiv preprint arXiv:2609.32396"
|
|
52
|
+
doi: "10.48550/arXiv.2609.32396"
|
|
53
|
+
url: "https://arxiv.org/abs/2609.32396"
|