fa-bench 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (312) hide show
  1. fa_bench-2.1.0/.fabench.env.example +153 -0
  2. fa_bench-2.1.0/CHANGELOG.md +407 -0
  3. fa_bench-2.1.0/CITATION.cff +53 -0
  4. fa_bench-2.1.0/CONTRIBUTING.md +89 -0
  5. fa_bench-2.1.0/LICENSE +202 -0
  6. fa_bench-2.1.0/MANIFEST.in +25 -0
  7. fa_bench-2.1.0/NOTICE +26 -0
  8. fa_bench-2.1.0/PKG-INFO +430 -0
  9. fa_bench-2.1.0/README.md +396 -0
  10. fa_bench-2.1.0/bin/fabench +38 -0
  11. fa_bench-2.1.0/datasets/README.md +72 -0
  12. fa_bench-2.1.0/datasets/languages/en/buckeye/buckeye.md +27 -0
  13. fa_bench-2.1.0/datasets/languages/en/buckeye/config.yaml +24 -0
  14. fa_bench-2.1.0/datasets/languages/en/buckeye/speakers.tsv +50 -0
  15. fa_bench-2.1.0/datasets/languages/en/buckeye/split/dev.list +4468 -0
  16. fa_bench-2.1.0/datasets/languages/en/buckeye/split/test.list +4525 -0
  17. fa_bench-2.1.0/datasets/languages/en/buckeye/split/train.list +13485 -0
  18. fa_bench-2.1.0/datasets/languages/en/config.yaml +24 -0
  19. fa_bench-2.1.0/datasets/languages/en/timit/config.yaml +33 -0
  20. fa_bench-2.1.0/datasets/languages/en/timit/split/README.md +129 -0
  21. fa_bench-2.1.0/datasets/languages/en/timit/split/core_test.list +196 -0
  22. fa_bench-2.1.0/datasets/languages/en/timit/split/dev.list +406 -0
  23. fa_bench-2.1.0/datasets/languages/en/timit/split/train.list +3699 -0
  24. fa_bench-2.1.0/datasets/languages/en/timit/timit.md +35 -0
  25. fa_bench-2.1.0/datasets/prep/README.md +236 -0
  26. fa_bench-2.1.0/datasets/prep/augment.sh +92 -0
  27. fa_bench-2.1.0/datasets/prep/compare_reference.py +37 -0
  28. fa_bench-2.1.0/datasets/prep/ingest.sh +41 -0
  29. fa_bench-2.1.0/datasets/prep/reference_audio_202610.sha256 +2764 -0
  30. fa_bench-2.1.0/evals/README.md +348 -0
  31. fa_bench-2.1.0/evals/__init__.py +2 -0
  32. fa_bench-2.1.0/evals/aligners/bfa/config.yaml +14 -0
  33. fa_bench-2.1.0/evals/aligners/bfa/download_and_install.sh +34 -0
  34. fa_bench-2.1.0/evals/aligners/bfa/requirements.lock +92 -0
  35. fa_bench-2.1.0/evals/aligners/bfa/worker.py +91 -0
  36. fa_bench-2.1.0/evals/aligners/charsiu/config.yaml +16 -0
  37. fa_bench-2.1.0/evals/aligners/charsiu/download_and_install.sh +59 -0
  38. fa_bench-2.1.0/evals/aligners/charsiu/requirements.lock +98 -0
  39. fa_bench-2.1.0/evals/aligners/charsiu/worker.py +47 -0
  40. fa_bench-2.1.0/evals/aligners/crisperwhisper_fa/config.yaml +26 -0
  41. fa_bench-2.1.0/evals/aligners/elevenlabs_fa/config.yaml +35 -0
  42. fa_bench-2.1.0/evals/aligners/falcon/config.yaml +20 -0
  43. fa_bench-2.1.0/evals/aligners/falcon/install.sh +30 -0
  44. fa_bench-2.1.0/evals/aligners/falcon/worker.py +94 -0
  45. fa_bench-2.1.0/evals/aligners/maps/config.yaml +28 -0
  46. fa_bench-2.1.0/evals/aligners/maps/download_and_install.sh +63 -0
  47. fa_bench-2.1.0/evals/aligners/maps/requirements.lock +48 -0
  48. fa_bench-2.1.0/evals/aligners/mfa/config.yaml +13 -0
  49. fa_bench-2.1.0/evals/aligners/mfa/download_and_install.sh +40 -0
  50. fa_bench-2.1.0/evals/aligners/mfa/exps/arpa_wide/config.yaml +18 -0
  51. fa_bench-2.1.0/evals/aligners/mfa/exps/english_mfa/config.yaml +16 -0
  52. fa_bench-2.1.0/evals/aligners/mfa/exps/english_mfa_wide/config.yaml +18 -0
  53. fa_bench-2.1.0/evals/aligners/mfa/v2.0.6/config.yaml +37 -0
  54. fa_bench-2.1.0/evals/aligners/mfa/v2.0.6/download_and_install.sh +77 -0
  55. fa_bench-2.1.0/evals/aligners/mms_fa/config.yaml +17 -0
  56. fa_bench-2.1.0/evals/aligners/nemo_fa/config.yaml +60 -0
  57. fa_bench-2.1.0/evals/aligners/nemo_fa/exps/conformer/config.yaml +44 -0
  58. fa_bench-2.1.0/evals/aligners/nemo_fa/worker.py +186 -0
  59. fa_bench-2.1.0/evals/aligners/neufa/config.yaml +46 -0
  60. fa_bench-2.1.0/evals/aligners/neufa/download_and_install.sh +35 -0
  61. fa_bench-2.1.0/evals/aligners/neufa/worker.py +147 -0
  62. fa_bench-2.1.0/evals/aligners/nyra_fa/config.yaml +19 -0
  63. fa_bench-2.1.0/evals/aligners/nyra_fa/download_and_install.sh +25 -0
  64. fa_bench-2.1.0/evals/aligners/nyra_fa/worker.py +48 -0
  65. fa_bench-2.1.0/evals/aligners/olign/config.yaml +16 -0
  66. fa_bench-2.1.0/evals/aligners/olign_v1/config.yaml +34 -0
  67. fa_bench-2.1.0/evals/aligners/pocketsphinx/config.yaml +23 -0
  68. fa_bench-2.1.0/evals/aligners/pocketsphinx/download_and_install.sh +50 -0
  69. fa_bench-2.1.0/evals/aligners/pocketsphinx/requirements.lock +20 -0
  70. fa_bench-2.1.0/evals/aligners/pocketsphinx/worker.py +133 -0
  71. fa_bench-2.1.0/evals/aligners/qwen3_fa/config.yaml +34 -0
  72. fa_bench-2.1.0/evals/aligners/qwen3_fa/download_and_install.sh +39 -0
  73. fa_bench-2.1.0/evals/aligners/qwen3_fa/requirements.lock +108 -0
  74. fa_bench-2.1.0/evals/aligners/qwen3_fa/worker.py +72 -0
  75. fa_bench-2.1.0/evals/aligners/stable_ts/config.yaml +27 -0
  76. fa_bench-2.1.0/evals/aligners/stable_ts/download_and_install.sh +55 -0
  77. fa_bench-2.1.0/evals/aligners/stable_ts/requirements.lock +40 -0
  78. fa_bench-2.1.0/evals/aligners/stable_ts/worker.py +73 -0
  79. fa_bench-2.1.0/evals/aligners/torchaudio_fa/config.yaml +37 -0
  80. fa_bench-2.1.0/evals/aligners/torchaudio_fa/download_and_install.sh +48 -0
  81. fa_bench-2.1.0/evals/aligners/torchaudio_fa/worker.py +218 -0
  82. fa_bench-2.1.0/evals/aligners/unity2/config.yaml +16 -0
  83. fa_bench-2.1.0/evals/aligners/unity2/worker.py +82 -0
  84. fa_bench-2.1.0/evals/aligners/whisperx/config.yaml +20 -0
  85. fa_bench-2.1.0/evals/aligners/whisperx/download_and_install.sh +34 -0
  86. fa_bench-2.1.0/evals/aligners/whisperx/requirements.lock +116 -0
  87. fa_bench-2.1.0/evals/aligners/whisperx/worker.py +106 -0
  88. fa_bench-2.1.0/evals/audit_cascades.py +168 -0
  89. fa_bench-2.1.0/evals/config.yaml +35 -0
  90. fa_bench-2.1.0/evals/env.sh +84 -0
  91. fa_bench-2.1.0/evals/gen_cascade_configs.py +145 -0
  92. fa_bench-2.1.0/evals/gen_config.py +230 -0
  93. fa_bench-2.1.0/evals/gen_noisy_configs.py +162 -0
  94. fa_bench-2.1.0/evals/launch_cascades.sh +42 -0
  95. fa_bench-2.1.0/evals/overnight.sh +77 -0
  96. fa_bench-2.1.0/evals/parse_options.sh +68 -0
  97. fa_bench-2.1.0/evals/run_cascade_cells.sh +63 -0
  98. fa_bench-2.1.0/evals/run_cpu_cells.sh +27 -0
  99. fa_bench-2.1.0/evals/run_evals.sh +193 -0
  100. fa_bench-2.1.0/evals/run_evals_parallel.sh +57 -0
  101. fa_bench-2.1.0/evals/run_noisy_evals.sh +157 -0
  102. fa_bench-2.1.0/evals/systems.py +383 -0
  103. fa_bench-2.1.0/evals/timestamp_asrs/assemblyai/config.yaml +51 -0
  104. fa_bench-2.1.0/evals/timestamp_asrs/aws_transcribe/config.yaml +128 -0
  105. fa_bench-2.1.0/evals/timestamp_asrs/azure_speech/config.yaml +108 -0
  106. fa_bench-2.1.0/evals/timestamp_asrs/bfa_on_qwen3asr/config.yaml +22 -0
  107. fa_bench-2.1.0/evals/timestamp_asrs/charsiu_on_qwen3asr/config.yaml +24 -0
  108. fa_bench-2.1.0/evals/timestamp_asrs/cloud_check.py +231 -0
  109. fa_bench-2.1.0/evals/timestamp_asrs/cloud_cost.py +313 -0
  110. fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/config.yaml +27 -0
  111. fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/download_and_install.sh +47 -0
  112. fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/requirements.lock +75 -0
  113. fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper/worker.py +99 -0
  114. fa_bench-2.1.0/evals/timestamp_asrs/crisperwhisper_fa_on_qwen3asr/config.yaml +25 -0
  115. fa_bench-2.1.0/evals/timestamp_asrs/deepgram/config.yaml +45 -0
  116. fa_bench-2.1.0/evals/timestamp_asrs/deepgram/exps/nova2/config.yaml +44 -0
  117. fa_bench-2.1.0/evals/timestamp_asrs/elevenlabs/config.yaml +39 -0
  118. fa_bench-2.1.0/evals/timestamp_asrs/google_stt/config.yaml +41 -0
  119. fa_bench-2.1.0/evals/timestamp_asrs/google_stt/exps/chirp2/config.yaml +101 -0
  120. fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/config.yaml +29 -0
  121. fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/download_and_install.sh +32 -0
  122. fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/requirements.observed +59 -0
  123. fa_bench-2.1.0/evals/timestamp_asrs/granite_speech_plus/worker.py +106 -0
  124. fa_bench-2.1.0/evals/timestamp_asrs/ibm/config.yaml +63 -0
  125. fa_bench-2.1.0/evals/timestamp_asrs/maps_on_qwen3asr/config.yaml +22 -0
  126. fa_bench-2.1.0/evals/timestamp_asrs/mfa2_on_qwen3asr/config.yaml +27 -0
  127. fa_bench-2.1.0/evals/timestamp_asrs/mfa3_on_parakeettdt/config.yaml +26 -0
  128. fa_bench-2.1.0/evals/timestamp_asrs/mfa3_on_qwen3asr/config.yaml +24 -0
  129. fa_bench-2.1.0/evals/timestamp_asrs/mms_fa_on_qwen3asr/config.yaml +27 -0
  130. fa_bench-2.1.0/evals/timestamp_asrs/nemo_fa_conformer_on_qwen3asr/config.yaml +32 -0
  131. fa_bench-2.1.0/evals/timestamp_asrs/nemo_fa_on_qwen3asr/config.yaml +31 -0
  132. fa_bench-2.1.0/evals/timestamp_asrs/olign_on_chirp2/config.yaml +26 -0
  133. fa_bench-2.1.0/evals/timestamp_asrs/olign_on_parakeettdt/config.yaml +26 -0
  134. fa_bench-2.1.0/evals/timestamp_asrs/olign_on_qwen3asr/config.yaml +24 -0
  135. fa_bench-2.1.0/evals/timestamp_asrs/olign_v1_asr/config.yaml +32 -0
  136. fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/config.yaml +39 -0
  137. fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/download_and_install.sh +52 -0
  138. fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/requirements.observed +204 -0
  139. fa_bench-2.1.0/evals/timestamp_asrs/parakeet_tdt/worker.py +56 -0
  140. fa_bench-2.1.0/evals/timestamp_asrs/pocketsphinx_on_qwen3asr/config.yaml +24 -0
  141. fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/config.yaml +19 -0
  142. fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/download_and_install.sh +30 -0
  143. fa_bench-2.1.0/evals/timestamp_asrs/qwen3_asr/worker.py +141 -0
  144. fa_bench-2.1.0/evals/timestamp_asrs/speechmatics/config.yaml +42 -0
  145. fa_bench-2.1.0/evals/timestamp_asrs/stable_ts_on_qwen3asr/config.yaml +26 -0
  146. fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/config.yaml +15 -0
  147. fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/download_and_install.sh +13 -0
  148. fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_asr/worker.py +94 -0
  149. fa_bench-2.1.0/evals/timestamp_asrs/torchaudio_fa_on_qwen3asr/config.yaml +29 -0
  150. fa_bench-2.1.0/evals/timestamp_asrs/unity2_on_qwen3asr/config.yaml +27 -0
  151. fa_bench-2.1.0/evals/timestamp_asrs/whisper3/config.yaml +17 -0
  152. fa_bench-2.1.0/evals/timestamp_asrs/whisper3/download_and_install.sh +17 -0
  153. fa_bench-2.1.0/evals/timestamp_asrs/whisper3/worker.py +62 -0
  154. fa_bench-2.1.0/evals/timestamp_asrs/whisper_ts/config.yaml +17 -0
  155. fa_bench-2.1.0/evals/timestamp_asrs/whisper_ts/worker.py +49 -0
  156. fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/config.yaml +27 -0
  157. fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/download_and_install.sh +21 -0
  158. fa_bench-2.1.0/evals/timestamp_asrs/whisperx_asr/worker.py +101 -0
  159. fa_bench-2.1.0/evals/timestamp_asrs/whisperx_on_qwen3asr/config.yaml +25 -0
  160. fa_bench-2.1.0/fa_bench.egg-info/PKG-INFO +430 -0
  161. fa_bench-2.1.0/fa_bench.egg-info/SOURCES.txt +310 -0
  162. fa_bench-2.1.0/fa_bench.egg-info/dependency_links.txt +1 -0
  163. fa_bench-2.1.0/fa_bench.egg-info/entry_points.txt +2 -0
  164. fa_bench-2.1.0/fa_bench.egg-info/requires.txt +15 -0
  165. fa_bench-2.1.0/fa_bench.egg-info/top_level.txt +1 -0
  166. fa_bench-2.1.0/fabench/__init__.py +11 -0
  167. fa_bench-2.1.0/fabench/__main__.py +9 -0
  168. fa_bench-2.1.0/fabench/aligners/__init__.py +96 -0
  169. fa_bench-2.1.0/fabench/aligners/base.py +110 -0
  170. fa_bench-2.1.0/fabench/aligners/bfa/__init__.py +7 -0
  171. fa_bench-2.1.0/fabench/aligners/bfa/adapter.py +32 -0
  172. fa_bench-2.1.0/fabench/aligners/charsiu/__init__.py +7 -0
  173. fa_bench-2.1.0/fabench/aligners/charsiu/adapter.py +30 -0
  174. fa_bench-2.1.0/fabench/aligners/crisperwhisper_fa/__init__.py +7 -0
  175. fa_bench-2.1.0/fabench/aligners/crisperwhisper_fa/adapter.py +38 -0
  176. fa_bench-2.1.0/fabench/aligners/elevenlabs_fa/__init__.py +7 -0
  177. fa_bench-2.1.0/fabench/aligners/elevenlabs_fa/adapter.py +29 -0
  178. fa_bench-2.1.0/fabench/aligners/falcon/__init__.py +6 -0
  179. fa_bench-2.1.0/fabench/aligners/falcon/adapter.py +79 -0
  180. fa_bench-2.1.0/fabench/aligners/maps/__init__.py +7 -0
  181. fa_bench-2.1.0/fabench/aligners/maps/adapter.py +171 -0
  182. fa_bench-2.1.0/fabench/aligners/mfa/__init__.py +7 -0
  183. fa_bench-2.1.0/fabench/aligners/mfa/adapter.py +191 -0
  184. fa_bench-2.1.0/fabench/aligners/nemo_fa/__init__.py +6 -0
  185. fa_bench-2.1.0/fabench/aligners/nemo_fa/adapter.py +47 -0
  186. fa_bench-2.1.0/fabench/aligners/neufa/__init__.py +6 -0
  187. fa_bench-2.1.0/fabench/aligners/neufa/adapter.py +37 -0
  188. fa_bench-2.1.0/fabench/aligners/nyra_fa/__init__.py +6 -0
  189. fa_bench-2.1.0/fabench/aligners/nyra_fa/adapter.py +27 -0
  190. fa_bench-2.1.0/fabench/aligners/olign/__init__.py +20 -0
  191. fa_bench-2.1.0/fabench/aligners/olign/adapter.py +236 -0
  192. fa_bench-2.1.0/fabench/aligners/olign_v1/__init__.py +7 -0
  193. fa_bench-2.1.0/fabench/aligners/olign_v1/adapter.py +27 -0
  194. fa_bench-2.1.0/fabench/aligners/pocketsphinx/__init__.py +8 -0
  195. fa_bench-2.1.0/fabench/aligners/pocketsphinx/adapter.py +24 -0
  196. fa_bench-2.1.0/fabench/aligners/qwen3_fa/__init__.py +7 -0
  197. fa_bench-2.1.0/fabench/aligners/qwen3_fa/adapter.py +160 -0
  198. fa_bench-2.1.0/fabench/aligners/relabel.py +122 -0
  199. fa_bench-2.1.0/fabench/aligners/runner.py +327 -0
  200. fa_bench-2.1.0/fabench/aligners/stable_ts/__init__.py +7 -0
  201. fa_bench-2.1.0/fabench/aligners/stable_ts/adapter.py +41 -0
  202. fa_bench-2.1.0/fabench/aligners/subprocess_aligner.py +245 -0
  203. fa_bench-2.1.0/fabench/aligners/torchaudio_fa/__init__.py +7 -0
  204. fa_bench-2.1.0/fabench/aligners/torchaudio_fa/adapter.py +64 -0
  205. fa_bench-2.1.0/fabench/aligners/unity2/__init__.py +6 -0
  206. fa_bench-2.1.0/fabench/aligners/unity2/adapter.py +32 -0
  207. fa_bench-2.1.0/fabench/aligners/whisperx/__init__.py +7 -0
  208. fa_bench-2.1.0/fabench/aligners/whisperx/adapter.py +31 -0
  209. fa_bench-2.1.0/fabench/audio.py +148 -0
  210. fa_bench-2.1.0/fabench/cli.py +275 -0
  211. fa_bench-2.1.0/fabench/config.py +618 -0
  212. fa_bench-2.1.0/fabench/config.yaml +58 -0
  213. fa_bench-2.1.0/fabench/dataprep/__init__.py +18 -0
  214. fa_bench-2.1.0/fabench/dataprep/datasets/__init__.py +171 -0
  215. fa_bench-2.1.0/fabench/dataprep/datasets/acquire.py +131 -0
  216. fa_bench-2.1.0/fabench/dataprep/datasets/en/__init__.py +11 -0
  217. fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/__init__.py +17 -0
  218. fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/download.py +55 -0
  219. fa_bench-2.1.0/fabench/dataprep/datasets/en/buckeye/processor.py +588 -0
  220. fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/__init__.py +11 -0
  221. fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/download.py +44 -0
  222. fa_bench-2.1.0/fabench/dataprep/datasets/en/l2arctic/processor.py +84 -0
  223. fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/__init__.py +17 -0
  224. fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/download.py +51 -0
  225. fa_bench-2.1.0/fabench/dataprep/datasets/en/timit/processor.py +241 -0
  226. fa_bench-2.1.0/fabench/dataprep/datasets/export_refs.py +138 -0
  227. fa_bench-2.1.0/fabench/dataprep/datasets/plausibility.py +84 -0
  228. fa_bench-2.1.0/fabench/dataprep/datasets/runner.py +88 -0
  229. fa_bench-2.1.0/fabench/dataprep/datasets/textgrid.py +44 -0
  230. fa_bench-2.1.0/fabench/dataprep/noisemix/__init__.py +10 -0
  231. fa_bench-2.1.0/fabench/dataprep/noisemix/datadirs.py +114 -0
  232. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/COPYING +364 -0
  233. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/PROVENANCE.md +76 -0
  234. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/augment_data_dir.py +303 -0
  235. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/data_dir_manipulation_lib.py +17 -0
  236. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/make_musan.py +178 -0
  237. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/make_musan.sh +71 -0
  238. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/data/reverberate_data_dir.py +687 -0
  239. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/libs/__init__.py +11 -0
  240. fa_bench-2.1.0/fabench/dataprep/noisemix/kaldi/steps/libs/common.py +498 -0
  241. fa_bench-2.1.0/fabench/dataprep/noisemix/make_noisy.py +346 -0
  242. fa_bench-2.1.0/fabench/dataprep/noisemix/make_noisy.sh +167 -0
  243. fa_bench-2.1.0/fabench/dataprep/noisemix/manifest.py +102 -0
  244. fa_bench-2.1.0/fabench/dataprep/noisemix/materialise.py +73 -0
  245. fa_bench-2.1.0/fabench/dataprep/noisemix/prepare.py +237 -0
  246. fa_bench-2.1.0/fabench/dataprep/noisemix/shadow_root.py +251 -0
  247. fa_bench-2.1.0/fabench/dataprep/noisemix/utt2dur.py +60 -0
  248. fa_bench-2.1.0/fabench/envfile.py +102 -0
  249. fa_bench-2.1.0/fabench/gates.py +261 -0
  250. fa_bench-2.1.0/fabench/init.py +318 -0
  251. fa_bench-2.1.0/fabench/noise/__init__.py +2 -0
  252. fa_bench-2.1.0/fabench/noise/config.yaml +34 -0
  253. fa_bench-2.1.0/fabench/noise/manifest.py +112 -0
  254. fa_bench-2.1.0/fabench/noise/mixer.py +149 -0
  255. fa_bench-2.1.0/fabench/noise/musan.py +165 -0
  256. fa_bench-2.1.0/fabench/noise/provider.py +63 -0
  257. fa_bench-2.1.0/fabench/noise/runner.py +90 -0
  258. fa_bench-2.1.0/fabench/noise/synth.py +44 -0
  259. fa_bench-2.1.0/fabench/normalize/__init__.py +93 -0
  260. fa_bench-2.1.0/fabench/normalize/check.py +52 -0
  261. fa_bench-2.1.0/fabench/normalize/en/__init__.py +91 -0
  262. fa_bench-2.1.0/fabench/normalize/en/canonical.py +89 -0
  263. fa_bench-2.1.0/fabench/normalize/en/config.yaml +12 -0
  264. fa_bench-2.1.0/fabench/normalize/en/maps.py +208 -0
  265. fa_bench-2.1.0/fabench/paths.py +211 -0
  266. fa_bench-2.1.0/fabench/pipeline.py +121 -0
  267. fa_bench-2.1.0/fabench/report/__init__.py +2 -0
  268. fa_bench-2.1.0/fabench/report/runner.py +76 -0
  269. fa_bench-2.1.0/fabench/schema.py +314 -0
  270. fa_bench-2.1.0/fabench/score/__init__.py +2 -0
  271. fa_bench-2.1.0/fabench/score/aggregate.py +159 -0
  272. fa_bench-2.1.0/fabench/score/boundary.py +125 -0
  273. fa_bench-2.1.0/fabench/score/config.yaml +29 -0
  274. fa_bench-2.1.0/fabench/score/core.py +363 -0
  275. fa_bench-2.1.0/fabench/score/matched.py +206 -0
  276. fa_bench-2.1.0/fabench/score/runner.py +250 -0
  277. fa_bench-2.1.0/fabench/score/segmentation.py +205 -0
  278. fa_bench-2.1.0/fabench/score/word.py +129 -0
  279. fa_bench-2.1.0/fabench/selftest.py +231 -0
  280. fa_bench-2.1.0/fabench/timestamp_asrs/__init__.py +41 -0
  281. fa_bench-2.1.0/fabench/timestamp_asrs/base.py +82 -0
  282. fa_bench-2.1.0/fabench/timestamp_asrs/cloud/__init__.py +37 -0
  283. fa_bench-2.1.0/fabench/timestamp_asrs/cloud/adapter.py +746 -0
  284. fa_bench-2.1.0/fabench/timestamp_asrs/cloud/awssig.py +159 -0
  285. fa_bench-2.1.0/fabench/timestamp_asrs/cloud/http.py +232 -0
  286. fa_bench-2.1.0/fabench/timestamp_asrs/cloud/providers.py +1264 -0
  287. fa_bench-2.1.0/fabench/timestamp_asrs/crisperwhisper/__init__.py +7 -0
  288. fa_bench-2.1.0/fabench/timestamp_asrs/crisperwhisper/adapter.py +27 -0
  289. fa_bench-2.1.0/fabench/timestamp_asrs/granite_speech_plus/__init__.py +7 -0
  290. fa_bench-2.1.0/fabench/timestamp_asrs/granite_speech_plus/adapter.py +28 -0
  291. fa_bench-2.1.0/fabench/timestamp_asrs/parakeet_tdt/__init__.py +7 -0
  292. fa_bench-2.1.0/fabench/timestamp_asrs/parakeet_tdt/adapter.py +25 -0
  293. fa_bench-2.1.0/fabench/timestamp_asrs/qwen3_asr/__init__.py +7 -0
  294. fa_bench-2.1.0/fabench/timestamp_asrs/qwen3_asr/adapter.py +30 -0
  295. fa_bench-2.1.0/fabench/timestamp_asrs/subprocess_asr.py +45 -0
  296. fa_bench-2.1.0/fabench/timestamp_asrs/torchaudio_asr/__init__.py +7 -0
  297. fa_bench-2.1.0/fabench/timestamp_asrs/torchaudio_asr/adapter.py +25 -0
  298. fa_bench-2.1.0/fabench/timestamp_asrs/whisper3/__init__.py +7 -0
  299. fa_bench-2.1.0/fabench/timestamp_asrs/whisper3/adapter.py +26 -0
  300. fa_bench-2.1.0/fabench/timestamp_asrs/whisper_ts/__init__.py +6 -0
  301. fa_bench-2.1.0/fabench/timestamp_asrs/whisper_ts/adapter.py +27 -0
  302. fa_bench-2.1.0/fabench/timestamp_asrs/whisperx_asr/__init__.py +6 -0
  303. fa_bench-2.1.0/fabench/timestamp_asrs/whisperx_asr/adapter.py +28 -0
  304. fa_bench-2.1.0/pyproject.toml +156 -0
  305. fa_bench-2.1.0/run_all.sh +163 -0
  306. fa_bench-2.1.0/setup.cfg +4 -0
  307. fa_bench-2.1.0/tests/repro/202610/headline_sha256.json +1208 -0
  308. fa_bench-2.1.0/tests/repro/202610/inputs.json +4658 -0
  309. fa_bench-2.1.0/tests/repro/test_records_202610.py +345 -0
  310. fa_bench-2.1.0/tools/build_wav_reverberate.sh +113 -0
  311. fa_bench-2.1.0/tools/ci.sh +92 -0
  312. fa_bench-2.1.0/tools/verify_binary.sh +55 -0
@@ -0,0 +1,153 @@
1
+ # FA-Bench — this machine's settings. Copy to `.fabench.env` (gitignored):
2
+ #
3
+ # cp .fabench.env.example .fabench.env
4
+ #
5
+ # Both worlds read that file: the `fabench` CLI applies it before loading a
6
+ # config, and `evals/env.sh` sources it for the shell drivers. KEY=value lines
7
+ # are valid shell, which is why one file can serve both.
8
+ #
9
+ # An already-exported variable ALWAYS wins over this file, so a one-off
10
+ # FABENCH_THREADS=8 ./evals/run_evals.sh mfa
11
+ # overrides without editing anything.
12
+ #
13
+ # WHAT BELONGS HERE: what the MACHINE is. What the RUN is — which corpora, on
14
+ # which subset, which systems, how scored — needs no file at all: it composes
15
+ # from the tree that owns it (datasets/<lang>/config.yaml, evals/config.yaml,
16
+ # fabench/*/config.yaml). This is the one file `fabench init` writes.
17
+
18
+ # --- staged corpora -------------------------------------------------------
19
+ # Licensed and never downloaded: stage them yourself, then say where they are.
20
+ # `fabench init` fills these in; a run config naming a root would win over them.
21
+ #FABENCH_TIMIT_ROOT=/data/speech/TIMIT
22
+ #FABENCH_BUCKEYE_ROOT=/data/speech/Buckeye
23
+
24
+ # --- where bulk output goes -----------------------------------------------
25
+ # work_dir holds mixes and caches and gets large; results holds leaderboards.
26
+ #FABENCH_WORK_DIR=/fast/scratch/fabench
27
+ # Defaults to summary/local -- YOUR runs, gitignored. summary/ is the sweep's
28
+ # script output (also gitignored). Published numbers live on the leaderboard,
29
+ # which a run never writes.
30
+ #FABENCH_RESULTS_DIR=/data/fabench-results
31
+
32
+ # --- commercial ASR credentials (track 2, one step) -----------------------
33
+ # Only needed to run deepgram, assemblyai, elevenlabs or google_stt. Nothing
34
+ # reaches a paid endpoint without one of these set, and the adapters read the
35
+ # environment only -- never a tracked config -- so a key cannot reach the
36
+ # history. Check one with:
37
+ # evals/timestamp_asrs/cloud_check.py deepgram
38
+ # Responses are cached under evals/timestamp_asrs/<tool>/cache/, so a rescore
39
+ # or a restart costs nothing.
40
+ #DEEPGRAM_STT_API_KEY=
41
+ #ASSEMBLYAI_STT_API_KEY=
42
+ #ELEVENLABS_STT_API_KEY=
43
+ # Google takes either an API key (v1, the default path) or an OAuth bearer
44
+ # token (required if the recipe sets project_id, i.e. v2). With neither set the
45
+ # adapter runs `gcloud auth print-access-token`, so an authenticated machine
46
+ # needs nothing here.
47
+ #GOOGLE_STT_API_KEY=
48
+ #GOOGLE_STT_ACCESS_TOKEN=
49
+ # Google Cloud project for the Chirp 2 recipe (the v2 API); your own project.
50
+ #GOOGLE_STT_PROJECT_ID=
51
+ #SPEECHMATICS_STT_API_KEY=
52
+ # IBM Watson needs a key AND the endpoint for your own service instance, which
53
+ # is account-specific and so does not belong in a tracked recipe. The bare
54
+ # regional URL also authenticates with an IAM key; naming the instance says
55
+ # which service the numbers were billed to.
56
+ #IBM_STT_API_KEY=
57
+ #IBM_STT_URL=https://api.us-south.speech-to-text.watson.cloud.ibm.com/instances/<instance-id>
58
+
59
+ # Amazon Transcribe. FOUR values, not one: the signature covers the whole
60
+ # request, so the key id and the secret are both part of it, and batch
61
+ # Transcribe reads its audio from S3 rather than taking it in the request. The
62
+ # bucket is staging only and each object is deleted as soon as its transcript
63
+ # returns. It MUST be in the same region as the endpoint.
64
+ #AWS_ACCESS_KEY_ID=
65
+ #AWS_SECRET_ACCESS_KEY=
66
+ #AWS_SESSION_TOKEN= # only for temporary credentials
67
+ #AWS_REGION=us-west-1
68
+ #AWS_S3_BUCKET=
69
+
70
+ # Azure AI Speech, fast transcription. The portal issues TWO interchangeable
71
+ # keys per resource so one can rotate while the other serves; either name
72
+ # works and the first one set wins. The endpoint is on the same Keys and
73
+ # Endpoint page and is NOT optional: a multi-service resource refuses the
74
+ # regional gateway and answers only on its own custom subdomain.
75
+ #AZURE_STT_API_KEY1=
76
+ #AZURE_STT_API_KEY2=
77
+ #AZURE_SPEECH_ENDPOINT=https://<resource>.cognitiveservices.azure.com
78
+ #AZURE_SPEECH_REGION= # instead of the endpoint, for a single-service resource
79
+
80
+ # --- repairing a damaged sweep --------------------------------------------
81
+ # A noisy cell holding fewer records than its split is re-run rather than
82
+ # skipped, but only below this fraction. Short is not always damaged: a cascade
83
+ # drops an utterance its upstream ASR heard as nothing, and NeMo-FA 80 ms drops
84
+ # 32 short Buckeye utterances by design, both landing at 93-100%. A cell cut off
85
+ # by an expired token sat at 19%. Default 0.90.
86
+ #FABENCH_REDO_BELOW=0.90
87
+
88
+ # --- how the box is used (read by evals/env.sh) ---------------------------
89
+ # Which GPUs a sweep may use. UNSET means all of them, which is correct on
90
+ # every machine; set it only to pin a sweep to specific cards. Set it EMPTY
91
+ # (`FABENCH_CUDA_DEVICES=`) to expose none and run on CPU.
92
+ #FABENCH_CUDA_DEVICES=0,1
93
+ # BLAS/OMP threads per aligner process. 1 keeps a multi-tool sweep from
94
+ # thrashing (measured 40x throughput loss unpinned); raise it when running a
95
+ # single tool.
96
+ #FABENCH_THREADS=1
97
+ # The Python every driver runs (run_evals.sh, augment.sh and the rest).
98
+ # Unset means the project's .venv/bin/python. The drivers read it before this
99
+ # file is loaded, so export it in the shell instead of setting it here.
100
+ #FABENCH_PY=/path/to/venv/bin/python
101
+
102
+ # --- external tool locations ----------------------------------------------
103
+ # MFA and the kalpy scoring bridge shell into a micromamba env.
104
+ #FABENCH_MICROMAMBA=$HOME/micromamba/bin/micromamba
105
+ #FABENCH_MAMBA_ROOT=$HOME/micromamba
106
+ # Research checkouts that are not pip-installable.
107
+ #FABENCH_MAPS_REPO=tools/MAPS
108
+ #FABENCH_MAPS_PYTHON=tools/maps_env/bin/python
109
+ #FABENCH_NEUFA_REPO=tools/NeuFA
110
+
111
+ # --- noise augmentation (fabench prepare, datasets/prep/augment.sh) --------
112
+ # The PARENT of the per-corpus trees, each holding noisy/<type>/<utt>.wav --
113
+ # i.e. <root>/TIMIT/noisy/babble/... So this is usually the directory the
114
+ # staged corpora themselves sit in, not a separate one.
115
+ #FABENCH_NOISY_ROOT=/data/speech/english
116
+ # Where the per-condition shadow roots are built (one dir per corpus x type).
117
+ # These are symlink trees, so they are small; the audio they point at is not.
118
+ # `fabench prepare` builds them under <out>/shadow.
119
+ #FABENCH_SHADOW=data/shadow
120
+ # A dir holding data/golden/<split>/wav.scp for make_noisy.py, when it runs on
121
+ # its own. `fabench prepare --out <dir>` writes one at <dir>/ref.
122
+ #REF=/path/to/ref
123
+ # MUSAN and the RIR corpus, read by fabench prepare and make_noisy.py.
124
+ #MUSAN=/data/speech/audio/musan
125
+ #RIRS=/data/speech/audio/RIRS_NOISES
126
+ # Kaldi's wav-reverberate, with wav-to-duration beside it, for fabench prepare.
127
+ # tools/build_wav_reverberate.sh builds both. Unset means the one on PATH.
128
+ #FABENCH_WAV_REVERBERATE=tools/kaldi/src/featbin/wav-reverberate
129
+
130
+ # --- optional -------------------------------------------------------------
131
+ # A run config to use when --config is omitted. There is no magic filename:
132
+ # without this (or --config) a run uses the composed defaults.
133
+ #FABENCH_CONFIG=/path/to/my.yaml
134
+ # Fixtures for tests that otherwise self-skip.
135
+ #FABENCH_TEST_LJ_WAV=/data/LJSpeech-1.1/wavs/LJ001-0002.wav
136
+ # The frozen inputs behind the October 2026 snapshot, for tests/repro (the .tar.zst or the
137
+ # directory it was extracted into). Local only and never uploaded, because the
138
+ # hyps carry the licensed transcripts. The test reads the environment, so export
139
+ # it in the shell. FABENCH_REPRO_JOBS sets how many cells score at once.
140
+ #FABENCH_REPRO_INPUTS=/path/to/inputs_202610.tar.zst
141
+ #FABENCH_REPRO_JOBS=16
142
+ # The map from the recipe names those frozen outputs carry to the published ones,
143
+ # a JSON object kept with them. The repro's cell checks skip without it.
144
+ #FABENCH_PUBLIC_NAMES=/path/to/public_names.json
145
+
146
+ # Restrict a sweep to specific cells, ';'-separated as "<corpus> <subset>".
147
+ # Unset means the full grid. Set it to split one sweep across several GPUs by
148
+ # launching one run_evals.sh per cell:
149
+ # FABENCH_CELLS="buckeye test" FABENCH_DEVICE=cuda:3 ./evals/run_evals.sh qwen3_asr
150
+ # FABENCH_CELLS="timit core_test;buckeye test"
151
+
152
+ # Where HuggingFace models are cached for the tool workers (default ~/.cache/huggingface).
153
+ #HF_HOME=
@@ -0,0 +1,407 @@
1
+ # Changelog
2
+
3
+ Every release of FA-Bench, newest first. A benchmark's numbers move when its
4
+ scoring does, so each release lists first the changes that move published
5
+ numbers, and names the snapshot it was scored with. Two snapshots can
6
+ be compared only when no such change lies between them.
7
+
8
+ ## [2.1.0] - 2026-11-10
9
+
10
+ No change moves a headline number of the October 2026 snapshot, and
11
+ `tests/repro/` still rebuilds every one of them. The records snapshot is still
12
+ October 2026, and the two new rows were measured in November 2026 and are on
13
+ the leaderboard. This release adds two systems and the options to run MFA with
14
+ either of its English models.
15
+
16
+ ### Added
17
+
18
+ - Granite Speech 4.1 Plus as a Track 2 row, contributed by IBM. The open model
19
+ `ibm-granite/granite-speech-4.1-2b-plus` returns word times with its
20
+ transcript, so it is scored as shipped. Recipe
21
+ `evals/timestamp_asrs/granite_speech_plus/`. FA-Bench ran all 20 cells again
22
+ on its own hardware, and each came within 0.012 F1 of IBM's run.
23
+ - The nyra forced aligner from nyra labs as a Track 1 row, on the word tier.
24
+ It puts WavLM features under a triphone GMM-HMM and reports a confidence per
25
+ word. Recipe `evals/aligners/nyra_fa/`. The package is MIT, and the model
26
+ `nyralabs/nyra_forced_aligner_en` is under the nyra health Non-Commercial
27
+ Research License, which the licence table in `evals/README.md` records.
28
+ - The MFA adapter takes `beam` and `retry_beam` in a recipe and passes them to
29
+ `mfa align`. With neither set MFA keeps its defaults of 10 and 40, so the
30
+ `mfa` recipe and its numbers are unchanged.
31
+ - Three MFA variant recipes under `evals/aligners/mfa/exps/`. They run
32
+ `english_us_arpa` at a beam of 400 and 4000, and `english_mfa` at the
33
+ default beam and at the wide one. The leaderboard row stays the `mfa`
34
+ recipe, `english_us_arpa` at the default beam, which is the stronger model
35
+ on TIMIT and Buckeye. The `english_mfa` recipes score the word tier only,
36
+ since that model uses MFA's own phone set.
37
+ - `evals/aligners/mfa/download_and_install.sh` also fetches `english_mfa`
38
+ v3.1.0.
39
+ - `fabench/aligners/mfa/README.md` has a Models and beams section. It says
40
+ which model the leaderboard runs and what the other settings change.
41
+
42
+ ### Changed
43
+
44
+ - Every CI job runs with a read-only token, so a pull request from a fork can
45
+ read the repository and nothing more.
46
+ - The README lists Granite Speech 4.1 Plus and the nyra forced aligner among
47
+ the systems scored, which now number 33, and `evals/README.md` carries their
48
+ licence rows.
49
+ - The README names the nyra speech timing benchmark under Related benchmarks
50
+ and says how its Buckeye protocol differs.
51
+
52
+ ## [2.0.0] - 2026-10-28
53
+
54
+ FA-Bench is licensed under the Apache License 2.0 from this release. Releases
55
+ 1.0.0 to 1.1.3 were published under PolyForm Noncommercial 1.0.0 and are no
56
+ longer distributed, and 2.0.0 reproduces every headline number of the October
57
+ 2026 snapshot. The major number marks that boundary, and this release also
58
+ removes public code, which no minor release would do. FA-Bench now ships the
59
+ scorer, the system adapters and the data loaders. The analysis metrics, the
60
+ full report, the result tables and the code that builds them leave with
61
+ `fabench.analyze`, and the numbers are published on the leaderboard. No change
62
+ moves a headline number of the October 2026 snapshot, and `tests/repro/`
63
+ rebuilds every one of them from the frozen October system outputs.
64
+
65
+ ### Licence
66
+
67
+ - `LICENSE` is the Apache License 2.0. `NOTICE` names Olewave and the
68
+ third-party material in the tree, and `CITATION.cff` cites the paper,
69
+ arXiv 2609.32396. Every source file carries a two-line SPDX header.
70
+ - `CONTRIBUTING.md` takes contributions under the project licence.
71
+ - The distribution is named `fa-bench`. The import name stays `fabench`. The
72
+ wheel and the sdist carry every licence and notice file.
73
+
74
+ ### Added
75
+
76
+ - Word timestamp columns, `wts_*`. Every reference word owns a start and an
77
+ end. A timestamp is a hit when the system returned the same word, matched by
78
+ label, and placed that timestamp within the tolerance. Recall divides by two
79
+ per reference word and precision by two per system word, so a recognition
80
+ error costs only the word it hit. `wts_f1` leads the word tier from this
81
+ release, and `wts_start_mae_ms` and `wts_end_mae_ms` report the two edges
82
+ apart. Every column this release keeps is unchanged.
83
+ - `fabench prepare` builds the four degraded conditions from your staged
84
+ corpora. It writes the Kaldi data directories the noise recipe reads from
85
+ the tracked split lists, so the recipe no longer needs files from outside
86
+ the repository, and it records a sha256 manifest of every input and output.
87
+ Each Buckeye recording is one line of those directories, since the noise is
88
+ drawn and mixed per recording. Its noise draws match the published
89
+ conditions line for line. See `datasets/prep/README.md` for how close the
90
+ audio comes.
91
+ - `datasets/prep/reference_audio_202610.sha256` holds the sha256 of every
92
+ degraded audio file of the published conditions, 2,764 in all, and
93
+ `datasets/prep/compare_reference.py` checks a local build against it.
94
+ `datasets/prep/README.md` says what an SNR means in these conditions.
95
+ - A cloud recipe can set `data_opt_out`. Deepgram then sends `mip_opt_out`
96
+ and IBM its learning opt-out header, and both recipes turn it on, since the
97
+ corpora's audio is licensed. The option stays outside the response cache
98
+ key, so every cached response and every published number stays as it was.
99
+ - `tests/repro/test_records_202610.py` rebuilds the 40 cells of the October
100
+ 2026 snapshot from frozen system outputs and compares every headline row
101
+ with its sha256 in `tests/repro/202610/headline_sha256.json`, so the check
102
+ is exact to the last digit and the numbers stay off the repository. It runs only when
103
+ `FABENCH_REPRO_INPUTS` names those inputs, which stay on the machine that
104
+ holds them.
105
+ - `score_pair` and `aggregate` take an `extras` list, which FA-Bench itself
106
+ never fills. Each extra sees the alignments a pair was scored on, or a
107
+ finished row with the scores behind it, so a further metric can be computed
108
+ without aligning twice. `runner.score_utts` returns the scores of a run.
109
+ - `evals/systems.py` holds the display name, family, pipeline and caveat flag
110
+ of every system, for the leaderboard and the repro test alike.
111
+ - `FABENCH_PY` names the Python every driver runs.
112
+
113
+ ### Changed
114
+
115
+ - FA-Bench needs Python 3.11 or newer. The GitHub pipeline runs the tests and
116
+ the self-test on 3.11 and 3.12.
117
+ - Each leaderboard row holds the headline columns at 20 ms and the utterance
118
+ counts `n_utts`, `n_absent` and `n_empty`. On the word tier these are
119
+ `wts_f1`, `wts_precision`, `wts_recall`, `wts_mae_ms`, `wts_start_mae_ms`
120
+ and `wts_end_mae_ms` with their counts, `wbe_ms` with `n_word_boundaries`,
121
+ `wbnd_f1_all_20ms` with its denominators, and `wer`. On the phone tier they
122
+ are `mae_ms` with `n_boundaries`, `bnd_f1_all_20ms` with its denominators,
123
+ and `per`. Each is computed as before, and every one the October records
124
+ hold equals its October value.
125
+ - `fabench report` writes a short `report.md` of those columns, and
126
+ `per_type.parquet` is no longer written.
127
+ - The self-test and gate checks that read an analysis metric read a headline
128
+ one now, each with an answer fixed in advance.
129
+ - `run_all.sh` scores each tool into its own cell and no longer pools a
130
+ rescore into `summary/`.
131
+ - `fabench run` fetches MUSAN only when a requested condition draws from it.
132
+ A clean run no longer downloads 11 GB.
133
+ - The MUSAN download and unpacking are rewritten. The tarball still lands only
134
+ when its size matches, and the unpacked tree still appears only when it is
135
+ complete, now from a staging directory that is always removed.
136
+ - `tools/build_wav_reverberate.sh` builds from a fresh Kaldi checkout again,
137
+ and builds `wav-to-duration` too.
138
+ - The shadow step writes its WAV files with ffmpeg's bit-exact flags, so their
139
+ headers carry no ffmpeg version and two builds from the same audio compare
140
+ byte for byte.
141
+ - The README banner carries no Exif or C2PA metadata. Its image data is
142
+ unchanged.
143
+ - The paper is linked on arXiv, and its PDFs are no longer in the repository.
144
+ - Examples and comments no longer quote corpus text.
145
+ - The README's leaderboard badge opens the live leaderboard at
146
+ https://olewave.com/en/leaderboard/fa-bench in place of the records page.
147
+
148
+ - Olign 1.0 given audio alone is classed one-step, as every commercial ASR
149
+ with timestamps is, since its words and times come back from one request.
150
+
151
+ ### Removed
152
+
153
+ - The cross-vendor analysis tools, `fabench/analyze/`, the eight `evals/`
154
+ scripts that read every system's per-utterance output side by side, and
155
+ `evals/gen_paper_tables.py`. They are Olewave's own work and are no longer
156
+ part of this repository.
157
+ - The analysis metrics, now part of Olewave's analysis work with the two
158
+ items after this one. They are every column at 10, 25, 50 and 100 ms, the
159
+ time-only boundary detection scores with over-segmentation and R-value, and
160
+ every boundary class other than `all`. Tolerance accuracy, median and
161
+ signed error and the bootstrap intervals go too. So do the S, D and I
162
+ percentages, onset and offset MAE, ARR and insertion rate, the error tail
163
+ share and the common-matched MAE. The wrong-hit share, the label variants of
164
+ the boundary F1, calibration, latency, RTF, coverage, the speaker count and
165
+ the underpowered flag complete the set. `fabench/metrics/` and
166
+ `fabench/score/{calibration,latency,recall}.py` go with them.
167
+ - The full report. `fabench/report/tables.py` and `curves.py` built its
168
+ per-type panel, degradation curves and calibration panel, and
169
+ `per_type.parquet` fed it.
170
+ - The code that builds the records pages, `evals/update_public_tables.py`,
171
+ `publish_records.py`, `report_parse.py`, `gen_tex_tables.py`,
172
+ `gen_provenance.py` with its cache, and `rescore_all.sh`.
173
+ - The result tables of every snapshot under `records/`, August, September
174
+ and October 2026, with their `Details.md` pages. They are part of Olewave's
175
+ analysis work, and the numbers are on the leaderboard.
176
+ `records/README.md` lists the snapshots and describes the corpora and the
177
+ four conditions.
178
+ - `docs/metrics/word_timestamp.{EN,CN}.md`, which compared named systems
179
+ under the word timestamp view. The README's Metrics section defines every
180
+ `wts_*` column.
181
+ - The MFA-paper scoring protocol, `scoring.protocol: mfa_paper`, with
182
+ `fabench/score/mfa_paper/` and its vendored mapping files. FA-Bench scores
183
+ under its own protocol only, so the `scoring_protocol` column goes too.
184
+ - The Olign test that needed a live service. The Olign adapter speaks REST
185
+ to the hosted endpoint, and a `transport` other than `rest` is an error.
186
+ - Two Olign 0.9 adapter options that no recipe set.
187
+ - Kaldi's unused `steps/libs/nnet3/` scripts.
188
+ - The GitHub workflows other than `ci.yml`. CI on GitHub runs `ci.yml` alone,
189
+ with lint, the tests and the self-test.
190
+
191
+ ### Fixed
192
+
193
+ - The cloud batch loop flags an exhausted quota on the worker thread that
194
+ hit it, so no further call leaves before the main thread notices. The CI
195
+ run on the 1.1.3 merge failed on the test for this, four calls past the
196
+ wall on a loaded runner, and the test now expects exactly one.
197
+ - The noise recipe's manifest reads the corpus splits only. It used to read
198
+ the noise corpora's own data directories as one more split.
199
+
200
+ ## [1.1.3] - 2026-10-12
201
+
202
+ Records snapshot `records/202610/`, scored with this release. No change
203
+ moves a number published in `records/202609/`, which stands as scored for
204
+ 1.1.0. The October snapshot adds PocketSphinx, Olign 1.0 and one cascade,
205
+ and its Olign rows are described below.
206
+
207
+ ### Added
208
+
209
+ - Olign 1.0 as its own row beside Olign 0.9, on both tracks. On Track 1 it
210
+ is given the reference transcript. On Track 2 it recognizes the words
211
+ itself and then aligns them, so the tables call it two-step, as they do
212
+ Qwen3-ASR. It is reached through Olewave's job API at
213
+ `https://api.olewave.com/olign/v1`, with the key in `OLIGN_API_KEY`. The
214
+ call lives with the cloud providers, which cache every response, retry
215
+ and run requests concurrently. Recipes `evals/aligners/olign_v1/` and
216
+ `evals/timestamp_asrs/olign_v1_asr/`.
217
+ - PocketSphinx as a Track 1 row, the HMM family's one member not built on
218
+ Kaldi. It aligns the way `pocketsphinx align` does, with the bundled
219
+ en-us model and CMUdict, and g2p_en for words outside the dictionary. It
220
+ is built from a pinned main-branch commit rather than the PyPI wheel,
221
+ because the alignment fixes that matter here landed after v5.1.1. An
222
+ utterance that fails on the default beams is aligned again with the beams
223
+ open. The utterances that still fail also fail upstream's own CLI, and
224
+ they count against the row as every system's failures do. See
225
+ `fabench/aligners/pocketsphinx/README.md`.
226
+ - Qwen3-ASR → PocketSphinx as a Track 2 cascade, generated the way the
227
+ other cascades are.
228
+ - The records snapshot `records/202610/`, with 19 forced aligners on
229
+ Track 1 and 34 systems on Track 2. `records/latest` and the README point
230
+ at it. ElevenLabs Scribe v2 has 12 of its 20 cells, since the account ran
231
+ out of credit before the dev split under the four degradations, and
232
+ ElevenLabs forced alignment is still unpublished for the same reason.
233
+ - Each Olign 0.9 hyp record keeps the `versions` object of the response
234
+ that made it, for responses fetched since 2026-10-11.
235
+ - A leaderboard badge at the top of the README, ahead of the paper badge,
236
+ that opens the current snapshot's results page.
237
+
238
+ ### Changed
239
+
240
+ - The three cascades into Olign 0.9 on Track 2 were rerun against the
241
+ Olign 0.9 service that produced its Track 1 row, so every Olign 0.9 row
242
+ comes from the same service. Under noise their word MAE improves by 0.8
243
+ to 1.6 ms and their F1 at 20 ms by 0.02 to 0.05. Clean cells move by
244
+ 0.3 ms or less.
245
+ - A cloud recipe's `base_url` is part of its response cache key, so a
246
+ recipe aimed at one endpoint cannot replay the cached answers of another.
247
+ No tracked recipe sets `base_url`, so no cached response is orphaned.
248
+ - The Olign live test runs only when it is configured.
249
+
250
+ ### Fixed
251
+
252
+ - The cloud adapter drops a phone whose end is not after its start, as the
253
+ Olign 0.9 adapter has since it was written. Olign 1.0 is the only cloud
254
+ row with a phone tier and it had kept them, so the two versions of one
255
+ system were scored under different rules. On Buckeye dev under additive
256
+ noise the difference was close to 10 ms of phone MAE. No published number
257
+ moves, since Olign 1.0 first appears in `records/202610/`.
258
+ - `evals/gen_cascade_configs.py` writes a base tool's relative paths
259
+ relative to the cascade's directory, and decides what is a path by the
260
+ key, as the composer does. A bare `venv: venv` was left pointing at
261
+ nothing.
262
+ - `evals/run_cpu_cells.sh` printed an empty tool name in every status line.
263
+
264
+ ## [1.1.2] - 2026-10-08
265
+
266
+ No change moves a published number. The records snapshot is still
267
+ `records/202609/`, as scored for 1.1.0.
268
+
269
+ ### Added
270
+
271
+ - ElevenLabs forced alignment as a Track 1 row, `elevenlabs_fa`. Each
272
+ utterance goes to ElevenLabs' `/v1/forced-alignment` endpoint with its
273
+ reference transcript, and the row is scored on word boundaries beside MFA
274
+ and Olign. It ships with `enabled: false` and no published numbers, because
275
+ every request bills the account. See
276
+ `fabench/aligners/elevenlabs_fa/README.md`.
277
+ - `cloud_check.py` takes `--text`, the words spoken in `--audio`, so it can
278
+ check a forced aligner. `cloud_cost.py` prices the new row and counts its
279
+ cache in `--actual`.
280
+ - A one-column, double-spaced copy of the paper,
281
+ `docs/paper_for_reviewer.pdf`, which the README's paper links now
282
+ open. Its tables are set larger and write every number with its leading
283
+ zero, 0.40 where the paper has .40, and its Fig. 1 is larger.
284
+ `evals/gen_paper_tables.py` writes its tables in the same run as the
285
+ paper's, so the two cannot disagree. The two-column `docs/paper.pdf`
286
+ stays, and a section at the end of the README links both and the paper
287
+ on arXiv.
288
+
289
+ ### Changed
290
+
291
+ - The commercial cloud adapter can send the reference transcript with the
292
+ audio. When it does, the transcript is part of the response cache key. The
293
+ Track 2 rows send none, so their keys and cached responses are unchanged.
294
+
295
+ ### Fixed
296
+
297
+ - Olign is named Olign 0.9, the version that produced its numbers. The records
298
+ in both snapshots, the Olign adapter's docs and the caption of the paper's
299
+ word table had called it Olign 1.0. Only the name changes.
300
+ - `docs/paper.pdf` is the revised paper. Its title is now "FA-Bench: A
301
+ Benchmark for Phone- and Word-Level Timestamp Accuracy in Forced Alignment
302
+ and ASR on Clean and Noisy Speech", its word table names Olign 0.9, and the
303
+ abstract and the introduction open with a new first sentence. The text
304
+ gives Qwen3-FA's word F1 at 20 ms as 0.40, the value in its table, where
305
+ it said 0.39. The tables are unchanged.
306
+
307
+ ## [1.1.1] - 2026-09-29
308
+
309
+ No change moves a published number. The records snapshot is still
310
+ `records/202609/`, as scored for 1.1.0.
311
+
312
+ ### Added
313
+
314
+ - The paper as `docs/paper.pdf`, linked from the top of the README by a PDF
315
+ icon beside the title, a badge and a sentence in the introduction.
316
+ - `tools/ci.sh` runs the three CI jobs on a clean export of HEAD, in a
317
+ Python 3.12 environment holding only what CI installs. GitHub and GitLab
318
+ call the same script, and ruff is pinned to 0.16.9.
319
+
320
+ ### Fixed
321
+
322
+ - The Olign adapter reaches the hosted endpoint. The endpoint rejects
323
+ Python's default User-Agent, so the adapter names itself, and it sends the
324
+ service token from `.fabench.env` when one is set. The default endpoint is `https://api.olewave.com/olign/v0.9`.
325
+ - Two workers kept an import they no longer use, which failed the lint job.
326
+ - `evals/README.md` again says the top-level `run_all.sh` uses the same
327
+ option parser. The house-style rewrite had dropped it.
328
+
329
+ ## [1.1.0] - 2026-09-26
330
+
331
+ Records snapshot `records/202609/`. This release was first tagged on
332
+ 2026-09-25 and re-cut on 2026-09-26 with further scorer fixes, so numbers
333
+ taken from the first tag are superseded.
334
+
335
+ ### Changes that move published numbers
336
+
337
+ - **Word MAE counts each boundary once.** A boundary shared by two words was
338
+ counted twice, which weighted the easy interior double. MAE rose by 3 to
339
+ 5 ms.
340
+ - **Word F1 is label-checked.** A word boundary counts only when the words on
341
+ both sides match the reference and its time is within the tolerance, the
342
+ two utterance edges included. It paired boundaries by time alone.
343
+ - **Phone boundary F1 checks the phone labels on both sides**, as the word
344
+ tier does.
345
+ - **An utterance a system returns nothing for is charged.** Whether it wrote
346
+ no record or an empty one, every boundary of that utterance counts against
347
+ F1 recall and every word and phone in it as a deletion in WER and PER. MAE
348
+ is still over what came back. Before, such utterances left the evaluation,
349
+ so a system that failed on hard material scored as if it had not. MFA, for
350
+ one, returns nothing for up to 354 of Buckeye test's 4,513 utterances under
351
+ noise.
352
+ - **A one-step recognizer is scored on the words it wrote.** The eight
353
+ commercial endpoints had their words rewritten onto the reference spelling
354
+ when saved (`boats` became the reference's `boat's`), and the open
355
+ recognizers never had.
356
+ - **Word MAE matches words without regard to case**, as F1 and WER already
357
+ did. It moved by at most 0.14 ms.
358
+ - **TIMIT's degraded audio is cut to the clean length.** Each file carried
359
+ about 475 ms of noise after its last word, which charged a system that
360
+ timed words into it. The TIMIT noisy cells were re-run, apart from
361
+ ElevenLabs, which a spot check found unchanged, and NeuFA, whose output is
362
+ cut at the clean length.
363
+ - **TIMIT's SA sentences are excluded, and Buckeye is re-split** 24/8/8 by
364
+ speaker, balanced for sex and age.
365
+ - **The records' Noisy column averages unrounded values**, so it matches the
366
+ paper to the last digit.
367
+
368
+ ### Added
369
+
370
+ - Track 2, timestamped ASR. One-step recognizers, including eight commercial
371
+ endpoints, and cascades that re-align a recognizer's transcript with a
372
+ Track 1 aligner.
373
+ - NeuFA, scored from output its authors produced with a checkpoint they have
374
+ not released.
375
+ - Boundary classes by position and by how many adjacent labels match.
376
+ - Leaderboard columns `n_absent` and `n_empty`, and an Utterance completion
377
+ note on every records page that says who returned nothing for how much.
378
+ - Records gain the version and checkpoint of every system, the commercial
379
+ endpoints with the model requested and the days of the calls, training data
380
+ and overlap for every system, and the output quirks the scores keep.
381
+ - `evals/publish_records.py --release <tag>`, so a snapshot names the public
382
+ release it was scored with.
383
+
384
+ ### Changed
385
+
386
+ - Every README and the records prose are rewritten in one house style, with
387
+ stale facts corrected along the way. There are eight commercial endpoints,
388
+ eight Track 1 systems emit phones, ElevenLabs has twelve of its twenty
389
+ cells, and NeuFA's rows come from its authors' checkpoint.
390
+
391
+ ### Fixed
392
+
393
+ - The MFA adapter finds the micromamba its install script puts under
394
+ `mamba_root`, and `fabench run` reads a recipe's relative paths against the
395
+ recipe folder.
396
+ - `max_calls: 0` means no API calls, a cache-only replay. It read as no
397
+ limit.
398
+ - The noise scripts exit non-zero when nothing was built or found.
399
+ - Audio files are written atomically.
400
+ - Machine-specific paths and account ids left the tracked recipes. They come
401
+ from `.fabench.env`.
402
+
403
+ ## [1.0.0] - 2026-08-17
404
+
405
+ First public release. Track 1 forced alignment on TIMIT and Buckeye, word
406
+ and phone tiers, clean audio and four degradations. Records snapshot
407
+ `records/202608/`.
@@ -0,0 +1,53 @@
1
+ cff-version: 1.2.0
2
+ message: "If you publish results produced with this software, please cite the FA-Bench paper."
3
+ title: "FA-Bench"
4
+ type: software
5
+ authors:
6
+ - name: "Olewave, LLC"
7
+ website: "https://olewave.com"
8
+ version: "2.1.0"
9
+ license: Apache-2.0
10
+ repository-code: "https://github.com/olewave/fa-bench"
11
+ url: "https://olewave.com/en/leaderboard/fa-bench"
12
+ keywords:
13
+ - forced alignment
14
+ - speech recognition
15
+ - word timestamps
16
+ - phone boundaries
17
+ - benchmark
18
+ preferred-citation:
19
+ type: article
20
+ title: "FA-Bench: A Benchmark for Phone- and Word-Level Timestamp Accuracy in Forced Alignment and ASR on Clean and Noisy Speech"
21
+ authors:
22
+ - given-names: Wei
23
+ family-names: Chu
24
+ - given-names: Yuanzhe
25
+ family-names: Dong
26
+ - given-names: Ke
27
+ family-names: Tan
28
+ - given-names: Dong
29
+ family-names: Han
30
+ - given-names: Yichao
31
+ family-names: Zhou
32
+ - given-names: Ruchao
33
+ family-names: Fan
34
+ - given-names: Bingshen
35
+ family-names: Mu
36
+ - given-names: Jingbei
37
+ family-names: Li
38
+ - given-names: Vishwas
39
+ family-names: Shetty
40
+ - given-names: Sarthak
41
+ family-names: Bisht
42
+ - given-names: Ziyue
43
+ family-names: Qiu
44
+ - given-names: Massa
45
+ family-names: Baali
46
+ - given-names: Rita
47
+ family-names: Singh
48
+ - given-names: Bhiksha
49
+ family-names: Raj
50
+ year: 2026
51
+ journal: "arXiv preprint arXiv:2609.32396"
52
+ doi: "10.48550/arXiv.2609.32396"
53
+ url: "https://arxiv.org/abs/2609.32396"