polywhisper 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polywhisper-0.2.0/.gitignore +27 -0
- polywhisper-0.2.0/LICENSE +21 -0
- polywhisper-0.2.0/PKG-INFO +309 -0
- polywhisper-0.2.0/README.md +272 -0
- polywhisper-0.2.0/polywhisper/__init__.py +7 -0
- polywhisper-0.2.0/polywhisper/__main__.py +3 -0
- polywhisper-0.2.0/polywhisper/audio.py +65 -0
- polywhisper-0.2.0/polywhisper/cli.py +229 -0
- polywhisper-0.2.0/polywhisper/model.py +241 -0
- polywhisper-0.2.0/polywhisper/onnx_backend.py +215 -0
- polywhisper-0.2.0/polywhisper/transcribe.py +205 -0
- polywhisper-0.2.0/pyproject.toml +73 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# secrets / env
|
|
2
|
+
.env
|
|
3
|
+
|
|
4
|
+
# macOS
|
|
5
|
+
.DS_Store
|
|
6
|
+
|
|
7
|
+
# artifacts: checkpoints, audio caches, logs, derived data
|
|
8
|
+
# (polywhisper_output is a symlink to an external SSD; result snapshots live in results/)
|
|
9
|
+
polywhisper_output
|
|
10
|
+
|
|
11
|
+
# old model checkpoints
|
|
12
|
+
models/
|
|
13
|
+
|
|
14
|
+
# stray redirects / scratch
|
|
15
|
+
nohup.log
|
|
16
|
+
|
|
17
|
+
# python
|
|
18
|
+
__pycache__/
|
|
19
|
+
*.pyc
|
|
20
|
+
.venv/
|
|
21
|
+
# ONNX build artifacts (hosted on HF under export/onnx/)
|
|
22
|
+
export/
|
|
23
|
+
|
|
24
|
+
# Build artifacts
|
|
25
|
+
dist/
|
|
26
|
+
build/
|
|
27
|
+
*.egg-info/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Eulogik
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: polywhisper
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Efficient multilingual Indic ASR via frozen Whisper + per-language LoRA
|
|
5
|
+
Project-URL: Homepage, https://github.com/eulogik/PolyWhisper
|
|
6
|
+
Project-URL: Documentation, https://github.com/eulogik/PolyWhisper#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/eulogik/PolyWhisper
|
|
8
|
+
Project-URL: Issues, https://github.com/eulogik/PolyWhisper/issues
|
|
9
|
+
Author: Eulogik
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: asr,bengali,hindi,indic,lora,marathi,multilingual,onnx,speech-recognition,tamil,telugu,whisper
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Requires-Dist: numpy
|
|
26
|
+
Requires-Dist: resampy
|
|
27
|
+
Requires-Dist: soundfile
|
|
28
|
+
Requires-Dist: torch>=2.0
|
|
29
|
+
Requires-Dist: transformers>=4.30
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
33
|
+
Provides-Extra: onnx
|
|
34
|
+
Requires-Dist: onnx>=1.15; extra == 'onnx'
|
|
35
|
+
Requires-Dist: onnxruntime>=1.16; extra == 'onnx'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
language:
|
|
40
|
+
- hi
|
|
41
|
+
- ta
|
|
42
|
+
- te
|
|
43
|
+
- bn
|
|
44
|
+
- mr
|
|
45
|
+
license: mit
|
|
46
|
+
library_name: transformers
|
|
47
|
+
pipeline_tag: automatic-speech-recognition
|
|
48
|
+
base_model: openai/whisper-small
|
|
49
|
+
tags:
|
|
50
|
+
- polywhisper
|
|
51
|
+
- indic-asr
|
|
52
|
+
- hindi-asr
|
|
53
|
+
- tamil-speech-recognition
|
|
54
|
+
- telugu-stt
|
|
55
|
+
- bengali-asr
|
|
56
|
+
- marathi-speech-to-text
|
|
57
|
+
- speech-recognition
|
|
58
|
+
- multilingual
|
|
59
|
+
- lora
|
|
60
|
+
- whisper
|
|
61
|
+
- hindi
|
|
62
|
+
- tamil
|
|
63
|
+
- telugu
|
|
64
|
+
- bengali
|
|
65
|
+
- marathi
|
|
66
|
+
- indic-languages
|
|
67
|
+
- indian-languages
|
|
68
|
+
- automatic-speech-recognition
|
|
69
|
+
- speech-to-text
|
|
70
|
+
- low-resource-asr
|
|
71
|
+
- fleurs
|
|
72
|
+
- indicvoices
|
|
73
|
+
- onnx
|
|
74
|
+
- quantized
|
|
75
|
+
- efficient-asr
|
|
76
|
+
- edge-asr
|
|
77
|
+
- peft
|
|
78
|
+
datasets:
|
|
79
|
+
- ai4bharat/indicvoices-st
|
|
80
|
+
- google/fleurs
|
|
81
|
+
model-index:
|
|
82
|
+
- name: PolyWhisper v9 (Whisper-Small + Per-Language LoRA)
|
|
83
|
+
results:
|
|
84
|
+
- task:
|
|
85
|
+
type: automatic-speech-recognition
|
|
86
|
+
name: Hindi Speech Recognition
|
|
87
|
+
dataset:
|
|
88
|
+
name: FLEURS Hindi (hi_in)
|
|
89
|
+
type: google/fleurs
|
|
90
|
+
metrics:
|
|
91
|
+
- type: wer
|
|
92
|
+
value: 46.3
|
|
93
|
+
name: WER (beam=1, normalized)
|
|
94
|
+
- task:
|
|
95
|
+
type: automatic-speech-recognition
|
|
96
|
+
name: Tamil Speech Recognition
|
|
97
|
+
dataset:
|
|
98
|
+
name: FLEURS Tamil (ta_in)
|
|
99
|
+
type: google/fleurs
|
|
100
|
+
metrics:
|
|
101
|
+
- type: wer
|
|
102
|
+
value: 70.1
|
|
103
|
+
name: WER (beam=1, normalized)
|
|
104
|
+
- task:
|
|
105
|
+
type: automatic-speech-recognition
|
|
106
|
+
name: Telugu Speech Recognition
|
|
107
|
+
dataset:
|
|
108
|
+
name: FLEURS Telugu (te_in)
|
|
109
|
+
type: google/fleurs
|
|
110
|
+
metrics:
|
|
111
|
+
- type: wer
|
|
112
|
+
value: 100.1
|
|
113
|
+
name: WER (beam=1, normalized)
|
|
114
|
+
- task:
|
|
115
|
+
type: automatic-speech-recognition
|
|
116
|
+
name: Bengali Speech Recognition
|
|
117
|
+
dataset:
|
|
118
|
+
name: FLEURS Bengali (bn_in)
|
|
119
|
+
type: google/fleurs
|
|
120
|
+
metrics:
|
|
121
|
+
- type: wer
|
|
122
|
+
value: 130.2
|
|
123
|
+
name: WER (beam=1, normalized)
|
|
124
|
+
- task:
|
|
125
|
+
type: automatic-speech-recognition
|
|
126
|
+
name: Marathi Speech Recognition
|
|
127
|
+
dataset:
|
|
128
|
+
name: FLEURS Marathi (mr_in)
|
|
129
|
+
type: google/fleurs
|
|
130
|
+
metrics:
|
|
131
|
+
- type: wer
|
|
132
|
+
value: 96.7
|
|
133
|
+
name: WER (beam=1, normalized)
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
[](https://huggingface.co/eulogik/polywhisper)
|
|
137
|
+
[](https://github.com/eulogik/PolyWhisper)
|
|
138
|
+
[](https://github.com/eulogik/PolyWhisper/releases)
|
|
139
|
+
[](https://opensource.org/licenses/MIT)
|
|
140
|
+
[](https://www.python.org/downloads/)
|
|
141
|
+
[](https://pytorch.org/)
|
|
142
|
+
[](https://onnxruntime.ai/)
|
|
143
|
+
    
|
|
144
|
+
|
|
145
|
+
# ๐๏ธ PolyWhisper v9 โ Efficient Multilingual Indic ASR
|
|
146
|
+
|
|
147
|
+
> **TL;DR:** PolyWhisper v9 is a production-ready automatic speech recognition (ASR) system for **Hindi, Tamil, Telugu, Bengali, and Marathi**. It pairs a **frozen OpenAI Whisper-Small backbone (244M params)** with tiny **per-language LoRA adapters (~14MB each)**. Bengali WER drops **โ34.5%** and Marathi **โ43.2%** versus the no-augmentation baseline โ at roughly **1% of the storage cost** of full fine-tuning.
|
|
148
|
+
|
|
149
|
+
## โจ Why PolyWhisper?
|
|
150
|
+
|
|
151
|
+
| | Full fine-tune (per language) | **PolyWhisper v9** |
|
|
152
|
+
|---|---|---|
|
|
153
|
+
| Storage per language | ~1.5 GB | **~14 MB (100ร smaller)** |
|
|
154
|
+
| Backbone | retrained each time | **frozen once, shared by all 5** |
|
|
155
|
+
| Bengali (bn) FLEURS WER | 198.8 (baseline) | **130.2 (โ34.5%)** |
|
|
156
|
+
| Marathi (mr) FLEURS WER | 170.1 (baseline) | **96.7 (โ43.2%)** |
|
|
157
|
+
| Telugu (te) FLEURS WER | 105.9 (baseline) | **100.1 (โ5.5%)** |
|
|
158
|
+
| Hindi (hi) FLEURS WER | 43.0 (baseline) | **46.3** |
|
|
159
|
+
| Tamil (ta) FLEURS WER | 68.2 (baseline) | **70.1** |
|
|
160
|
+
| CPU deployment | heavy | **ONNX INT8, no GPU needed** |
|
|
161
|
+
|
|
162
|
+
*WER = word error rate (lower is better). FLEURS test set, beam=1, punctuation-normalized scoring.*
|
|
163
|
+
|
|
164
|
+
## ๐ Benchmarks (FLEURS, beam=1, normalized WER)
|
|
165
|
+
|
|
166
|
+
| Language | Code | Script | v7 (no augment) | **v9 final** | ฮ vs v7 |
|
|
167
|
+
|---|---|---|---|---|---|
|
|
168
|
+
| Hindi | `hi` | Devanagari | 43.0 | **46.3** | +7.7% |
|
|
169
|
+
| Tamil | `ta` | Tamil | 68.2 | **70.1** | +2.8% |
|
|
170
|
+
| Telugu | `te` | Telugu | 105.9 | **100.1** | โ
**โ5.5%** |
|
|
171
|
+
| Bengali | `bn` | Bengali | 198.8 | **130.2** | โ
**โ34.5%** |
|
|
172
|
+
| Marathi | `mr` | Devanagari | 170.1 | **96.7** | โ
**โ43.2%** |
|
|
173
|
+
|
|
174
|
+
### ๐งช The v9 finding: augment per language, not globally
|
|
175
|
+
|
|
176
|
+
Training with SpecAugment + speed perturbation on **all** languages damaged Hindi/Tamil (token-loop degeneration) while massively helping Bengali/Marathi. The v9 recipe augments **only `bn`/`mr`** and trains `hi`/`ta` clean:
|
|
177
|
+
|
|
178
|
+
| Language | Augmentation | Result |
|
|
179
|
+
|---|---|---|
|
|
180
|
+
| Hindi, Tamil | none (clean) | matches no-augment baseline |
|
|
181
|
+
| Telugu, Bengali, Marathi | SpecAugment + 0.9ร/1.1ร speed perturb | large gains on hard languages |
|
|
182
|
+
|
|
183
|
+
## ๐ฆ Which adapter should I use?
|
|
184
|
+
|
|
185
|
+
| Language | Adapter file | Backbone | WER |
|
|
186
|
+
|---|---|---|---|
|
|
187
|
+
| Hindi (`hi`) | `polywhisper_output_hi/adapters_v3/hi_best_clean.pt` | `openai/whisper-small` | 46.3 |
|
|
188
|
+
| Tamil (`ta`) | `polywhisper_output_ta/adapters_v3/ta_best_clean.pt` | `openai/whisper-small` | 70.1 |
|
|
189
|
+
| Telugu (`te`) | `polywhisper_output_gpu0/adapters_v3/te_best_prod.pt` | `openai/whisper-small` | 100.1 |
|
|
190
|
+
| Bengali (`bn`) | `polywhisper_output_gpu0/adapters_v3/bn_best_prod.pt` | `openai/whisper-small` | 130.2 |
|
|
191
|
+
| Marathi (`mr`) | `polywhisper_output_gpu1/adapters_v3/mr_best_prod.pt` | `openai/whisper-small` | 96.7 |
|
|
192
|
+
|
|
193
|
+
All adapters are rank-16 LoRA (decoder + encoder attention), ~14MB each. Backbone weights are **not** included โ they load from `openai/whisper-small` at runtime.
|
|
194
|
+
|
|
195
|
+
## ๐ Quickstart
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
pip install -e .
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
# Hindi speech to text
|
|
203
|
+
polywhisper transcribe audio.wav --lang hi
|
|
204
|
+
|
|
205
|
+
# Tamil with JSON output
|
|
206
|
+
polywhisper transcribe audio.wav --lang ta --format json
|
|
207
|
+
|
|
208
|
+
# Auto-detect language, SRT subtitles
|
|
209
|
+
polywhisper transcribe audio.wav --format srt > subs.srt
|
|
210
|
+
|
|
211
|
+
# Batch a folder
|
|
212
|
+
polywhisper batch ./audio_folder/ --lang bn --output results.json
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
from polywhisper import transcribe
|
|
217
|
+
|
|
218
|
+
result = transcribe("audio.wav", lang="mr")
|
|
219
|
+
print(result.text)
|
|
220
|
+
print(result.segments) # timestamped segments
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## ๐ฅ๏ธ CPU-only inference (ONNX Runtime)
|
|
224
|
+
|
|
225
|
+
Export INT8-quantized ONNX graphs (no PyTorch, no GPU needed at inference):
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
polywhisper export --lang hi --variant prod --int8
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Pre-exported v9 graphs live under `export/onnx/` on the [Hub](https://huggingface.co/eulogik/polywhisper/tree/main/export/onnx) โ per language, fp32 + INT8:
|
|
232
|
+
|
|
233
|
+
| Lang | Encoder (fp32 / INT8) | Decoder (fp32 / INT8) |
|
|
234
|
+
|---|---|---|
|
|
235
|
+
| hi | 358MB / 97MB | 784MB / 204MB |
|
|
236
|
+
| ta | 358MB / 97MB | 784MB / 204MB |
|
|
237
|
+
| te | 358MB / 97MB | 784MB / 204MB |
|
|
238
|
+
| bn | 358MB / 97MB | 784MB / 204MB |
|
|
239
|
+
| mr | 358MB / 97MB | 784MB / 204MB |
|
|
240
|
+
|
|
241
|
+
Files are named `{lang}_{lang}_best_prod_{encoder,decoder}{,_int8}.onnx`. INT8 is ~4ร smaller.
|
|
242
|
+
|
|
243
|
+
**Verification:** fp32 ONNX vs PyTorch max diff < 1e-3 on all five languages (encoder + decoder). End-to-end greedy spot-checks (FLEURS audio, beam=1):
|
|
244
|
+
|
|
245
|
+
| Lang | torch WER | ONNX INT8 WER |
|
|
246
|
+
|---|---|---|
|
|
247
|
+
| hi (10 samples) | 43.4% | 48.3% |
|
|
248
|
+
| ta (5 samples) | 100.0% | 100.0% |
|
|
249
|
+
| te (5 samples) | 100.0% | 101.6% |
|
|
250
|
+
| bn (5 samples) | 104.9% | 118.7% |
|
|
251
|
+
| mr (5 samples) | 82.9% | 89.4% |
|
|
252
|
+
|
|
253
|
+
*Spot-checks are tiny (5โ10 utterances) so single-sentence flips move the numbers; fp32 ONNX is at parity with torch. INT8 trades a few points for 4ร smaller files.*
|
|
254
|
+
|
|
255
|
+
## ๐๏ธ Training recipe (reproducible)
|
|
256
|
+
|
|
257
|
+
- **Data:** [IndicVoices-ST](https://huggingface.co/datasets/ai4bharat/indicvoices-st) (~19โ20k clips/language) ยท **Eval:** [FLEURS](https://huggingface.co/datasets/google/fleurs)
|
|
258
|
+
- **Backbone:** `openai/whisper-small`, frozen ยท **Adapters:** LoRA rank-16, encoder + decoder attention
|
|
259
|
+
- **Schedule:** 3โ5 epochs/language, batch 4, AdamW, cosine LR (peak 1e-4), 2ร NVIDIA T4
|
|
260
|
+
- **Augmentation (v9):** SpecAugment + speed perturb for `bn`/`mr` only; `hi`/`ta`/`te` clean
|
|
261
|
+
- **Selection:** WER-gated checkpoints (`*_best_*.pt`) on FLEURS dev slices
|
|
262
|
+
- **Code:** [`train_v3.py`](https://github.com/eulogik/PolyWhisper/blob/main/train_v3.py) ยท orchestrator [`kaggle_train_resumable.py`](https://github.com/eulogik/PolyWhisper/blob/main/kaggle_train_resumable.py) ยท scoring [`normalize_ortho.py`](https://github.com/eulogik/PolyWhisper/blob/main/normalize_ortho.py)
|
|
263
|
+
|
|
264
|
+
## โ FAQ
|
|
265
|
+
|
|
266
|
+
**What is PolyWhisper?**
|
|
267
|
+
PolyWhisper is an open-source Indic ASR toolkit: one frozen Whisper-Small backbone plus five small per-language LoRA adapters covering Hindi, Tamil, Telugu, Bengali, and Marathi.
|
|
268
|
+
|
|
269
|
+
**How is it different from fine-tuning Whisper?**
|
|
270
|
+
Full fine-tuning rewrites ~244Mโ1.5B weights per language. PolyWhisper freezes the backbone and trains ~3.5M LoRA parameters per language (~14MB), so five languages ship for the storage cost of a rounding error.
|
|
271
|
+
|
|
272
|
+
**Which languages are production-ready?**
|
|
273
|
+
All five ship working adapters. Hindi (46.3 WER) and Tamil (70.1) are strongest; Bengali and Marathi improved dramatically in v9 (โ34.5% / โ43.2% vs baseline) but remain the hardest languages.
|
|
274
|
+
|
|
275
|
+
**Can I run it on CPU?**
|
|
276
|
+
Yes โ export to ONNX INT8 and run with ONNX Runtime, no GPU required.
|
|
277
|
+
|
|
278
|
+
**Can I run it on a Mac?**
|
|
279
|
+
Yes โ PyTorch MPS is supported (`Device: mps`), plus CPU via ONNX.
|
|
280
|
+
|
|
281
|
+
**What data was it trained/evaluated on?**
|
|
282
|
+
Trained on IndicVoices-ST conversational speech, evaluated on FLEURS read speech with punctuation-normalized, script-aware scoring.
|
|
283
|
+
|
|
284
|
+
## โ ๏ธ Limitations
|
|
285
|
+
|
|
286
|
+
- Absolute WER on Telugu/Bengali/Marathi is still high โ usable for assistive/search/subtitle-draft workflows, not verbatim legal/medical transcription.
|
|
287
|
+
- Evaluated on read speech (FLEURS); spontaneous conversational accuracy will differ.
|
|
288
|
+
- Beam=1 numbers above; beam=5 decoding improves results at higher latency.
|
|
289
|
+
|
|
290
|
+
## ๐ License & citation
|
|
291
|
+
|
|
292
|
+
Apache 2.0. Whisper weights ยฉ OpenAI. Training data: IndicVoices-ST (CC-BY) ยท Eval: FLEURS (CC-BY).
|
|
293
|
+
|
|
294
|
+
```bibtex
|
|
295
|
+
@misc{polywhisper2026,
|
|
296
|
+
title = {PolyWhisper: Efficient Multilingual Indic ASR via Frozen Backbone + Per-Language LoRA},
|
|
297
|
+
author = {Eulogik},
|
|
298
|
+
year = {2026},
|
|
299
|
+
publisher = {HuggingFace},
|
|
300
|
+
url = {https://huggingface.co/eulogik/polywhisper}
|
|
301
|
+
}
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
## ๐ Links
|
|
305
|
+
|
|
306
|
+
- ๐ค Model: [huggingface.co/eulogik/polywhisper](https://huggingface.co/eulogik/polywhisper)
|
|
307
|
+
- ๐ป Code: [github.com/eulogik/PolyWhisper](https://github.com/eulogik/PolyWhisper)
|
|
308
|
+
- ๐ฃ๏ธ Train data: [ai4bharat/indicvoices-st](https://huggingface.co/datasets/ai4bharat/indicvoices-st)
|
|
309
|
+
- ๐งช Eval data: [google/fleurs](https://huggingface.co/datasets/google/fleurs)
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
---
|
|
2
|
+
language:
|
|
3
|
+
- hi
|
|
4
|
+
- ta
|
|
5
|
+
- te
|
|
6
|
+
- bn
|
|
7
|
+
- mr
|
|
8
|
+
license: mit
|
|
9
|
+
library_name: transformers
|
|
10
|
+
pipeline_tag: automatic-speech-recognition
|
|
11
|
+
base_model: openai/whisper-small
|
|
12
|
+
tags:
|
|
13
|
+
- polywhisper
|
|
14
|
+
- indic-asr
|
|
15
|
+
- hindi-asr
|
|
16
|
+
- tamil-speech-recognition
|
|
17
|
+
- telugu-stt
|
|
18
|
+
- bengali-asr
|
|
19
|
+
- marathi-speech-to-text
|
|
20
|
+
- speech-recognition
|
|
21
|
+
- multilingual
|
|
22
|
+
- lora
|
|
23
|
+
- whisper
|
|
24
|
+
- hindi
|
|
25
|
+
- tamil
|
|
26
|
+
- telugu
|
|
27
|
+
- bengali
|
|
28
|
+
- marathi
|
|
29
|
+
- indic-languages
|
|
30
|
+
- indian-languages
|
|
31
|
+
- automatic-speech-recognition
|
|
32
|
+
- speech-to-text
|
|
33
|
+
- low-resource-asr
|
|
34
|
+
- fleurs
|
|
35
|
+
- indicvoices
|
|
36
|
+
- onnx
|
|
37
|
+
- quantized
|
|
38
|
+
- efficient-asr
|
|
39
|
+
- edge-asr
|
|
40
|
+
- peft
|
|
41
|
+
datasets:
|
|
42
|
+
- ai4bharat/indicvoices-st
|
|
43
|
+
- google/fleurs
|
|
44
|
+
model-index:
|
|
45
|
+
- name: PolyWhisper v9 (Whisper-Small + Per-Language LoRA)
|
|
46
|
+
results:
|
|
47
|
+
- task:
|
|
48
|
+
type: automatic-speech-recognition
|
|
49
|
+
name: Hindi Speech Recognition
|
|
50
|
+
dataset:
|
|
51
|
+
name: FLEURS Hindi (hi_in)
|
|
52
|
+
type: google/fleurs
|
|
53
|
+
metrics:
|
|
54
|
+
- type: wer
|
|
55
|
+
value: 46.3
|
|
56
|
+
name: WER (beam=1, normalized)
|
|
57
|
+
- task:
|
|
58
|
+
type: automatic-speech-recognition
|
|
59
|
+
name: Tamil Speech Recognition
|
|
60
|
+
dataset:
|
|
61
|
+
name: FLEURS Tamil (ta_in)
|
|
62
|
+
type: google/fleurs
|
|
63
|
+
metrics:
|
|
64
|
+
- type: wer
|
|
65
|
+
value: 70.1
|
|
66
|
+
name: WER (beam=1, normalized)
|
|
67
|
+
- task:
|
|
68
|
+
type: automatic-speech-recognition
|
|
69
|
+
name: Telugu Speech Recognition
|
|
70
|
+
dataset:
|
|
71
|
+
name: FLEURS Telugu (te_in)
|
|
72
|
+
type: google/fleurs
|
|
73
|
+
metrics:
|
|
74
|
+
- type: wer
|
|
75
|
+
value: 100.1
|
|
76
|
+
name: WER (beam=1, normalized)
|
|
77
|
+
- task:
|
|
78
|
+
type: automatic-speech-recognition
|
|
79
|
+
name: Bengali Speech Recognition
|
|
80
|
+
dataset:
|
|
81
|
+
name: FLEURS Bengali (bn_in)
|
|
82
|
+
type: google/fleurs
|
|
83
|
+
metrics:
|
|
84
|
+
- type: wer
|
|
85
|
+
value: 130.2
|
|
86
|
+
name: WER (beam=1, normalized)
|
|
87
|
+
- task:
|
|
88
|
+
type: automatic-speech-recognition
|
|
89
|
+
name: Marathi Speech Recognition
|
|
90
|
+
dataset:
|
|
91
|
+
name: FLEURS Marathi (mr_in)
|
|
92
|
+
type: google/fleurs
|
|
93
|
+
metrics:
|
|
94
|
+
- type: wer
|
|
95
|
+
value: 96.7
|
|
96
|
+
name: WER (beam=1, normalized)
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
[](https://huggingface.co/eulogik/polywhisper)
|
|
100
|
+
[](https://github.com/eulogik/PolyWhisper)
|
|
101
|
+
[](https://github.com/eulogik/PolyWhisper/releases)
|
|
102
|
+
[](https://opensource.org/licenses/MIT)
|
|
103
|
+
[](https://www.python.org/downloads/)
|
|
104
|
+
[](https://pytorch.org/)
|
|
105
|
+
[](https://onnxruntime.ai/)
|
|
106
|
+
    
|
|
107
|
+
|
|
108
|
+
# ๐๏ธ PolyWhisper v9 โ Efficient Multilingual Indic ASR
|
|
109
|
+
|
|
110
|
+
> **TL;DR:** PolyWhisper v9 is a production-ready automatic speech recognition (ASR) system for **Hindi, Tamil, Telugu, Bengali, and Marathi**. It pairs a **frozen OpenAI Whisper-Small backbone (244M params)** with tiny **per-language LoRA adapters (~14MB each)**. Bengali WER drops **โ34.5%** and Marathi **โ43.2%** versus the no-augmentation baseline โ at roughly **1% of the storage cost** of full fine-tuning.
|
|
111
|
+
|
|
112
|
+
## โจ Why PolyWhisper?
|
|
113
|
+
|
|
114
|
+
| | Full fine-tune (per language) | **PolyWhisper v9** |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| Storage per language | ~1.5 GB | **~14 MB (100ร smaller)** |
|
|
117
|
+
| Backbone | retrained each time | **frozen once, shared by all 5** |
|
|
118
|
+
| Bengali (bn) FLEURS WER | 198.8 (baseline) | **130.2 (โ34.5%)** |
|
|
119
|
+
| Marathi (mr) FLEURS WER | 170.1 (baseline) | **96.7 (โ43.2%)** |
|
|
120
|
+
| Telugu (te) FLEURS WER | 105.9 (baseline) | **100.1 (โ5.5%)** |
|
|
121
|
+
| Hindi (hi) FLEURS WER | 43.0 (baseline) | **46.3** |
|
|
122
|
+
| Tamil (ta) FLEURS WER | 68.2 (baseline) | **70.1** |
|
|
123
|
+
| CPU deployment | heavy | **ONNX INT8, no GPU needed** |
|
|
124
|
+
|
|
125
|
+
*WER = word error rate (lower is better). FLEURS test set, beam=1, punctuation-normalized scoring.*
|
|
126
|
+
|
|
127
|
+
## ๐ Benchmarks (FLEURS, beam=1, normalized WER)
|
|
128
|
+
|
|
129
|
+
| Language | Code | Script | v7 (no augment) | **v9 final** | ฮ vs v7 |
|
|
130
|
+
|---|---|---|---|---|---|
|
|
131
|
+
| Hindi | `hi` | Devanagari | 43.0 | **46.3** | +7.7% |
|
|
132
|
+
| Tamil | `ta` | Tamil | 68.2 | **70.1** | +2.8% |
|
|
133
|
+
| Telugu | `te` | Telugu | 105.9 | **100.1** | โ
**โ5.5%** |
|
|
134
|
+
| Bengali | `bn` | Bengali | 198.8 | **130.2** | โ
**โ34.5%** |
|
|
135
|
+
| Marathi | `mr` | Devanagari | 170.1 | **96.7** | โ
**โ43.2%** |
|
|
136
|
+
|
|
137
|
+
### ๐งช The v9 finding: augment per language, not globally
|
|
138
|
+
|
|
139
|
+
Training with SpecAugment + speed perturbation on **all** languages damaged Hindi/Tamil (token-loop degeneration) while massively helping Bengali/Marathi. The v9 recipe augments **only `bn`/`mr`** and trains `hi`/`ta` clean:
|
|
140
|
+
|
|
141
|
+
| Language | Augmentation | Result |
|
|
142
|
+
|---|---|---|
|
|
143
|
+
| Hindi, Tamil | none (clean) | matches no-augment baseline |
|
|
144
|
+
| Telugu, Bengali, Marathi | SpecAugment + 0.9ร/1.1ร speed perturb | large gains on hard languages |
|
|
145
|
+
|
|
146
|
+
## ๐ฆ Which adapter should I use?
|
|
147
|
+
|
|
148
|
+
| Language | Adapter file | Backbone | WER |
|
|
149
|
+
|---|---|---|---|
|
|
150
|
+
| Hindi (`hi`) | `polywhisper_output_hi/adapters_v3/hi_best_clean.pt` | `openai/whisper-small` | 46.3 |
|
|
151
|
+
| Tamil (`ta`) | `polywhisper_output_ta/adapters_v3/ta_best_clean.pt` | `openai/whisper-small` | 70.1 |
|
|
152
|
+
| Telugu (`te`) | `polywhisper_output_gpu0/adapters_v3/te_best_prod.pt` | `openai/whisper-small` | 100.1 |
|
|
153
|
+
| Bengali (`bn`) | `polywhisper_output_gpu0/adapters_v3/bn_best_prod.pt` | `openai/whisper-small` | 130.2 |
|
|
154
|
+
| Marathi (`mr`) | `polywhisper_output_gpu1/adapters_v3/mr_best_prod.pt` | `openai/whisper-small` | 96.7 |
|
|
155
|
+
|
|
156
|
+
All adapters are rank-16 LoRA (decoder + encoder attention), ~14MB each. Backbone weights are **not** included โ they load from `openai/whisper-small` at runtime.
|
|
157
|
+
|
|
158
|
+
## ๐ Quickstart
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
pip install -e .
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
# Hindi speech to text
|
|
166
|
+
polywhisper transcribe audio.wav --lang hi
|
|
167
|
+
|
|
168
|
+
# Tamil with JSON output
|
|
169
|
+
polywhisper transcribe audio.wav --lang ta --format json
|
|
170
|
+
|
|
171
|
+
# Auto-detect language, SRT subtitles
|
|
172
|
+
polywhisper transcribe audio.wav --format srt > subs.srt
|
|
173
|
+
|
|
174
|
+
# Batch a folder
|
|
175
|
+
polywhisper batch ./audio_folder/ --lang bn --output results.json
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
from polywhisper import transcribe
|
|
180
|
+
|
|
181
|
+
result = transcribe("audio.wav", lang="mr")
|
|
182
|
+
print(result.text)
|
|
183
|
+
print(result.segments) # timestamped segments
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## ๐ฅ๏ธ CPU-only inference (ONNX Runtime)
|
|
187
|
+
|
|
188
|
+
Export INT8-quantized ONNX graphs (no PyTorch, no GPU needed at inference):
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
polywhisper export --lang hi --variant prod --int8
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Pre-exported v9 graphs live under `export/onnx/` on the [Hub](https://huggingface.co/eulogik/polywhisper/tree/main/export/onnx) โ per language, fp32 + INT8:
|
|
195
|
+
|
|
196
|
+
| Lang | Encoder (fp32 / INT8) | Decoder (fp32 / INT8) |
|
|
197
|
+
|---|---|---|
|
|
198
|
+
| hi | 358MB / 97MB | 784MB / 204MB |
|
|
199
|
+
| ta | 358MB / 97MB | 784MB / 204MB |
|
|
200
|
+
| te | 358MB / 97MB | 784MB / 204MB |
|
|
201
|
+
| bn | 358MB / 97MB | 784MB / 204MB |
|
|
202
|
+
| mr | 358MB / 97MB | 784MB / 204MB |
|
|
203
|
+
|
|
204
|
+
Files are named `{lang}_{lang}_best_prod_{encoder,decoder}{,_int8}.onnx`. INT8 is ~4ร smaller.
|
|
205
|
+
|
|
206
|
+
**Verification:** fp32 ONNX vs PyTorch max diff < 1e-3 on all five languages (encoder + decoder). End-to-end greedy spot-checks (FLEURS audio, beam=1):
|
|
207
|
+
|
|
208
|
+
| Lang | torch WER | ONNX INT8 WER |
|
|
209
|
+
|---|---|---|
|
|
210
|
+
| hi (10 samples) | 43.4% | 48.3% |
|
|
211
|
+
| ta (5 samples) | 100.0% | 100.0% |
|
|
212
|
+
| te (5 samples) | 100.0% | 101.6% |
|
|
213
|
+
| bn (5 samples) | 104.9% | 118.7% |
|
|
214
|
+
| mr (5 samples) | 82.9% | 89.4% |
|
|
215
|
+
|
|
216
|
+
*Spot-checks are tiny (5โ10 utterances) so single-sentence flips move the numbers; fp32 ONNX is at parity with torch. INT8 trades a few points for 4ร smaller files.*
|
|
217
|
+
|
|
218
|
+
## ๐๏ธ Training recipe (reproducible)
|
|
219
|
+
|
|
220
|
+
- **Data:** [IndicVoices-ST](https://huggingface.co/datasets/ai4bharat/indicvoices-st) (~19โ20k clips/language) ยท **Eval:** [FLEURS](https://huggingface.co/datasets/google/fleurs)
|
|
221
|
+
- **Backbone:** `openai/whisper-small`, frozen ยท **Adapters:** LoRA rank-16, encoder + decoder attention
|
|
222
|
+
- **Schedule:** 3โ5 epochs/language, batch 4, AdamW, cosine LR (peak 1e-4), 2ร NVIDIA T4
|
|
223
|
+
- **Augmentation (v9):** SpecAugment + speed perturb for `bn`/`mr` only; `hi`/`ta`/`te` clean
|
|
224
|
+
- **Selection:** WER-gated checkpoints (`*_best_*.pt`) on FLEURS dev slices
|
|
225
|
+
- **Code:** [`train_v3.py`](https://github.com/eulogik/PolyWhisper/blob/main/train_v3.py) ยท orchestrator [`kaggle_train_resumable.py`](https://github.com/eulogik/PolyWhisper/blob/main/kaggle_train_resumable.py) ยท scoring [`normalize_ortho.py`](https://github.com/eulogik/PolyWhisper/blob/main/normalize_ortho.py)
|
|
226
|
+
|
|
227
|
+
## โ FAQ
|
|
228
|
+
|
|
229
|
+
**What is PolyWhisper?**
|
|
230
|
+
PolyWhisper is an open-source Indic ASR toolkit: one frozen Whisper-Small backbone plus five small per-language LoRA adapters covering Hindi, Tamil, Telugu, Bengali, and Marathi.
|
|
231
|
+
|
|
232
|
+
**How is it different from fine-tuning Whisper?**
|
|
233
|
+
Full fine-tuning rewrites ~244Mโ1.5B weights per language. PolyWhisper freezes the backbone and trains ~3.5M LoRA parameters per language (~14MB), so five languages ship for the storage cost of a rounding error.
|
|
234
|
+
|
|
235
|
+
**Which languages are production-ready?**
|
|
236
|
+
All five ship working adapters. Hindi (46.3 WER) and Tamil (70.1) are strongest; Bengali and Marathi improved dramatically in v9 (โ34.5% / โ43.2% vs baseline) but remain the hardest languages.
|
|
237
|
+
|
|
238
|
+
**Can I run it on CPU?**
|
|
239
|
+
Yes โ export to ONNX INT8 and run with ONNX Runtime, no GPU required.
|
|
240
|
+
|
|
241
|
+
**Can I run it on a Mac?**
|
|
242
|
+
Yes โ PyTorch MPS is supported (`Device: mps`), plus CPU via ONNX.
|
|
243
|
+
|
|
244
|
+
**What data was it trained/evaluated on?**
|
|
245
|
+
Trained on IndicVoices-ST conversational speech, evaluated on FLEURS read speech with punctuation-normalized, script-aware scoring.
|
|
246
|
+
|
|
247
|
+
## โ ๏ธ Limitations
|
|
248
|
+
|
|
249
|
+
- Absolute WER on Telugu/Bengali/Marathi is still high โ usable for assistive/search/subtitle-draft workflows, not verbatim legal/medical transcription.
|
|
250
|
+
- Evaluated on read speech (FLEURS); spontaneous conversational accuracy will differ.
|
|
251
|
+
- Beam=1 numbers above; beam=5 decoding improves results at higher latency.
|
|
252
|
+
|
|
253
|
+
## ๐ License & citation
|
|
254
|
+
|
|
255
|
+
Apache 2.0. Whisper weights ยฉ OpenAI. Training data: IndicVoices-ST (CC-BY) ยท Eval: FLEURS (CC-BY).
|
|
256
|
+
|
|
257
|
+
```bibtex
|
|
258
|
+
@misc{polywhisper2026,
|
|
259
|
+
title = {PolyWhisper: Efficient Multilingual Indic ASR via Frozen Backbone + Per-Language LoRA},
|
|
260
|
+
author = {Eulogik},
|
|
261
|
+
year = {2026},
|
|
262
|
+
publisher = {HuggingFace},
|
|
263
|
+
url = {https://huggingface.co/eulogik/polywhisper}
|
|
264
|
+
}
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## ๐ Links
|
|
268
|
+
|
|
269
|
+
- ๐ค Model: [huggingface.co/eulogik/polywhisper](https://huggingface.co/eulogik/polywhisper)
|
|
270
|
+
- ๐ป Code: [github.com/eulogik/PolyWhisper](https://github.com/eulogik/PolyWhisper)
|
|
271
|
+
- ๐ฃ๏ธ Train data: [ai4bharat/indicvoices-st](https://huggingface.co/datasets/ai4bharat/indicvoices-st)
|
|
272
|
+
- ๐งช Eval data: [google/fleurs](https://huggingface.co/datasets/google/fleurs)
|