baari 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- baari-0.1.0/.gitignore +8 -0
- baari-0.1.0/LICENSE +24 -0
- baari-0.1.0/LICENSE.indic-smart-turn +27 -0
- baari-0.1.0/LICENSE.pipecat +24 -0
- baari-0.1.0/PKG-INFO +166 -0
- baari-0.1.0/README.md +130 -0
- baari-0.1.0/pyproject.toml +58 -0
- baari-0.1.0/src/baari/__init__.py +5 -0
- baari-0.1.0/src/baari/cli.py +242 -0
- baari-0.1.0/src/baari/clips.py +91 -0
- baari-0.1.0/src/baari/detector.py +62 -0
- baari-0.1.0/src/baari/evaluate.py +152 -0
- baari-0.1.0/src/baari/features.py +189 -0
- baari-0.1.0/src/baari/indicvoices.py +87 -0
- baari-0.1.0/src/baari/labels.py +71 -0
- baari-0.1.0/src/baari/settings.py +30 -0
- baari-0.1.0/src/baari/spotcheck.html +69 -0
- baari-0.1.0/src/baari/train.py +359 -0
- baari-0.1.0/tests/test_detector.py +63 -0
- baari-0.1.0/tests/test_pipeline.py +74 -0
baari-0.1.0/.gitignore
ADDED
baari-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Abdul Samad
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Parts of src/baari/clips.py and prompts/label.md are adapted from Indic Smart Turn
|
|
2
|
+
(https://github.com/adimyth/indic-smart-turn), under this license:
|
|
3
|
+
|
|
4
|
+
BSD 2-Clause License
|
|
5
|
+
|
|
6
|
+
Copyright (c) 2026, adimyth
|
|
7
|
+
|
|
8
|
+
Redistribution and use in source and binary forms, with or without
|
|
9
|
+
modification, are permitted provided that the following conditions are met:
|
|
10
|
+
|
|
11
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
12
|
+
list of conditions and the following disclaimer.
|
|
13
|
+
|
|
14
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
15
|
+
this list of conditions and the following disclaimer in the documentation
|
|
16
|
+
and/or other materials provided with the distribution.
|
|
17
|
+
|
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
19
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
20
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
21
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
22
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
23
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
24
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
25
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
26
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
27
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024–2025, Daily
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
baari-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: baari
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: End-of-turn detection for Urdu voice agents
|
|
5
|
+
Project-URL: Homepage, https://github.com/samadpls/baari
|
|
6
|
+
Project-URL: Issues, https://github.com/samadpls/baari/issues
|
|
7
|
+
Project-URL: Author, https://samadpls.com
|
|
8
|
+
Author-email: Abdul Samad <abdulsamadsid1@gmail.com>
|
|
9
|
+
License-Expression: BSD-2-Clause
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: LICENSE.indic-smart-turn
|
|
12
|
+
License-File: LICENSE.pipecat
|
|
13
|
+
Keywords: end-of-turn,onnx,pipecat,speech,turn-detection,urdu,voice-agents
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Natural Language :: Urdu
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: numpy
|
|
23
|
+
Requires-Dist: onnxruntime
|
|
24
|
+
Provides-Extra: pipeline
|
|
25
|
+
Requires-Dist: huggingface-hub>=0.30; extra == 'pipeline'
|
|
26
|
+
Requires-Dist: onnx; extra == 'pipeline'
|
|
27
|
+
Requires-Dist: pyarrow>=17; extra == 'pipeline'
|
|
28
|
+
Requires-Dist: pydantic-settings>=2; extra == 'pipeline'
|
|
29
|
+
Requires-Dist: pydantic>=2; extra == 'pipeline'
|
|
30
|
+
Requires-Dist: scikit-learn; extra == 'pipeline'
|
|
31
|
+
Requires-Dist: soundfile; extra == 'pipeline'
|
|
32
|
+
Requires-Dist: torch; extra == 'pipeline'
|
|
33
|
+
Requires-Dist: transformers>=4.48; extra == 'pipeline'
|
|
34
|
+
Requires-Dist: typer; extra == 'pipeline'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
<p align="center">
|
|
38
|
+
<img alt="baari: knows when an Urdu speaker has finished talking" src="https://raw.githubusercontent.com/samadpls/baari/main/assets/hero.svg" width="100%">
|
|
39
|
+
</p>
|
|
40
|
+
|
|
41
|
+
<p align="center">
|
|
42
|
+
<a href="https://samadpls.com"><img alt="by samadpls.com" src="https://img.shields.io/badge/by-samadpls.com-0E6B4F"></a>
|
|
43
|
+
<a href="https://github.com/samadpls/baari/blob/main/LICENSE"><img alt="Code license: BSD-2-Clause" src="https://img.shields.io/badge/code-BSD--2--Clause-0E6B4F"></a>
|
|
44
|
+
<a href="https://huggingface.co/datasets/ai4bharat/IndicVoices"><img alt="Data: IndicVoices, CC BY 4.0" src="https://img.shields.io/badge/data-IndicVoices%20(CC%20BY%204.0)-F2B544?logo=huggingface&logoColor=000"></a>
|
|
45
|
+
<a href="https://pypi.org/project/baari/"><img alt="PyPI" src="https://img.shields.io/pypi/v/baari?color=0E6B4F&label=pypi"></a>
|
|
46
|
+
<a href="https://pepy.tech/projects/baari"><img alt="Downloads" src="https://static.pepy.tech/badge/baari"></a>
|
|
47
|
+
<img alt="Python 3.10+" src="https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=fff">
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<b>End-of-turn detection for Urdu voice agents.</b><br>
|
|
52
|
+
At every pause, it decides whether the speaker has finished (reply now) or is only pausing mid-sentence (keep listening).
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
baari (باری) means "turn" in Urdu. The table below shows that today's turn detectors often cut Urdu
|
|
58
|
+
speakers off mid-sentence, or wait after they have finished. baari is a small audio model trained to
|
|
59
|
+
get this right. It follows the [Smart Turn](https://github.com/pipecat-ai/smart-turn) recipe and takes
|
|
60
|
+
the same input and output, so it can stand in for Smart Turn.
|
|
61
|
+
|
|
62
|
+
**Status: work in progress.** Done so far: the Urdu test set, the scores of existing models, and
|
|
63
|
+
labels for 61,251 training segments. The training code is in place. The first model hasn't been
|
|
64
|
+
trained yet.
|
|
65
|
+
|
|
66
|
+
## 🎧 Do existing models understand Urdu?
|
|
67
|
+
|
|
68
|
+
1,588 clips of real Urdu phone conversations, cut from the official held-out split of IndicVoices: 1,470
|
|
69
|
+
where the speaker had finished and 118 where they were pausing mid-sentence. Because most clips are
|
|
70
|
+
"finished", a model that always says "finished" gets 92.6% plain accuracy, so the fair score is the
|
|
71
|
+
average of the two middle columns (balanced accuracy).
|
|
72
|
+
|
|
73
|
+
| Model | Balanced accuracy (95% CI) | Waits when unfinished | Replies when finished |
|
|
74
|
+
|---|---|---|---|
|
|
75
|
+
| Always says "finished" (reference) | 50.0% | 0.0% | 100.0% |
|
|
76
|
+
| Smart Turn v3.2 (CPU, int8) | 63.4% (58.8–68.1) | 57.6% | 69.2% |
|
|
77
|
+
| Smart Turn v3.2 (GPU, fp32) | 58.8% (54.8–63.0) | 76.3% | 41.3% |
|
|
78
|
+
| Indic Smart Turn tiny (int8) | 73.4% (69.0–77.8) | 61.0% | 85.9% |
|
|
79
|
+
| Indic Smart Turn tiny (fp32) | 70.0% (65.8–74.4) | 51.7% | 88.4% |
|
|
80
|
+
| Indic Smart Turn base (fp32) | 70.9% (66.5–75.4) | 47.5% | 94.4% |
|
|
81
|
+
| PuristanLabs1/urdu-turn-v2 (Urdu) | 51.8% (48.4–55.4) | 16.9% | 86.7% |
|
|
82
|
+
|
|
83
|
+
Scored on a MacBook Pro (M3 Pro), with the same audio preparation Pipecat uses. int8 models are
|
|
84
|
+
sensitive: their scores shift by a few points between Arm and x86 CPUs, and even tiny changes in how
|
|
85
|
+
the audio is prepared flip about 1 answer in 100. fp32 results stay the same.
|
|
86
|
+
|
|
87
|
+
<details>
|
|
88
|
+
<summary>How the test set was made</summary>
|
|
89
|
+
|
|
90
|
+
- **Source:** the conversation recordings in the `valid` split of
|
|
91
|
+
[IndicVoices](https://huggingface.co/datasets/ai4bharat/IndicVoices) Urdu (CC BY 4.0): 1,654 segments,
|
|
92
|
+
81 calls, 80 speakers, 1.7 hours. Each segment ends at a pause.
|
|
93
|
+
- **Labels:** two LLM agents each read every call's transcript in order and marked whether the speaker
|
|
94
|
+
had finished at the end of each segment ([prompt](https://github.com/samadpls/baari/blob/main/prompts/label.md)). They agreed on 96.0%. Agreed
|
|
95
|
+
segments form the test set; the 66 others are kept apart as `test_ambiguous`.
|
|
96
|
+
- **Clips:** the last 8 s of 16 kHz audio ending at the segment, with the speaker's preceding segments
|
|
97
|
+
as context, the same recipe as [Indic Smart Turn](https://github.com/adimyth/indic-smart-turn).
|
|
98
|
+
- **Checked by ear:** a native speaker listened to 50 random clips. All 25 "finished" labels were
|
|
99
|
+
right, and 19 of 25 "unfinished" labels were. In the wrong ones the words stop mid-sentence but the
|
|
100
|
+
voice sounds finished, which a transcript cannot show. So roughly 1 in 4 "unfinished" clips may
|
|
101
|
+
really be finished, and the "waits when unfinished" column is a rough measure.
|
|
102
|
+
- **Leakage:** 70 of the 80 test speakers, and the other side of 57 of the 81 test calls, also appear in
|
|
103
|
+
the IndicVoices `train` split. Any training data will leave out those speakers and calls.
|
|
104
|
+
|
|
105
|
+
</details>
|
|
106
|
+
|
|
107
|
+
## Use it
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install baari
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from baari import TurnDetector
|
|
115
|
+
|
|
116
|
+
detector = TurnDetector(threads=4) # downloads baari's 9 MB int8 model once
|
|
117
|
+
detector.is_finished(audio) # 16 kHz mono audio (float or int16) that ends at the pause
|
|
118
|
+
detector.probability(audio) # the same decision as a number from 0 to 1
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
It needs only numpy and onnxruntime. One call takes about 12 ms on 4 CPU threads and 37 ms on one
|
|
122
|
+
(MacBook Pro, M3 Pro). Use `threads=1` on a server that handles many calls at once.
|
|
123
|
+
|
|
124
|
+
**The baari model is not released yet.** Until it is, `TurnDetector(path=...)` runs any Smart Turn
|
|
125
|
+
style ONNX file, such as [Indic Smart Turn tiny](https://huggingface.co/adimyth/indic-smart-turn), the
|
|
126
|
+
best existing model in the table above.
|
|
127
|
+
|
|
128
|
+
## 🛠️ Run it
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
uv sync --extra pipeline
|
|
132
|
+
echo "HF_TOKEN=..." > .env # accept the IndicVoices terms on Hugging Face first
|
|
133
|
+
uv run baari scan --split valid # index the conversation segments (text only)
|
|
134
|
+
uv run baari batches # label batches for the LLM agents
|
|
135
|
+
# the agents write data/labels/valid/results/{a,b}/batch-*.jsonl
|
|
136
|
+
uv run baari build # cut the 8 s clips
|
|
137
|
+
uv run baari evaluate # score the existing models
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
<details>
|
|
141
|
+
<summary>Training</summary>
|
|
142
|
+
|
|
143
|
+
The same steps on the IndicVoices `train` split, with one labeler. Test speakers and both sides of
|
|
144
|
+
test calls are left out, and about 5% of the remaining speakers form a dev split that picks the best
|
|
145
|
+
epoch.
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
uv run baari scan --split train
|
|
149
|
+
uv run baari batches --split train
|
|
150
|
+
# the agents write data/labels/train/results/a/batch-*.jsonl
|
|
151
|
+
uv run baari build --split train --labelers a --delete-shards
|
|
152
|
+
uv run baari train # fine-tune Indic Smart Turn tiny, export fp32 and int8 ONNX
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Training starts from Indic Smart Turn tiny. That model ships only as ONNX, so `baari train` first
|
|
156
|
+
reads its weights back into PyTorch. The [kaggle](https://github.com/samadpls/baari/tree/main/kaggle) folder runs the clip build and the
|
|
157
|
+
training on Kaggle.
|
|
158
|
+
|
|
159
|
+
</details>
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
<p align="center">
|
|
164
|
+
<a href="https://github.com/samadpls/baari/blob/main/docs/CREDITS.md">Credits</a> · <a href="https://github.com/samadpls/baari/blob/main/CONTRIBUTING.md">Contributing</a> · <a href="https://github.com/samadpls/baari/blob/main/LICENSE">BSD-2-Clause license</a><br>
|
|
165
|
+
Made by <a href="https://samadpls.com">Abdul Samad</a> (abdulsamadsid1@gmail.com)
|
|
166
|
+
</p>
|
baari-0.1.0/README.md
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img alt="baari: knows when an Urdu speaker has finished talking" src="https://raw.githubusercontent.com/samadpls/baari/main/assets/hero.svg" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://samadpls.com"><img alt="by samadpls.com" src="https://img.shields.io/badge/by-samadpls.com-0E6B4F"></a>
|
|
7
|
+
<a href="https://github.com/samadpls/baari/blob/main/LICENSE"><img alt="Code license: BSD-2-Clause" src="https://img.shields.io/badge/code-BSD--2--Clause-0E6B4F"></a>
|
|
8
|
+
<a href="https://huggingface.co/datasets/ai4bharat/IndicVoices"><img alt="Data: IndicVoices, CC BY 4.0" src="https://img.shields.io/badge/data-IndicVoices%20(CC%20BY%204.0)-F2B544?logo=huggingface&logoColor=000"></a>
|
|
9
|
+
<a href="https://pypi.org/project/baari/"><img alt="PyPI" src="https://img.shields.io/pypi/v/baari?color=0E6B4F&label=pypi"></a>
|
|
10
|
+
<a href="https://pepy.tech/projects/baari"><img alt="Downloads" src="https://static.pepy.tech/badge/baari"></a>
|
|
11
|
+
<img alt="Python 3.10+" src="https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=fff">
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<p align="center">
|
|
15
|
+
<b>End-of-turn detection for Urdu voice agents.</b><br>
|
|
16
|
+
At every pause, it decides whether the speaker has finished (reply now) or is only pausing mid-sentence (keep listening).
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
baari (باری) means "turn" in Urdu. The table below shows that today's turn detectors often cut Urdu
|
|
22
|
+
speakers off mid-sentence, or wait after they have finished. baari is a small audio model trained to
|
|
23
|
+
get this right. It follows the [Smart Turn](https://github.com/pipecat-ai/smart-turn) recipe and takes
|
|
24
|
+
the same input and output, so it can stand in for Smart Turn.
|
|
25
|
+
|
|
26
|
+
**Status: work in progress.** Done so far: the Urdu test set, the scores of existing models, and
|
|
27
|
+
labels for 61,251 training segments. The training code is in place. The first model hasn't been
|
|
28
|
+
trained yet.
|
|
29
|
+
|
|
30
|
+
## 🎧 Do existing models understand Urdu?
|
|
31
|
+
|
|
32
|
+
1,588 clips of real Urdu phone conversations, cut from the official held-out split of IndicVoices: 1,470
|
|
33
|
+
where the speaker had finished and 118 where they were pausing mid-sentence. Because most clips are
|
|
34
|
+
"finished", a model that always says "finished" gets 92.6% plain accuracy, so the fair score is the
|
|
35
|
+
average of the two middle columns (balanced accuracy).
|
|
36
|
+
|
|
37
|
+
| Model | Balanced accuracy (95% CI) | Waits when unfinished | Replies when finished |
|
|
38
|
+
|---|---|---|---|
|
|
39
|
+
| Always says "finished" (reference) | 50.0% | 0.0% | 100.0% |
|
|
40
|
+
| Smart Turn v3.2 (CPU, int8) | 63.4% (58.8–68.1) | 57.6% | 69.2% |
|
|
41
|
+
| Smart Turn v3.2 (GPU, fp32) | 58.8% (54.8–63.0) | 76.3% | 41.3% |
|
|
42
|
+
| Indic Smart Turn tiny (int8) | 73.4% (69.0–77.8) | 61.0% | 85.9% |
|
|
43
|
+
| Indic Smart Turn tiny (fp32) | 70.0% (65.8–74.4) | 51.7% | 88.4% |
|
|
44
|
+
| Indic Smart Turn base (fp32) | 70.9% (66.5–75.4) | 47.5% | 94.4% |
|
|
45
|
+
| PuristanLabs1/urdu-turn-v2 (Urdu) | 51.8% (48.4–55.4) | 16.9% | 86.7% |
|
|
46
|
+
|
|
47
|
+
Scored on a MacBook Pro (M3 Pro), with the same audio preparation Pipecat uses. int8 models are
|
|
48
|
+
sensitive: their scores shift by a few points between Arm and x86 CPUs, and even tiny changes in how
|
|
49
|
+
the audio is prepared flip about 1 answer in 100. fp32 results stay the same.
|
|
50
|
+
|
|
51
|
+
<details>
|
|
52
|
+
<summary>How the test set was made</summary>
|
|
53
|
+
|
|
54
|
+
- **Source:** the conversation recordings in the `valid` split of
|
|
55
|
+
[IndicVoices](https://huggingface.co/datasets/ai4bharat/IndicVoices) Urdu (CC BY 4.0): 1,654 segments,
|
|
56
|
+
81 calls, 80 speakers, 1.7 hours. Each segment ends at a pause.
|
|
57
|
+
- **Labels:** two LLM agents each read every call's transcript in order and marked whether the speaker
|
|
58
|
+
had finished at the end of each segment ([prompt](https://github.com/samadpls/baari/blob/main/prompts/label.md)). They agreed on 96.0%. Agreed
|
|
59
|
+
segments form the test set; the 66 others are kept apart as `test_ambiguous`.
|
|
60
|
+
- **Clips:** the last 8 s of 16 kHz audio ending at the segment, with the speaker's preceding segments
|
|
61
|
+
as context, the same recipe as [Indic Smart Turn](https://github.com/adimyth/indic-smart-turn).
|
|
62
|
+
- **Checked by ear:** a native speaker listened to 50 random clips. All 25 "finished" labels were
|
|
63
|
+
right, and 19 of 25 "unfinished" labels were. In the wrong ones the words stop mid-sentence but the
|
|
64
|
+
voice sounds finished, which a transcript cannot show. So roughly 1 in 4 "unfinished" clips may
|
|
65
|
+
really be finished, and the "waits when unfinished" column is a rough measure.
|
|
66
|
+
- **Leakage:** 70 of the 80 test speakers, and the other side of 57 of the 81 test calls, also appear in
|
|
67
|
+
the IndicVoices `train` split. Any training data will leave out those speakers and calls.
|
|
68
|
+
|
|
69
|
+
</details>
|
|
70
|
+
|
|
71
|
+
## Use it
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install baari
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from baari import TurnDetector
|
|
79
|
+
|
|
80
|
+
detector = TurnDetector(threads=4) # downloads baari's 9 MB int8 model once
|
|
81
|
+
detector.is_finished(audio) # 16 kHz mono audio (float or int16) that ends at the pause
|
|
82
|
+
detector.probability(audio) # the same decision as a number from 0 to 1
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
It needs only numpy and onnxruntime. One call takes about 12 ms on 4 CPU threads and 37 ms on one
|
|
86
|
+
(MacBook Pro, M3 Pro). Use `threads=1` on a server that handles many calls at once.
|
|
87
|
+
|
|
88
|
+
**The baari model is not released yet.** Until it is, `TurnDetector(path=...)` runs any Smart Turn
|
|
89
|
+
style ONNX file, such as [Indic Smart Turn tiny](https://huggingface.co/adimyth/indic-smart-turn), the
|
|
90
|
+
best existing model in the table above.
|
|
91
|
+
|
|
92
|
+
## 🛠️ Run it
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
uv sync --extra pipeline
|
|
96
|
+
echo "HF_TOKEN=..." > .env # accept the IndicVoices terms on Hugging Face first
|
|
97
|
+
uv run baari scan --split valid # index the conversation segments (text only)
|
|
98
|
+
uv run baari batches # label batches for the LLM agents
|
|
99
|
+
# the agents write data/labels/valid/results/{a,b}/batch-*.jsonl
|
|
100
|
+
uv run baari build # cut the 8 s clips
|
|
101
|
+
uv run baari evaluate # score the existing models
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
<details>
|
|
105
|
+
<summary>Training</summary>
|
|
106
|
+
|
|
107
|
+
The same steps on the IndicVoices `train` split, with one labeler. Test speakers and both sides of
|
|
108
|
+
test calls are left out, and about 5% of the remaining speakers form a dev split that picks the best
|
|
109
|
+
epoch.
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
uv run baari scan --split train
|
|
113
|
+
uv run baari batches --split train
|
|
114
|
+
# the agents write data/labels/train/results/a/batch-*.jsonl
|
|
115
|
+
uv run baari build --split train --labelers a --delete-shards
|
|
116
|
+
uv run baari train # fine-tune Indic Smart Turn tiny, export fp32 and int8 ONNX
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Training starts from Indic Smart Turn tiny. That model ships only as ONNX, so `baari train` first
|
|
120
|
+
reads its weights back into PyTorch. The [kaggle](https://github.com/samadpls/baari/tree/main/kaggle) folder runs the clip build and the
|
|
121
|
+
training on Kaggle.
|
|
122
|
+
|
|
123
|
+
</details>
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
<p align="center">
|
|
128
|
+
<a href="https://github.com/samadpls/baari/blob/main/docs/CREDITS.md">Credits</a> · <a href="https://github.com/samadpls/baari/blob/main/CONTRIBUTING.md">Contributing</a> · <a href="https://github.com/samadpls/baari/blob/main/LICENSE">BSD-2-Clause license</a><br>
|
|
129
|
+
Made by <a href="https://samadpls.com">Abdul Samad</a> (abdulsamadsid1@gmail.com)
|
|
130
|
+
</p>
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "baari"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "End-of-turn detection for Urdu voice agents"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "BSD-2-Clause"
|
|
7
|
+
authors = [{ name = "Abdul Samad", email = "abdulsamadsid1@gmail.com" }]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
dependencies = ["numpy", "onnxruntime"]
|
|
10
|
+
keywords = ["urdu", "turn-detection", "end-of-turn", "voice-agents", "speech", "pipecat", "onnx"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Intended Audience :: Developers",
|
|
14
|
+
"Natural Language :: Urdu",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Multimedia :: Sound/Audio :: Speech",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/samadpls/baari"
|
|
23
|
+
Issues = "https://github.com/samadpls/baari/issues"
|
|
24
|
+
Author = "https://samadpls.com"
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
pipeline = [
|
|
28
|
+
"huggingface_hub>=0.30",
|
|
29
|
+
"pyarrow>=17",
|
|
30
|
+
"soundfile",
|
|
31
|
+
"transformers>=4.48",
|
|
32
|
+
"scikit-learn",
|
|
33
|
+
"pydantic>=2",
|
|
34
|
+
"pydantic-settings>=2",
|
|
35
|
+
"typer",
|
|
36
|
+
"torch",
|
|
37
|
+
"onnx",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
baari = "baari.cli:app"
|
|
42
|
+
|
|
43
|
+
[dependency-groups]
|
|
44
|
+
dev = ["pytest", "ruff"]
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["hatchling"]
|
|
48
|
+
build-backend = "hatchling.build"
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.sdist]
|
|
51
|
+
# An allow-list, so local-only files (the plan, data, .env) can never end up in a release.
|
|
52
|
+
include = ["src/baari", "tests", "LICENSE*", "README.md", "pyproject.toml"]
|
|
53
|
+
|
|
54
|
+
[tool.uv]
|
|
55
|
+
cache-dir = ".uv-cache"
|
|
56
|
+
|
|
57
|
+
[tool.ruff]
|
|
58
|
+
line-length = 110
|