islkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- islkit-0.1.0/.gitignore +14 -0
- islkit-0.1.0/LICENSE +21 -0
- islkit-0.1.0/PKG-INFO +200 -0
- islkit-0.1.0/README.md +173 -0
- islkit-0.1.0/docs/README.md +21 -0
- islkit-0.1.0/experiments/README.md +55 -0
- islkit-0.1.0/pyproject.toml +65 -0
- islkit-0.1.0/src/islkit/__init__.py +88 -0
- islkit-0.1.0/src/islkit/adapters.py +237 -0
- islkit-0.1.0/src/islkit/baseline.py +287 -0
- islkit-0.1.0/src/islkit/data.py +320 -0
- islkit-0.1.0/src/islkit/device.py +38 -0
- islkit-0.1.0/src/islkit/domain.py +187 -0
- islkit-0.1.0/src/islkit/features.py +323 -0
- islkit-0.1.0/src/islkit/infer.py +911 -0
- islkit-0.1.0/src/islkit/labels.py +213 -0
- islkit-0.1.0/src/islkit/metrics.py +81 -0
- islkit-0.1.0/src/islkit/model.py +623 -0
- islkit-0.1.0/src/islkit/pipeline.py +717 -0
- islkit-0.1.0/src/islkit/plotting.py +131 -0
- islkit-0.1.0/src/islkit/seeding.py +19 -0
- islkit-0.1.0/src/islkit/server.py +246 -0
- islkit-0.1.0/src/islkit/view.py +287 -0
- islkit-0.1.0/src/islkit/viz.py +435 -0
- islkit-0.1.0/tests/fixtures/recognition-events-v1.jsonl +13 -0
- islkit-0.1.0/tests/test_baseline.py +219 -0
- islkit-0.1.0/tests/test_data.py +236 -0
- islkit-0.1.0/tests/test_domain.py +180 -0
- islkit-0.1.0/tests/test_features.py +325 -0
- islkit-0.1.0/tests/test_infer.py +952 -0
- islkit-0.1.0/tests/test_islkit.py +91 -0
- islkit-0.1.0/tests/test_labels.py +137 -0
- islkit-0.1.0/tests/test_model.py +490 -0
- islkit-0.1.0/tests/test_pipeline.py +855 -0
- islkit-0.1.0/tests/test_server.py +416 -0
- islkit-0.1.0/tests/test_view.py +220 -0
- islkit-0.1.0/tests/test_viz.py +149 -0
islkit-0.1.0/.gitignore
ADDED
islkit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jayraj
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
islkit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: islkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Indian Sign Language recognition from MediaPipe landmarks: features, a dual-branch TCN, and a live recognition service
|
|
5
|
+
Project-URL: Homepage, https://github.com/jbrathwa/islkit
|
|
6
|
+
Project-URL: Issues, https://github.com/jbrathwa/islkit/issues
|
|
7
|
+
Author: Jayraj
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: accessibility,isl,landmarks,mediapipe,sign-language,tcn
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
18
|
+
Requires-Python: <3.13,>=3.12
|
|
19
|
+
Requires-Dist: mediapipe==0.10.18
|
|
20
|
+
Requires-Dist: numpy<2,>=1.26
|
|
21
|
+
Requires-Dist: torch>=2.5
|
|
22
|
+
Provides-Extra: train
|
|
23
|
+
Requires-Dist: pandas>=3.0.5; extra == 'train'
|
|
24
|
+
Requires-Dist: pyarrow>=25.0.1; extra == 'train'
|
|
25
|
+
Requires-Dist: xgboost>=3.4.1; extra == 'train'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# islkit
|
|
29
|
+
|
|
30
|
+
Indian Sign Language (ISL) recognition from MediaPipe Holistic landmarks.
|
|
31
|
+
|
|
32
|
+
islkit turns camera frames into glosses. It covers every step: a landmark
|
|
33
|
+
feature encoder, a dual-branch temporal convolutional network (TCN), a loader
|
|
34
|
+
for the INCLUDE dataset, live inference with a confidence-gated decline, and a
|
|
35
|
+
small HTTP/SSE recognition service. It runs on CPU and is built to work on small
|
|
36
|
+
ARM boards as well as laptops.
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
camera → MediaPipe Holistic → RawFrame → encode_clip (T×352) → TCN → gloss | None
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
pip install islkit # inference and the recognition service
|
|
46
|
+
pip install "islkit[train]" # + pandas, pyarrow, xgboost for INCLUDE and the baseline
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Requires **Python 3.12**. `mediapipe` is pinned to `0.10.18`, which also pins
|
|
50
|
+
`numpy<2`. That is the last MediaPipe release that keeps `mp.solutions` and
|
|
51
|
+
still runs on ARMv8.0-A (Cortex-A53) boards.
|
|
52
|
+
|
|
53
|
+
On macOS, XGBoost needs `brew install libomp`. Don't import `torch` and
|
|
54
|
+
`xgboost` in the same process: each bundles its own libomp, and the process
|
|
55
|
+
aborts with `OMP: Error #15`. `import islkit` keeps torch lazy for this reason.
|
|
56
|
+
|
|
57
|
+
## Design
|
|
58
|
+
|
|
59
|
+
**Features, not raw landmarks.** Holistic's flattened output has 1,662 values,
|
|
60
|
+
and 85% of them are face mesh. The encoder reduces each frame to **183 floats**
|
|
61
|
+
(352 with velocity):
|
|
62
|
+
|
|
63
|
+
| Block | Dims |
|
|
64
|
+
|--------------------------------|------|
|
|
65
|
+
| Hand-local shapes (2 × 21 × 3) | 126 |
|
|
66
|
+
| Wrist positions in body frame | 6 |
|
|
67
|
+
| Upper-body pose (11 landmarks) | 33 |
|
|
68
|
+
| Non-manual scalars from face | 4 |
|
|
69
|
+
| Validity mask | 14 |
|
|
70
|
+
|
|
71
|
+
The encoder uses three coordinate frames rather than one normalisation. Body
|
|
72
|
+
frame: origin at the shoulder midpoint, scaled by shoulder width. Hand-local
|
|
73
|
+
frame: origin at the wrist, scaled by the wrist-to-middle-MCP distance. Wrist
|
|
74
|
+
position is recovered separately in the body frame. This keeps handshape
|
|
75
|
+
separate from location.
|
|
76
|
+
|
|
77
|
+
**Principles the code enforces:**
|
|
78
|
+
|
|
79
|
+
- **Hand slots are geometric.** Slot 0 is the hand nearer the dominant-side
|
|
80
|
+
shoulder. MediaPipe's handedness label flips under occlusion and is never read.
|
|
81
|
+
- **Mask, never zero-fill.** A missing hand is not a hand at the origin. A
|
|
82
|
+
validity bit is carried per part and multiplied through the network.
|
|
83
|
+
- **The label map is frozen.** `LabelMap` is saved beside the weights. Class
|
|
84
|
+
order rebuilt from a directory listing silently shifts indices.
|
|
85
|
+
- **Checkpoints are tied to the encoder.** A checkpoint records an encoder
|
|
86
|
+
fingerprint, and `SignRecogniser` refuses to serve it through a different
|
|
87
|
+
encoder.
|
|
88
|
+
- **Store raw landmarks.** Recordings hold raw MediaPipe output, so they can
|
|
89
|
+
be re-encoded when the normalisation changes.
|
|
90
|
+
- **The model can say "I don't know".** Below the confidence threshold a
|
|
91
|
+
prediction is `None`. For an accessibility device, silence is better than a
|
|
92
|
+
confident wrong answer.
|
|
93
|
+
|
|
94
|
+
## Usage
|
|
95
|
+
|
|
96
|
+
### Encode a clip
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from islkit import encode_clip
|
|
100
|
+
from islkit.infer import HolisticExtractor
|
|
101
|
+
|
|
102
|
+
frames = []
|
|
103
|
+
with HolisticExtractor() as extractor:
|
|
104
|
+
for frame_bgr in video_frames: # BGR numpy arrays, e.g. from cv2
|
|
105
|
+
_, raw, _ = extractor.process(frame_bgr)
|
|
106
|
+
frames.append(raw)
|
|
107
|
+
|
|
108
|
+
clip = encode_clip(frames, T=48) # (48, 352) float32
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Recognise a sign
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from islkit import SignRecogniser
|
|
115
|
+
|
|
116
|
+
recogniser = SignRecogniser("classifier.pt", threshold=0.6) # labels_*.json beside it
|
|
117
|
+
prediction = recogniser.classify(frames)
|
|
118
|
+
print(prediction.gloss, prediction.confidence, prediction.top3)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`prediction.gloss` is `None` when the model declines.
|
|
122
|
+
|
|
123
|
+
### Train
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from islkit import build_model, load_include
|
|
127
|
+
from islkit.model import fit
|
|
128
|
+
|
|
129
|
+
data = load_include("path/to/isl-mediapipe-holistic-landmarks") # data.X: (N, 48, 352)
|
|
130
|
+
model = build_model(n_classes=len(data.label_map))
|
|
131
|
+
fit(model, data.X, data.y)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`load_include` reads the Kaggle
|
|
135
|
+
[`indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
|
|
136
|
+
dump of INCLUDE and caches the encoded arrays. That dump records no signer or
|
|
137
|
+
session, so a class-stratified k-fold over whole clips is the only honest split
|
|
138
|
+
available. It still leaks signers, so **report numbers from it as optimistic**.
|
|
139
|
+
For your own recordings, use leave-one-session-out: `build_dataset` returns a
|
|
140
|
+
`sessions` array for this.
|
|
141
|
+
|
|
142
|
+
`replace_head` and `freeze_backbone` support fine-tuning a pretrained backbone
|
|
143
|
+
on a smaller vocabulary. Always report per-class F1 (`per_class_f1`), not just
|
|
144
|
+
aggregate accuracy.
|
|
145
|
+
|
|
146
|
+
### Run the recognition service
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
import threading
|
|
150
|
+
from islkit.infer import ClipStore, HolisticExtractor, SignRecogniser
|
|
151
|
+
from islkit.pipeline import CameraSource, RecognitionPipeline
|
|
152
|
+
from islkit.server import EventHub, make_server
|
|
153
|
+
|
|
154
|
+
recogniser = SignRecogniser("classifier.pt")
|
|
155
|
+
hub = EventHub()
|
|
156
|
+
pipeline = RecognitionPipeline(
|
|
157
|
+
recogniser=recogniser,
|
|
158
|
+
source_factory=lambda: CameraSource(0, "1280x720"),
|
|
159
|
+
on_event=hub.publish,
|
|
160
|
+
extractor_factory=HolisticExtractor,
|
|
161
|
+
store=ClipStore(),
|
|
162
|
+
on_pause=hub.clear_take,
|
|
163
|
+
)
|
|
164
|
+
threading.Thread(target=pipeline.run, daemon=True).start()
|
|
165
|
+
make_server(pipeline, hub).serve_forever() # 127.0.0.1:9978
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
| Route | Does |
|
|
169
|
+
|-----------------|----------------------------------------|
|
|
170
|
+
| `GET /results` | Server-sent events: tracking, takes, predictions |
|
|
171
|
+
| `POST /capture` | Start or pause capture |
|
|
172
|
+
| `GET /health` | Pipeline and subscriber status |
|
|
173
|
+
|
|
174
|
+
`islkit.view.make_view_server` serves an optional annotated MJPEG debug view
|
|
175
|
+
on a separate port.
|
|
176
|
+
|
|
177
|
+
Pretrained weights are not shipped with the package.
|
|
178
|
+
|
|
179
|
+
## Guides and experiments
|
|
180
|
+
|
|
181
|
+
- [`docs/`](docs/README.md): how the features, dataset, baseline, pretraining and fine-tuning
|
|
182
|
+
work, each argued from committed results.
|
|
183
|
+
- [`experiments/`](experiments/README.md): the scripts that produce those results, runnable from
|
|
184
|
+
a clone of this repository.
|
|
185
|
+
|
|
186
|
+
## Development
|
|
187
|
+
|
|
188
|
+
```sh
|
|
189
|
+
uv sync --extra train
|
|
190
|
+
uv run pytest
|
|
191
|
+
uv run ruff check .
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
The tests cover the properties that would silently break recognition: encoder
|
|
195
|
+
invariances, mask gating, label-map and checkpoint round-trips, and the
|
|
196
|
+
saved-clip layout.
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
MIT. See [LICENSE](LICENSE).
|
islkit-0.1.0/README.md
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# islkit
|
|
2
|
+
|
|
3
|
+
Indian Sign Language (ISL) recognition from MediaPipe Holistic landmarks.
|
|
4
|
+
|
|
5
|
+
islkit turns camera frames into glosses. It covers every step: a landmark
|
|
6
|
+
feature encoder, a dual-branch temporal convolutional network (TCN), a loader
|
|
7
|
+
for the INCLUDE dataset, live inference with a confidence-gated decline, and a
|
|
8
|
+
small HTTP/SSE recognition service. It runs on CPU and is built to work on small
|
|
9
|
+
ARM boards as well as laptops.
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
camera → MediaPipe Holistic → RawFrame → encode_clip (T×352) → TCN → gloss | None
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
pip install islkit # inference and the recognition service
|
|
19
|
+
pip install "islkit[train]" # + pandas, pyarrow, xgboost for INCLUDE and the baseline
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Requires **Python 3.12**. `mediapipe` is pinned to `0.10.18`, which also pins
|
|
23
|
+
`numpy<2`. That is the last MediaPipe release that keeps `mp.solutions` and
|
|
24
|
+
still runs on ARMv8.0-A (Cortex-A53) boards.
|
|
25
|
+
|
|
26
|
+
On macOS, XGBoost needs `brew install libomp`. Don't import `torch` and
|
|
27
|
+
`xgboost` in the same process: each bundles its own libomp, and the process
|
|
28
|
+
aborts with `OMP: Error #15`. `import islkit` keeps torch lazy for this reason.
|
|
29
|
+
|
|
30
|
+
## Design
|
|
31
|
+
|
|
32
|
+
**Features, not raw landmarks.** Holistic's flattened output has 1,662 values,
|
|
33
|
+
and 85% of them are face mesh. The encoder reduces each frame to **183 floats**
|
|
34
|
+
(352 with velocity):
|
|
35
|
+
|
|
36
|
+
| Block | Dims |
|
|
37
|
+
|--------------------------------|------|
|
|
38
|
+
| Hand-local shapes (2 × 21 × 3) | 126 |
|
|
39
|
+
| Wrist positions in body frame | 6 |
|
|
40
|
+
| Upper-body pose (11 landmarks) | 33 |
|
|
41
|
+
| Non-manual scalars from face | 4 |
|
|
42
|
+
| Validity mask | 14 |
|
|
43
|
+
|
|
44
|
+
The encoder uses three coordinate frames rather than one normalisation. Body
|
|
45
|
+
frame: origin at the shoulder midpoint, scaled by shoulder width. Hand-local
|
|
46
|
+
frame: origin at the wrist, scaled by the wrist-to-middle-MCP distance. Wrist
|
|
47
|
+
position is recovered separately in the body frame. This keeps handshape
|
|
48
|
+
separate from location.
|
|
49
|
+
|
|
50
|
+
**Principles the code enforces:**
|
|
51
|
+
|
|
52
|
+
- **Hand slots are geometric.** Slot 0 is the hand nearer the dominant-side
|
|
53
|
+
shoulder. MediaPipe's handedness label flips under occlusion and is never read.
|
|
54
|
+
- **Mask, never zero-fill.** A missing hand is not a hand at the origin. A
|
|
55
|
+
validity bit is carried per part and multiplied through the network.
|
|
56
|
+
- **The label map is frozen.** `LabelMap` is saved beside the weights. Class
|
|
57
|
+
order rebuilt from a directory listing silently shifts indices.
|
|
58
|
+
- **Checkpoints are tied to the encoder.** A checkpoint records an encoder
|
|
59
|
+
fingerprint, and `SignRecogniser` refuses to serve it through a different
|
|
60
|
+
encoder.
|
|
61
|
+
- **Store raw landmarks.** Recordings hold raw MediaPipe output, so they can
|
|
62
|
+
be re-encoded when the normalisation changes.
|
|
63
|
+
- **The model can say "I don't know".** Below the confidence threshold a
|
|
64
|
+
prediction is `None`. For an accessibility device, silence is better than a
|
|
65
|
+
confident wrong answer.
|
|
66
|
+
|
|
67
|
+
## Usage
|
|
68
|
+
|
|
69
|
+
### Encode a clip
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from islkit import encode_clip
|
|
73
|
+
from islkit.infer import HolisticExtractor
|
|
74
|
+
|
|
75
|
+
frames = []
|
|
76
|
+
with HolisticExtractor() as extractor:
|
|
77
|
+
for frame_bgr in video_frames: # BGR numpy arrays, e.g. from cv2
|
|
78
|
+
_, raw, _ = extractor.process(frame_bgr)
|
|
79
|
+
frames.append(raw)
|
|
80
|
+
|
|
81
|
+
clip = encode_clip(frames, T=48) # (48, 352) float32
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Recognise a sign
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from islkit import SignRecogniser
|
|
88
|
+
|
|
89
|
+
recogniser = SignRecogniser("classifier.pt", threshold=0.6) # labels_*.json beside it
|
|
90
|
+
prediction = recogniser.classify(frames)
|
|
91
|
+
print(prediction.gloss, prediction.confidence, prediction.top3)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
`prediction.gloss` is `None` when the model declines.
|
|
95
|
+
|
|
96
|
+
### Train
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from islkit import build_model, load_include
|
|
100
|
+
from islkit.model import fit
|
|
101
|
+
|
|
102
|
+
data = load_include("path/to/isl-mediapipe-holistic-landmarks") # data.X: (N, 48, 352)
|
|
103
|
+
model = build_model(n_classes=len(data.label_map))
|
|
104
|
+
fit(model, data.X, data.y)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
`load_include` reads the Kaggle
|
|
108
|
+
[`indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
|
|
109
|
+
dump of INCLUDE and caches the encoded arrays. That dump records no signer or
|
|
110
|
+
session, so a class-stratified k-fold over whole clips is the only honest split
|
|
111
|
+
available. It still leaks signers, so **report numbers from it as optimistic**.
|
|
112
|
+
For your own recordings, use leave-one-session-out: `build_dataset` returns a
|
|
113
|
+
`sessions` array for this.
|
|
114
|
+
|
|
115
|
+
`replace_head` and `freeze_backbone` support fine-tuning a pretrained backbone
|
|
116
|
+
on a smaller vocabulary. Always report per-class F1 (`per_class_f1`), not just
|
|
117
|
+
aggregate accuracy.
|
|
118
|
+
|
|
119
|
+
### Run the recognition service
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
import threading
|
|
123
|
+
from islkit.infer import ClipStore, HolisticExtractor, SignRecogniser
|
|
124
|
+
from islkit.pipeline import CameraSource, RecognitionPipeline
|
|
125
|
+
from islkit.server import EventHub, make_server
|
|
126
|
+
|
|
127
|
+
recogniser = SignRecogniser("classifier.pt")
|
|
128
|
+
hub = EventHub()
|
|
129
|
+
pipeline = RecognitionPipeline(
|
|
130
|
+
recogniser=recogniser,
|
|
131
|
+
source_factory=lambda: CameraSource(0, "1280x720"),
|
|
132
|
+
on_event=hub.publish,
|
|
133
|
+
extractor_factory=HolisticExtractor,
|
|
134
|
+
store=ClipStore(),
|
|
135
|
+
on_pause=hub.clear_take,
|
|
136
|
+
)
|
|
137
|
+
threading.Thread(target=pipeline.run, daemon=True).start()
|
|
138
|
+
make_server(pipeline, hub).serve_forever() # 127.0.0.1:9978
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
| Route | Does |
|
|
142
|
+
|-----------------|----------------------------------------|
|
|
143
|
+
| `GET /results` | Server-sent events: tracking, takes, predictions |
|
|
144
|
+
| `POST /capture` | Start or pause capture |
|
|
145
|
+
| `GET /health` | Pipeline and subscriber status |
|
|
146
|
+
|
|
147
|
+
`islkit.view.make_view_server` serves an optional annotated MJPEG debug view
|
|
148
|
+
on a separate port.
|
|
149
|
+
|
|
150
|
+
Pretrained weights are not shipped with the package.
|
|
151
|
+
|
|
152
|
+
## Guides and experiments
|
|
153
|
+
|
|
154
|
+
- [`docs/`](docs/README.md): how the features, dataset, baseline, pretraining and fine-tuning
|
|
155
|
+
work, each argued from committed results.
|
|
156
|
+
- [`experiments/`](experiments/README.md): the scripts that produce those results, runnable from
|
|
157
|
+
a clone of this repository.
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```sh
|
|
162
|
+
uv sync --extra train
|
|
163
|
+
uv run pytest
|
|
164
|
+
uv run ruff check .
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The tests cover the properties that would silently break recognition: encoder
|
|
168
|
+
invariances, mask gating, label-map and checkpoint round-trips, and the
|
|
169
|
+
saved-clip layout.
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# islkit guides
|
|
2
|
+
|
|
3
|
+
Each guide explains one part of the pipeline and argues from results committed under
|
|
4
|
+
[`runs/`](../runs), produced by the scripts in [`experiments/`](../experiments). Every number can be
|
|
5
|
+
regenerated with the command next to it.
|
|
6
|
+
|
|
7
|
+
| Guide | Covers |
|
|
8
|
+
|---|---|
|
|
9
|
+
| [Landmark features](features.md) | The 183-dim encoder, its invariances, and the replay instrument that checks them by eye |
|
|
10
|
+
| [The INCLUDE dataset](include-dataset.md) | Layout, the missing signer identifier, tracking quality, and the replay on real landmarks |
|
|
11
|
+
| [Baseline](baseline.md) | Pooled statistics + XGBoost: the floor a sequence model has to clear |
|
|
12
|
+
| [Pretraining](pretraining.md) | The dual-branch TCN on INCLUDE, its controls, and why its accuracy is not a device accuracy |
|
|
13
|
+
| [Fine-tuning](fine-tuning.md) | Choosing a vocabulary, two-stage fine-tuning on your own recordings, and training for a device camera |
|
|
14
|
+
|
|
15
|
+
## One caveat that applies everywhere
|
|
16
|
+
|
|
17
|
+
The public INCLUDE landmark dump does not record who signed each clip, so no split can hold out
|
|
18
|
+
signers. Every INCLUDE number in these guides comes from a class-stratified k-fold over whole clips,
|
|
19
|
+
which is **optimistic**: the same signers appear on both sides of every fold. They show that the
|
|
20
|
+
features and models work; they do not estimate accuracy for a new signer. See
|
|
21
|
+
[the dataset guide](include-dataset.md#2-there-is-no-signer-identifier).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Experiments
|
|
2
|
+
|
|
3
|
+
Runnable scripts that exercise `islkit` end to end. Each one writes to `runs/<experiment>/`, and
|
|
4
|
+
the guides in [`docs/`](../docs) argue from those outputs. Figures and summaries are committed;
|
|
5
|
+
weights (`*.pt`, `*.onnx`) and logs are not.
|
|
6
|
+
|
|
7
|
+
## Setup
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
uv sync --extra train # pandas, pyarrow and xgboost for the INCLUDE experiments
|
|
11
|
+
export ISLKIT_INCLUDE_ROOT=/path/to/isl-mediapipe-holistic-landmarks
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Run every script from the repository root, so `runs/` and the `data/cache/` of encoded INCLUDE
|
|
15
|
+
clips land in the right place. The INCLUDE dump is the Kaggle dataset
|
|
16
|
+
[`swaptr/indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
|
|
17
|
+
(3.4 GB). The first script that loads it encodes all 4,284 clips into a 291 MB cache.
|
|
18
|
+
|
|
19
|
+
On macOS, XGBoost needs `brew install libomp`. torch and XGBoost each bundle their own OpenMP
|
|
20
|
+
runtime and abort when loaded into one process, which is why the baseline and the TCN are separate
|
|
21
|
+
scripts.
|
|
22
|
+
|
|
23
|
+
## The scripts
|
|
24
|
+
|
|
25
|
+
| Script | Needs | Writes | Guide |
|
|
26
|
+
|---|---|---|---|
|
|
27
|
+
| `feature_smoke.py` | nothing | stdout only | [features](../docs/features.md#4-cost) |
|
|
28
|
+
| `replay_features.py` | nothing | `runs/replay_features/` | [features](../docs/features.md#5-seeing-the-features-the-replay-instrument) |
|
|
29
|
+
| `tcn_gestures.py` | nothing | `runs/tcn_gestures/` | below |
|
|
30
|
+
| `inspect_include.py` | INCLUDE | `runs/inspect_include/` | [INCLUDE](../docs/include-dataset.md) |
|
|
31
|
+
| `replay_include.py` | INCLUDE | `runs/replay_include/` | [INCLUDE](../docs/include-dataset.md#4-the-replay-on-real-landmarks) |
|
|
32
|
+
| `baseline_xgb.py` | INCLUDE | `runs/baseline_xgb/` | [baseline](../docs/baseline.md) |
|
|
33
|
+
| `pretrain.py` | INCLUDE, baseline summary | `runs/pretrain/` | [pretraining](../docs/pretraining.md) |
|
|
34
|
+
| `vocab_confusability.py` | INCLUDE | `data/vocab.json` | [fine-tuning](../docs/fine-tuning.md#2-choose-the-vocabulary-before-recording-it) |
|
|
35
|
+
| `finetune.py` | INCLUDE, own recordings, pretrained classifier | `runs/finetune/` | [fine-tuning](../docs/fine-tuning.md#4-two-stage-fine-tuning) |
|
|
36
|
+
| `device_head.py` | INCLUDE, pretrained backbone, a vocabulary | `runs/device_head/` | [fine-tuning](../docs/fine-tuning.md#5-training-for-the-devices-camera) |
|
|
37
|
+
|
|
38
|
+
Each script's docstring lists its flags; `--help` prints them. `pretrain.py`, `finetune.py` and
|
|
39
|
+
`device_head.py` take `--smoke` for a fast wiring check.
|
|
40
|
+
|
|
41
|
+
Run them in table order the first time: `pretrain.py` reads its bar from
|
|
42
|
+
`runs/baseline_xgb/summary.json`, and the fine-tuning scripts load pretraining weights.
|
|
43
|
+
|
|
44
|
+
## A first TCN: `tcn_gestures.py`
|
|
45
|
+
|
|
46
|
+
A 30-second, data-free introduction to temporal convolutional networks. It fabricates six
|
|
47
|
+
"signs" as noisy sine trajectories over 32 features, gives each take a recording session, holds out
|
|
48
|
+
an entire session, and trains a small residual TCN.
|
|
49
|
+
|
|
50
|
+

|
|
51
|
+
|
|
52
|
+
Training accuracy reaches ~99% while accuracy on the held-out session levels off near 78%. That gap
|
|
53
|
+
is the point: takes from one session share their noise and timing, so a random split over takes
|
|
54
|
+
would put near-duplicates on both sides and report something much closer to the training curve.
|
|
55
|
+
Hold out whole sessions whenever your data has them.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "islkit"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Indian Sign Language recognition from MediaPipe landmarks: features, a dual-branch TCN, and a live recognition service"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Jayraj" }]
|
|
9
|
+
requires-python = ">=3.12,<3.13"
|
|
10
|
+
keywords = ["sign-language", "isl", "mediapipe", "landmarks", "tcn", "accessibility"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Intended Audience :: Science/Research",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
19
|
+
]
|
|
20
|
+
urls = { Homepage = "https://github.com/jbrathwa/islkit", Issues = "https://github.com/jbrathwa/islkit/issues" }
|
|
21
|
+
dependencies = [
|
|
22
|
+
"torch>=2.5",
|
|
23
|
+
# 0.10.18 is pinned exactly and pins numpy<2 behind it. It is the last release
|
|
24
|
+
# that keeps mp.solutions (1.0.x removed it), and 1.0.x's aarch64 wheel needs
|
|
25
|
+
# ARMv8.1-A LSE atomics, so it will not run on Cortex-A53 boards.
|
|
26
|
+
"mediapipe==0.10.18",
|
|
27
|
+
"numpy>=1.26,<2",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# cv2 is deliberately not declared: mediapipe hard-requires opencv-contrib-python,
|
|
31
|
+
# and adding another OpenCV package on top puts two providers of one module in the
|
|
32
|
+
# environment. matplotlib likewise arrives with mediapipe.
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
train = [
|
|
35
|
+
"pandas>=3.0.5",
|
|
36
|
+
"pyarrow>=25.0.1",
|
|
37
|
+
"xgboost>=3.4.1",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = [
|
|
42
|
+
"pytest>=8.0",
|
|
43
|
+
"ruff>=0.6",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["hatchling>=1.27"]
|
|
48
|
+
build-backend = "hatchling.build"
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/islkit"]
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.targets.sdist]
|
|
54
|
+
include = ["src/islkit", "tests", "LICENSE", "README.md"]
|
|
55
|
+
|
|
56
|
+
[tool.ruff]
|
|
57
|
+
line-length = 100
|
|
58
|
+
src = ["src", "tests", "experiments"]
|
|
59
|
+
|
|
60
|
+
[tool.ruff.lint]
|
|
61
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
62
|
+
|
|
63
|
+
[tool.pytest.ini_options]
|
|
64
|
+
testpaths = ["tests"]
|
|
65
|
+
addopts = "-q"
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""islkit — Indian Sign Language recognition from MediaPipe landmarks.
|
|
2
|
+
|
|
3
|
+
Landmark features, a dual-branch TCN, the INCLUDE loader, live inference and a
|
|
4
|
+
small HTTP/SSE recognition service.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
|
|
9
|
+
from islkit.baseline import pooled_stats, stratified_folds
|
|
10
|
+
from islkit.data import IncludeData, load_include
|
|
11
|
+
from islkit.features import DIM_FRAME, DIM_GEOM, RawFrame, encode_clip, encode_frame
|
|
12
|
+
from islkit.labels import LabelMap, build_dataset, decode_prediction
|
|
13
|
+
from islkit.metrics import accuracy, confusion_matrix, per_class_f1, top_confusions
|
|
14
|
+
from islkit.plotting import plot_confusion, plot_curves, plot_f1_distribution
|
|
15
|
+
from islkit.viz import animate_replay, reconstruct, still_replay
|
|
16
|
+
|
|
17
|
+
# device, seeding and model are the only modules that import torch, and they are
|
|
18
|
+
# loaded on first use rather than at import time. Not for speed: torch bundles its
|
|
19
|
+
# own libomp, XGBoost links Homebrew's, and macOS aborts the process when both land
|
|
20
|
+
# in it (OMP Error #15). The XGBoost baseline is pure numpy and must be able to import
|
|
21
|
+
# load_include without dragging torch in behind it. See islkit/baseline.py.
|
|
22
|
+
_LAZY = {
|
|
23
|
+
"describe_device": "islkit.device",
|
|
24
|
+
"pick_device": "islkit.device",
|
|
25
|
+
"set_seed": "islkit.seeding",
|
|
26
|
+
"Backbone": "islkit.model",
|
|
27
|
+
"SignClassifier": "islkit.model",
|
|
28
|
+
"build_model": "islkit.model",
|
|
29
|
+
"freeze_backbone": "islkit.model",
|
|
30
|
+
"replace_head": "islkit.model",
|
|
31
|
+
"ClipStore": "islkit.infer",
|
|
32
|
+
"ClipTooShort": "islkit.infer",
|
|
33
|
+
"HolisticExtractor": "islkit.infer",
|
|
34
|
+
"Prediction": "islkit.infer",
|
|
35
|
+
"SignRecogniser": "islkit.infer",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def __getattr__(name: str):
|
|
40
|
+
module = _LAZY.get(name)
|
|
41
|
+
if module is None:
|
|
42
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
43
|
+
value = getattr(importlib.import_module(module), name)
|
|
44
|
+
globals()[name] = value # bind so this runs once per name
|
|
45
|
+
return value
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def __dir__() -> list[str]:
|
|
49
|
+
return sorted(__all__)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"DIM_FRAME",
|
|
54
|
+
"DIM_GEOM",
|
|
55
|
+
"Backbone",
|
|
56
|
+
"ClipStore",
|
|
57
|
+
"ClipTooShort",
|
|
58
|
+
"HolisticExtractor",
|
|
59
|
+
"IncludeData",
|
|
60
|
+
"LabelMap",
|
|
61
|
+
"Prediction",
|
|
62
|
+
"RawFrame",
|
|
63
|
+
"SignClassifier",
|
|
64
|
+
"SignRecogniser",
|
|
65
|
+
"accuracy",
|
|
66
|
+
"animate_replay",
|
|
67
|
+
"build_dataset",
|
|
68
|
+
"build_model",
|
|
69
|
+
"confusion_matrix",
|
|
70
|
+
"decode_prediction",
|
|
71
|
+
"describe_device",
|
|
72
|
+
"encode_clip",
|
|
73
|
+
"encode_frame",
|
|
74
|
+
"freeze_backbone",
|
|
75
|
+
"load_include",
|
|
76
|
+
"per_class_f1",
|
|
77
|
+
"pick_device",
|
|
78
|
+
"plot_confusion",
|
|
79
|
+
"plot_curves",
|
|
80
|
+
"plot_f1_distribution",
|
|
81
|
+
"pooled_stats",
|
|
82
|
+
"reconstruct",
|
|
83
|
+
"replace_head",
|
|
84
|
+
"set_seed",
|
|
85
|
+
"still_replay",
|
|
86
|
+
"stratified_folds",
|
|
87
|
+
"top_confusions",
|
|
88
|
+
]
|