islkit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. islkit-0.1.0/.gitignore +14 -0
  2. islkit-0.1.0/LICENSE +21 -0
  3. islkit-0.1.0/PKG-INFO +200 -0
  4. islkit-0.1.0/README.md +173 -0
  5. islkit-0.1.0/docs/README.md +21 -0
  6. islkit-0.1.0/experiments/README.md +55 -0
  7. islkit-0.1.0/pyproject.toml +65 -0
  8. islkit-0.1.0/src/islkit/__init__.py +88 -0
  9. islkit-0.1.0/src/islkit/adapters.py +237 -0
  10. islkit-0.1.0/src/islkit/baseline.py +287 -0
  11. islkit-0.1.0/src/islkit/data.py +320 -0
  12. islkit-0.1.0/src/islkit/device.py +38 -0
  13. islkit-0.1.0/src/islkit/domain.py +187 -0
  14. islkit-0.1.0/src/islkit/features.py +323 -0
  15. islkit-0.1.0/src/islkit/infer.py +911 -0
  16. islkit-0.1.0/src/islkit/labels.py +213 -0
  17. islkit-0.1.0/src/islkit/metrics.py +81 -0
  18. islkit-0.1.0/src/islkit/model.py +623 -0
  19. islkit-0.1.0/src/islkit/pipeline.py +717 -0
  20. islkit-0.1.0/src/islkit/plotting.py +131 -0
  21. islkit-0.1.0/src/islkit/seeding.py +19 -0
  22. islkit-0.1.0/src/islkit/server.py +246 -0
  23. islkit-0.1.0/src/islkit/view.py +287 -0
  24. islkit-0.1.0/src/islkit/viz.py +435 -0
  25. islkit-0.1.0/tests/fixtures/recognition-events-v1.jsonl +13 -0
  26. islkit-0.1.0/tests/test_baseline.py +219 -0
  27. islkit-0.1.0/tests/test_data.py +236 -0
  28. islkit-0.1.0/tests/test_domain.py +180 -0
  29. islkit-0.1.0/tests/test_features.py +325 -0
  30. islkit-0.1.0/tests/test_infer.py +952 -0
  31. islkit-0.1.0/tests/test_islkit.py +91 -0
  32. islkit-0.1.0/tests/test_labels.py +137 -0
  33. islkit-0.1.0/tests/test_model.py +490 -0
  34. islkit-0.1.0/tests/test_pipeline.py +855 -0
  35. islkit-0.1.0/tests/test_server.py +416 -0
  36. islkit-0.1.0/tests/test_view.py +220 -0
  37. islkit-0.1.0/tests/test_viz.py +149 -0
@@ -0,0 +1,14 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ # Figures and summaries under runs/ are committed; weights and logs are not.
10
+ *.log
11
+ data/
12
+ *.pt
13
+ *.onnx
14
+ .DS_Store
islkit-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Jayraj
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
islkit-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,200 @@
1
+ Metadata-Version: 2.5
2
+ Name: islkit
3
+ Version: 0.1.0
4
+ Summary: Indian Sign Language recognition from MediaPipe landmarks: features, a dual-branch TCN, and a live recognition service
5
+ Project-URL: Homepage, https://github.com/jbrathwa/islkit
6
+ Project-URL: Issues, https://github.com/jbrathwa/islkit/issues
7
+ Author: Jayraj
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: accessibility,isl,landmarks,mediapipe,sign-language,tcn
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Classifier: Topic :: Scientific/Engineering :: Image Recognition
18
+ Requires-Python: <3.13,>=3.12
19
+ Requires-Dist: mediapipe==0.10.18
20
+ Requires-Dist: numpy<2,>=1.26
21
+ Requires-Dist: torch>=2.5
22
+ Provides-Extra: train
23
+ Requires-Dist: pandas>=3.0.5; extra == 'train'
24
+ Requires-Dist: pyarrow>=25.0.1; extra == 'train'
25
+ Requires-Dist: xgboost>=3.4.1; extra == 'train'
26
+ Description-Content-Type: text/markdown
27
+
28
+ # islkit
29
+
30
+ Indian Sign Language (ISL) recognition from MediaPipe Holistic landmarks.
31
+
32
+ islkit turns camera frames into glosses. It covers every step: a landmark
33
+ feature encoder, a dual-branch temporal convolutional network (TCN), a loader
34
+ for the INCLUDE dataset, live inference with a confidence-gated decline, and a
35
+ small HTTP/SSE recognition service. It runs on CPU and is built to work on small
36
+ ARM boards as well as laptops.
37
+
38
+ ```
39
+ camera → MediaPipe Holistic → RawFrame → encode_clip (T×352) → TCN → gloss | None
40
+ ```
41
+
42
+ ## Install
43
+
44
+ ```sh
45
+ pip install islkit # inference and the recognition service
46
+ pip install "islkit[train]" # + pandas, pyarrow, xgboost for INCLUDE and the baseline
47
+ ```
48
+
49
+ Requires **Python 3.12**. `mediapipe` is pinned to `0.10.18`, which also pins
50
+ `numpy<2`. That is the last MediaPipe release that keeps `mp.solutions` and
51
+ still runs on ARMv8.0-A (Cortex-A53) boards.
52
+
53
+ On macOS, XGBoost needs `brew install libomp`. Don't import `torch` and
54
+ `xgboost` in the same process: each bundles its own libomp, and the process
55
+ aborts with `OMP: Error #15`. `import islkit` keeps torch lazy for this reason.
56
+
57
+ ## Design
58
+
59
+ **Features, not raw landmarks.** Holistic's flattened output has 1,662 values,
60
+ and 85% of them are face mesh. The encoder reduces each frame to **183 floats**
61
+ (352 with velocity):
62
+
63
+ | Block | Dims |
64
+ |--------------------------------|------|
65
+ | Hand-local shapes (2 × 21 × 3) | 126 |
66
+ | Wrist positions in body frame | 6 |
67
+ | Upper-body pose (11 landmarks) | 33 |
68
+ | Non-manual scalars from face | 4 |
69
+ | Validity mask | 14 |
70
+
71
+ The encoder uses three coordinate frames rather than one normalisation. Body
72
+ frame: origin at the shoulder midpoint, scaled by shoulder width. Hand-local
73
+ frame: origin at the wrist, scaled by the wrist-to-middle-MCP distance. Wrist
74
+ position is recovered separately in the body frame. This keeps handshape
75
+ separate from location.
76
+
77
+ **Principles the code enforces:**
78
+
79
+ - **Hand slots are geometric.** Slot 0 is the hand nearer the dominant-side
80
+ shoulder. MediaPipe's handedness label flips under occlusion and is never read.
81
+ - **Mask, never zero-fill.** A missing hand is not a hand at the origin. A
82
+ validity bit is carried per part and multiplied through the network.
83
+ - **The label map is frozen.** `LabelMap` is saved beside the weights. Class
84
+ order rebuilt from a directory listing silently shifts indices.
85
+ - **Checkpoints are tied to the encoder.** A checkpoint records an encoder
86
+ fingerprint, and `SignRecogniser` refuses to serve it through a different
87
+ encoder.
88
+ - **Store raw landmarks.** Recordings hold raw MediaPipe output, so they can
89
+ be re-encoded when the normalisation changes.
90
+ - **The model can say "I don't know".** Below the confidence threshold a
91
+ prediction is `None`. For an accessibility device, silence is better than a
92
+ confident wrong answer.
93
+
94
+ ## Usage
95
+
96
+ ### Encode a clip
97
+
98
+ ```python
99
+ from islkit import encode_clip
100
+ from islkit.infer import HolisticExtractor
101
+
102
+ frames = []
103
+ with HolisticExtractor() as extractor:
104
+ for frame_bgr in video_frames: # BGR numpy arrays, e.g. from cv2
105
+ _, raw, _ = extractor.process(frame_bgr)
106
+ frames.append(raw)
107
+
108
+ clip = encode_clip(frames, T=48) # (48, 352) float32
109
+ ```
110
+
111
+ ### Recognise a sign
112
+
113
+ ```python
114
+ from islkit import SignRecogniser
115
+
116
+ recogniser = SignRecogniser("classifier.pt", threshold=0.6) # labels_*.json beside it
117
+ prediction = recogniser.classify(frames)
118
+ print(prediction.gloss, prediction.confidence, prediction.top3)
119
+ ```
120
+
121
+ `prediction.gloss` is `None` when the model declines.
122
+
123
+ ### Train
124
+
125
+ ```python
126
+ from islkit import build_model, load_include
127
+ from islkit.model import fit
128
+
129
+ data = load_include("path/to/isl-mediapipe-holistic-landmarks") # data.X: (N, 48, 352)
130
+ model = build_model(n_classes=len(data.label_map))
131
+ fit(model, data.X, data.y)
132
+ ```
133
+
134
+ `load_include` reads the Kaggle
135
+ [`indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
136
+ dump of INCLUDE and caches the encoded arrays. That dump records no signer or
137
+ session, so a class-stratified k-fold over whole clips is the only honest split
138
+ available. It still leaks signers, so **report numbers from it as optimistic**.
139
+ For your own recordings, use leave-one-session-out: `build_dataset` returns a
140
+ `sessions` array for this.
141
+
142
+ `replace_head` and `freeze_backbone` support fine-tuning a pretrained backbone
143
+ on a smaller vocabulary. Always report per-class F1 (`per_class_f1`), not just
144
+ aggregate accuracy.
145
+
146
+ ### Run the recognition service
147
+
148
+ ```python
149
+ import threading
150
+ from islkit.infer import ClipStore, HolisticExtractor, SignRecogniser
151
+ from islkit.pipeline import CameraSource, RecognitionPipeline
152
+ from islkit.server import EventHub, make_server
153
+
154
+ recogniser = SignRecogniser("classifier.pt")
155
+ hub = EventHub()
156
+ pipeline = RecognitionPipeline(
157
+ recogniser=recogniser,
158
+ source_factory=lambda: CameraSource(0, "1280x720"),
159
+ on_event=hub.publish,
160
+ extractor_factory=HolisticExtractor,
161
+ store=ClipStore(),
162
+ on_pause=hub.clear_take,
163
+ )
164
+ threading.Thread(target=pipeline.run, daemon=True).start()
165
+ make_server(pipeline, hub).serve_forever() # 127.0.0.1:9978
166
+ ```
167
+
168
+ | Route | Does |
169
+ |-----------------|----------------------------------------|
170
+ | `GET /results` | Server-sent events: tracking, takes, predictions |
171
+ | `POST /capture` | Start or pause capture |
172
+ | `GET /health` | Pipeline and subscriber status |
173
+
174
+ `islkit.view.make_view_server` serves an optional annotated MJPEG debug view
175
+ on a separate port.
176
+
177
+ Pretrained weights are not shipped with the package.
178
+
179
+ ## Guides and experiments
180
+
181
+ - [`docs/`](docs/README.md): how the features, dataset, baseline, pretraining and fine-tuning
182
+ work, each argued from committed results.
183
+ - [`experiments/`](experiments/README.md): the scripts that produce those results, runnable from
184
+ a clone of this repository.
185
+
186
+ ## Development
187
+
188
+ ```sh
189
+ uv sync --extra train
190
+ uv run pytest
191
+ uv run ruff check .
192
+ ```
193
+
194
+ The tests cover the properties that would silently break recognition: encoder
195
+ invariances, mask gating, label-map and checkpoint round-trips, and the
196
+ saved-clip layout.
197
+
198
+ ## License
199
+
200
+ MIT. See [LICENSE](LICENSE).
islkit-0.1.0/README.md ADDED
@@ -0,0 +1,173 @@
1
+ # islkit
2
+
3
+ Indian Sign Language (ISL) recognition from MediaPipe Holistic landmarks.
4
+
5
+ islkit turns camera frames into glosses. It covers every step: a landmark
6
+ feature encoder, a dual-branch temporal convolutional network (TCN), a loader
7
+ for the INCLUDE dataset, live inference with a confidence-gated decline, and a
8
+ small HTTP/SSE recognition service. It runs on CPU and is built to work on small
9
+ ARM boards as well as laptops.
10
+
11
+ ```
12
+ camera → MediaPipe Holistic → RawFrame → encode_clip (T×352) → TCN → gloss | None
13
+ ```
14
+
15
+ ## Install
16
+
17
+ ```sh
18
+ pip install islkit # inference and the recognition service
19
+ pip install "islkit[train]" # + pandas, pyarrow, xgboost for INCLUDE and the baseline
20
+ ```
21
+
22
+ Requires **Python 3.12**. `mediapipe` is pinned to `0.10.18`, which also pins
23
+ `numpy<2`. That is the last MediaPipe release that keeps `mp.solutions` and
24
+ still runs on ARMv8.0-A (Cortex-A53) boards.
25
+
26
+ On macOS, XGBoost needs `brew install libomp`. Don't import `torch` and
27
+ `xgboost` in the same process: each bundles its own libomp, and the process
28
+ aborts with `OMP: Error #15`. `import islkit` keeps torch lazy for this reason.
29
+
30
+ ## Design
31
+
32
+ **Features, not raw landmarks.** Holistic's flattened output has 1,662 values,
33
+ and 85% of them are face mesh. The encoder reduces each frame to **183 floats**
34
+ (352 with velocity):
35
+
36
+ | Block | Dims |
37
+ |--------------------------------|------|
38
+ | Hand-local shapes (2 × 21 × 3) | 126 |
39
+ | Wrist positions in body frame | 6 |
40
+ | Upper-body pose (11 landmarks) | 33 |
41
+ | Non-manual scalars from face | 4 |
42
+ | Validity mask | 14 |
43
+
44
+ The encoder uses three coordinate frames rather than one normalisation. Body
45
+ frame: origin at the shoulder midpoint, scaled by shoulder width. Hand-local
46
+ frame: origin at the wrist, scaled by the wrist-to-middle-MCP distance. Wrist
47
+ position is recovered separately in the body frame. This keeps handshape
48
+ separate from location.
49
+
50
+ **Principles the code enforces:**
51
+
52
+ - **Hand slots are geometric.** Slot 0 is the hand nearer the dominant-side
53
+ shoulder. MediaPipe's handedness label flips under occlusion and is never read.
54
+ - **Mask, never zero-fill.** A missing hand is not a hand at the origin. A
55
+ validity bit is carried per part and multiplied through the network.
56
+ - **The label map is frozen.** `LabelMap` is saved beside the weights. Class
57
+ order rebuilt from a directory listing silently shifts indices.
58
+ - **Checkpoints are tied to the encoder.** A checkpoint records an encoder
59
+ fingerprint, and `SignRecogniser` refuses to serve it through a different
60
+ encoder.
61
+ - **Store raw landmarks.** Recordings hold raw MediaPipe output, so they can
62
+ be re-encoded when the normalisation changes.
63
+ - **The model can say "I don't know".** Below the confidence threshold a
64
+ prediction is `None`. For an accessibility device, silence is better than a
65
+ confident wrong answer.
66
+
67
+ ## Usage
68
+
69
+ ### Encode a clip
70
+
71
+ ```python
72
+ from islkit import encode_clip
73
+ from islkit.infer import HolisticExtractor
74
+
75
+ frames = []
76
+ with HolisticExtractor() as extractor:
77
+ for frame_bgr in video_frames: # BGR numpy arrays, e.g. from cv2
78
+ _, raw, _ = extractor.process(frame_bgr)
79
+ frames.append(raw)
80
+
81
+ clip = encode_clip(frames, T=48) # (48, 352) float32
82
+ ```
83
+
84
+ ### Recognise a sign
85
+
86
+ ```python
87
+ from islkit import SignRecogniser
88
+
89
+ recogniser = SignRecogniser("classifier.pt", threshold=0.6) # labels_*.json beside it
90
+ prediction = recogniser.classify(frames)
91
+ print(prediction.gloss, prediction.confidence, prediction.top3)
92
+ ```
93
+
94
+ `prediction.gloss` is `None` when the model declines.
95
+
96
+ ### Train
97
+
98
+ ```python
99
+ from islkit import build_model, load_include
100
+ from islkit.model import fit
101
+
102
+ data = load_include("path/to/isl-mediapipe-holistic-landmarks") # data.X: (N, 48, 352)
103
+ model = build_model(n_classes=len(data.label_map))
104
+ fit(model, data.X, data.y)
105
+ ```
106
+
107
+ `load_include` reads the Kaggle
108
+ [`indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
109
+ dump of INCLUDE and caches the encoded arrays. That dump records no signer or
110
+ session, so a class-stratified k-fold over whole clips is the only honest split
111
+ available. It still leaks signers, so **report numbers from it as optimistic**.
112
+ For your own recordings, use leave-one-session-out: `build_dataset` returns a
113
+ `sessions` array for this.
114
+
115
+ `replace_head` and `freeze_backbone` support fine-tuning a pretrained backbone
116
+ on a smaller vocabulary. Always report per-class F1 (`per_class_f1`), not just
117
+ aggregate accuracy.
118
+
119
+ ### Run the recognition service
120
+
121
+ ```python
122
+ import threading
123
+ from islkit.infer import ClipStore, HolisticExtractor, SignRecogniser
124
+ from islkit.pipeline import CameraSource, RecognitionPipeline
125
+ from islkit.server import EventHub, make_server
126
+
127
+ recogniser = SignRecogniser("classifier.pt")
128
+ hub = EventHub()
129
+ pipeline = RecognitionPipeline(
130
+ recogniser=recogniser,
131
+ source_factory=lambda: CameraSource(0, "1280x720"),
132
+ on_event=hub.publish,
133
+ extractor_factory=HolisticExtractor,
134
+ store=ClipStore(),
135
+ on_pause=hub.clear_take,
136
+ )
137
+ threading.Thread(target=pipeline.run, daemon=True).start()
138
+ make_server(pipeline, hub).serve_forever() # 127.0.0.1:9978
139
+ ```
140
+
141
+ | Route | Does |
142
+ |-----------------|----------------------------------------|
143
+ | `GET /results` | Server-sent events: tracking, takes, predictions |
144
+ | `POST /capture` | Start or pause capture |
145
+ | `GET /health` | Pipeline and subscriber status |
146
+
147
+ `islkit.view.make_view_server` serves an optional annotated MJPEG debug view
148
+ on a separate port.
149
+
150
+ Pretrained weights are not shipped with the package.
151
+
152
+ ## Guides and experiments
153
+
154
+ - [`docs/`](docs/README.md): how the features, dataset, baseline, pretraining and fine-tuning
155
+ work, each argued from committed results.
156
+ - [`experiments/`](experiments/README.md): the scripts that produce those results, runnable from
157
+ a clone of this repository.
158
+
159
+ ## Development
160
+
161
+ ```sh
162
+ uv sync --extra train
163
+ uv run pytest
164
+ uv run ruff check .
165
+ ```
166
+
167
+ The tests cover the properties that would silently break recognition: encoder
168
+ invariances, mask gating, label-map and checkpoint round-trips, and the
169
+ saved-clip layout.
170
+
171
+ ## License
172
+
173
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,21 @@
1
+ # islkit guides
2
+
3
+ Each guide explains one part of the pipeline and argues from results committed under
4
+ [`runs/`](../runs), produced by the scripts in [`experiments/`](../experiments). Every number can be
5
+ regenerated with the command next to it.
6
+
7
+ | Guide | Covers |
8
+ |---|---|
9
+ | [Landmark features](features.md) | The 183-dim encoder, its invariances, and the replay instrument that checks them by eye |
10
+ | [The INCLUDE dataset](include-dataset.md) | Layout, the missing signer identifier, tracking quality, and the replay on real landmarks |
11
+ | [Baseline](baseline.md) | Pooled statistics + XGBoost: the floor a sequence model has to clear |
12
+ | [Pretraining](pretraining.md) | The dual-branch TCN on INCLUDE, its controls, and why its accuracy is not a device accuracy |
13
+ | [Fine-tuning](fine-tuning.md) | Choosing a vocabulary, two-stage fine-tuning on your own recordings, and training for a device camera |
14
+
15
+ ## One caveat that applies everywhere
16
+
17
+ The public INCLUDE landmark dump does not record who signed each clip, so no split can hold out
18
+ signers. Every INCLUDE number in these guides comes from a class-stratified k-fold over whole clips,
19
+ which is **optimistic**: the same signers appear on both sides of every fold. They show that the
20
+ features and models work; they do not estimate accuracy for a new signer. See
21
+ [the dataset guide](include-dataset.md#2-there-is-no-signer-identifier).
@@ -0,0 +1,55 @@
1
+ # Experiments
2
+
3
+ Runnable scripts that exercise `islkit` end to end. Each one writes to `runs/<experiment>/`, and
4
+ the guides in [`docs/`](../docs) argue from those outputs. Figures and summaries are committed;
5
+ weights (`*.pt`, `*.onnx`) and logs are not.
6
+
7
+ ## Setup
8
+
9
+ ```sh
10
+ uv sync --extra train # pandas, pyarrow and xgboost for the INCLUDE experiments
11
+ export ISLKIT_INCLUDE_ROOT=/path/to/isl-mediapipe-holistic-landmarks
12
+ ```
13
+
14
+ Run every script from the repository root, so `runs/` and the `data/cache/` of encoded INCLUDE
15
+ clips land in the right place. The INCLUDE dump is the Kaggle dataset
16
+ [`swaptr/indian-sign-language-mediapipe-holistic-landmarks`](https://www.kaggle.com/datasets/swaptr/indian-sign-language-mediapipe-holistic-landmarks)
17
+ (3.4 GB). The first script that loads it encodes all 4,284 clips into a 291 MB cache.
18
+
19
+ On macOS, XGBoost needs `brew install libomp`. torch and XGBoost each bundle their own OpenMP
20
+ runtime and abort when loaded into one process, which is why the baseline and the TCN are separate
21
+ scripts.
22
+
23
+ ## The scripts
24
+
25
+ | Script | Needs | Writes | Guide |
26
+ |---|---|---|---|
27
+ | `feature_smoke.py` | nothing | stdout only | [features](../docs/features.md#4-cost) |
28
+ | `replay_features.py` | nothing | `runs/replay_features/` | [features](../docs/features.md#5-seeing-the-features-the-replay-instrument) |
29
+ | `tcn_gestures.py` | nothing | `runs/tcn_gestures/` | below |
30
+ | `inspect_include.py` | INCLUDE | `runs/inspect_include/` | [INCLUDE](../docs/include-dataset.md) |
31
+ | `replay_include.py` | INCLUDE | `runs/replay_include/` | [INCLUDE](../docs/include-dataset.md#4-the-replay-on-real-landmarks) |
32
+ | `baseline_xgb.py` | INCLUDE | `runs/baseline_xgb/` | [baseline](../docs/baseline.md) |
33
+ | `pretrain.py` | INCLUDE, baseline summary | `runs/pretrain/` | [pretraining](../docs/pretraining.md) |
34
+ | `vocab_confusability.py` | INCLUDE | `data/vocab.json` | [fine-tuning](../docs/fine-tuning.md#2-choose-the-vocabulary-before-recording-it) |
35
+ | `finetune.py` | INCLUDE, own recordings, pretrained classifier | `runs/finetune/` | [fine-tuning](../docs/fine-tuning.md#4-two-stage-fine-tuning) |
36
+ | `device_head.py` | INCLUDE, pretrained backbone, a vocabulary | `runs/device_head/` | [fine-tuning](../docs/fine-tuning.md#5-training-for-the-devices-camera) |
37
+
38
+ Each script's docstring lists its flags; `--help` prints them. `pretrain.py`, `finetune.py` and
39
+ `device_head.py` take `--smoke` for a fast wiring check.
40
+
41
+ Run them in table order the first time: `pretrain.py` reads its bar from
42
+ `runs/baseline_xgb/summary.json`, and the fine-tuning scripts load pretraining weights.
43
+
44
+ ## A first TCN: `tcn_gestures.py`
45
+
46
+ A 30-second, data-free introduction to temporal convolutional networks. It fabricates six
47
+ "signs" as noisy sine trajectories over 32 features, gives each take a recording session, holds out
48
+ an entire session, and trains a small residual TCN.
49
+
50
+ ![Train vs unseen-session accuracy on synthetic gestures](../runs/tcn_gestures/curves.png)
51
+
52
+ Training accuracy reaches ~99% while accuracy on the held-out session levels off near 78%. That gap
53
+ is the point: takes from one session share their noise and timing, so a random split over takes
54
+ would put near-duplicates on both sides and report something much closer to the training curve.
55
+ Hold out whole sessions whenever your data has them.
@@ -0,0 +1,65 @@
1
+ [project]
2
+ name = "islkit"
3
+ version = "0.1.0"
4
+ description = "Indian Sign Language recognition from MediaPipe landmarks: features, a dual-branch TCN, and a live recognition service"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [{ name = "Jayraj" }]
9
+ requires-python = ">=3.12,<3.13"
10
+ keywords = ["sign-language", "isl", "mediapipe", "landmarks", "tcn", "accessibility"]
11
+ classifiers = [
12
+ "Development Status :: 3 - Alpha",
13
+ "Intended Audience :: Science/Research",
14
+ "Operating System :: OS Independent",
15
+ "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: 3.12",
17
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
18
+ "Topic :: Scientific/Engineering :: Image Recognition",
19
+ ]
20
+ urls = { Homepage = "https://github.com/jbrathwa/islkit", Issues = "https://github.com/jbrathwa/islkit/issues" }
21
+ dependencies = [
22
+ "torch>=2.5",
23
+ # 0.10.18 is pinned exactly and pins numpy<2 behind it. It is the last release
24
+ # that keeps mp.solutions (1.0.x removed it), and 1.0.x's aarch64 wheel needs
25
+ # ARMv8.1-A LSE atomics, so it will not run on Cortex-A53 boards.
26
+ "mediapipe==0.10.18",
27
+ "numpy>=1.26,<2",
28
+ ]
29
+
30
+ # cv2 is deliberately not declared: mediapipe hard-requires opencv-contrib-python,
31
+ # and adding another OpenCV package on top puts two providers of one module in the
32
+ # environment. matplotlib likewise arrives with mediapipe.
33
+ [project.optional-dependencies]
34
+ train = [
35
+ "pandas>=3.0.5",
36
+ "pyarrow>=25.0.1",
37
+ "xgboost>=3.4.1",
38
+ ]
39
+
40
+ [dependency-groups]
41
+ dev = [
42
+ "pytest>=8.0",
43
+ "ruff>=0.6",
44
+ ]
45
+
46
+ [build-system]
47
+ requires = ["hatchling>=1.27"]
48
+ build-backend = "hatchling.build"
49
+
50
+ [tool.hatch.build.targets.wheel]
51
+ packages = ["src/islkit"]
52
+
53
+ [tool.hatch.build.targets.sdist]
54
+ include = ["src/islkit", "tests", "LICENSE", "README.md"]
55
+
56
+ [tool.ruff]
57
+ line-length = 100
58
+ src = ["src", "tests", "experiments"]
59
+
60
+ [tool.ruff.lint]
61
+ select = ["E", "F", "I", "UP", "B"]
62
+
63
+ [tool.pytest.ini_options]
64
+ testpaths = ["tests"]
65
+ addopts = "-q"
@@ -0,0 +1,88 @@
1
+ """islkit — Indian Sign Language recognition from MediaPipe landmarks.
2
+
3
+ Landmark features, a dual-branch TCN, the INCLUDE loader, live inference and a
4
+ small HTTP/SSE recognition service.
5
+ """
6
+
7
+ import importlib
8
+
9
+ from islkit.baseline import pooled_stats, stratified_folds
10
+ from islkit.data import IncludeData, load_include
11
+ from islkit.features import DIM_FRAME, DIM_GEOM, RawFrame, encode_clip, encode_frame
12
+ from islkit.labels import LabelMap, build_dataset, decode_prediction
13
+ from islkit.metrics import accuracy, confusion_matrix, per_class_f1, top_confusions
14
+ from islkit.plotting import plot_confusion, plot_curves, plot_f1_distribution
15
+ from islkit.viz import animate_replay, reconstruct, still_replay
16
+
17
+ # device, seeding and model are the only modules that import torch, and they are
18
+ # loaded on first use rather than at import time. Not for speed: torch bundles its
19
+ # own libomp, XGBoost links Homebrew's, and macOS aborts the process when both land
20
+ # in it (OMP Error #15). The XGBoost baseline is pure numpy and must be able to import
21
+ # load_include without dragging torch in behind it. See islkit/baseline.py.
22
+ _LAZY = {
23
+ "describe_device": "islkit.device",
24
+ "pick_device": "islkit.device",
25
+ "set_seed": "islkit.seeding",
26
+ "Backbone": "islkit.model",
27
+ "SignClassifier": "islkit.model",
28
+ "build_model": "islkit.model",
29
+ "freeze_backbone": "islkit.model",
30
+ "replace_head": "islkit.model",
31
+ "ClipStore": "islkit.infer",
32
+ "ClipTooShort": "islkit.infer",
33
+ "HolisticExtractor": "islkit.infer",
34
+ "Prediction": "islkit.infer",
35
+ "SignRecogniser": "islkit.infer",
36
+ }
37
+
38
+
39
+ def __getattr__(name: str):
40
+ module = _LAZY.get(name)
41
+ if module is None:
42
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
43
+ value = getattr(importlib.import_module(module), name)
44
+ globals()[name] = value # bind so this runs once per name
45
+ return value
46
+
47
+
48
+ def __dir__() -> list[str]:
49
+ return sorted(__all__)
50
+
51
+
52
+ __all__ = [
53
+ "DIM_FRAME",
54
+ "DIM_GEOM",
55
+ "Backbone",
56
+ "ClipStore",
57
+ "ClipTooShort",
58
+ "HolisticExtractor",
59
+ "IncludeData",
60
+ "LabelMap",
61
+ "Prediction",
62
+ "RawFrame",
63
+ "SignClassifier",
64
+ "SignRecogniser",
65
+ "accuracy",
66
+ "animate_replay",
67
+ "build_dataset",
68
+ "build_model",
69
+ "confusion_matrix",
70
+ "decode_prediction",
71
+ "describe_device",
72
+ "encode_clip",
73
+ "encode_frame",
74
+ "freeze_backbone",
75
+ "load_include",
76
+ "per_class_f1",
77
+ "pick_device",
78
+ "plot_confusion",
79
+ "plot_curves",
80
+ "plot_f1_distribution",
81
+ "pooled_stats",
82
+ "reconstruct",
83
+ "replace_head",
84
+ "set_seed",
85
+ "still_replay",
86
+ "stratified_folds",
87
+ "top_confusions",
88
+ ]