project-away 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. project_away-0.1.1/PKG-INFO +406 -0
  2. project_away-0.1.1/README.md +345 -0
  3. project_away-0.1.1/pyproject.toml +114 -0
  4. project_away-0.1.1/setup.cfg +4 -0
  5. project_away-0.1.1/src/away/__init__.py +4 -0
  6. project_away-0.1.1/src/away/__main__.py +3 -0
  7. project_away-0.1.1/src/away/actions.py +329 -0
  8. project_away-0.1.1/src/away/app/__init__.py +1 -0
  9. project_away-0.1.1/src/away/app/autopause.py +119 -0
  10. project_away-0.1.1/src/away/app/autostart.py +83 -0
  11. project_away-0.1.1/src/away/app/cursor_overlay.py +110 -0
  12. project_away-0.1.1/src/away/app/frozen.py +175 -0
  13. project_away-0.1.1/src/away/app/models.py +87 -0
  14. project_away-0.1.1/src/away/app/overlay.py +61 -0
  15. project_away-0.1.1/src/away/app/runner.py +723 -0
  16. project_away-0.1.1/src/away/app/selftest.py +194 -0
  17. project_away-0.1.1/src/away/app/settings_gui.py +1202 -0
  18. project_away-0.1.1/src/away/app/setup_wizard.py +130 -0
  19. project_away-0.1.1/src/away/app/tray.py +197 -0
  20. project_away-0.1.1/src/away/app/uninstall.py +56 -0
  21. project_away-0.1.1/src/away/bindings.py +548 -0
  22. project_away-0.1.1/src/away/cli.py +747 -0
  23. project_away-0.1.1/src/away/config.py +411 -0
  24. project_away-0.1.1/src/away/control/__init__.py +1 -0
  25. project_away-0.1.1/src/away/control/keys.py +245 -0
  26. project_away-0.1.1/src/away/control/mouse.py +237 -0
  27. project_away-0.1.1/src/away/data/gesture_tcn.json +15 -0
  28. project_away-0.1.1/src/away/data/gesture_tcn.metrics.json +18 -0
  29. project_away-0.1.1/src/away/data/gesture_tcn.onnx +0 -0
  30. project_away-0.1.1/src/away/data/static_pose.metrics.json +77 -0
  31. project_away-0.1.1/src/away/data/static_pose.npz +0 -0
  32. project_away-0.1.1/src/away/events.py +67 -0
  33. project_away-0.1.1/src/away/face/__init__.py +1 -0
  34. project_away-0.1.1/src/away/face/commands.py +101 -0
  35. project_away-0.1.1/src/away/face/gate.py +65 -0
  36. project_away-0.1.1/src/away/face/landmarker.py +48 -0
  37. project_away-0.1.1/src/away/face/speaker.py +74 -0
  38. project_away-0.1.1/src/away/face/speaking.py +50 -0
  39. project_away-0.1.1/src/away/face/tracking.py +52 -0
  40. project_away-0.1.1/src/away/fusion.py +90 -0
  41. project_away-0.1.1/src/away/hands/__init__.py +1 -0
  42. project_away-0.1.1/src/away/hands/cursor.py +128 -0
  43. project_away-0.1.1/src/away/hands/dataset.py +171 -0
  44. project_away-0.1.1/src/away/hands/filters.py +71 -0
  45. project_away-0.1.1/src/away/hands/gestures.py +196 -0
  46. project_away-0.1.1/src/away/hands/landmarks.py +135 -0
  47. project_away-0.1.1/src/away/hands/model.py +158 -0
  48. project_away-0.1.1/src/away/hands/recorder.py +86 -0
  49. project_away-0.1.1/src/away/hands/runtime.py +67 -0
  50. project_away-0.1.1/src/away/hands/stabilize.py +110 -0
  51. project_away-0.1.1/src/away/hands/synthetic.py +192 -0
  52. project_away-0.1.1/src/away/hands/tracker.py +98 -0
  53. project_away-0.1.1/src/away/hands/train.py +220 -0
  54. project_away-0.1.1/src/away/multicam/__init__.py +1 -0
  55. project_away-0.1.1/src/away/multicam/calibration.py +208 -0
  56. project_away-0.1.1/src/away/multicam/capture.py +232 -0
  57. project_away-0.1.1/src/away/multicam/fusion.py +329 -0
  58. project_away-0.1.1/src/away/multicam/geometry.py +317 -0
  59. project_away-0.1.1/src/away/paths.py +54 -0
  60. project_away-0.1.1/src/away/py.typed +0 -0
  61. project_away-0.1.1/src/away/voice/__init__.py +1 -0
  62. project_away-0.1.1/src/away/voice/always_on.py +505 -0
  63. project_away-0.1.1/src/away/voice/asr.py +138 -0
  64. project_away-0.1.1/src/away/voice/cleanup.py +269 -0
  65. project_away-0.1.1/src/away/voice/context.py +89 -0
  66. project_away-0.1.1/src/away/voice/dictation.py +203 -0
  67. project_away-0.1.1/src/away/voice/edit.py +234 -0
  68. project_away-0.1.1/src/away/voice/numbers.py +150 -0
  69. project_away-0.1.1/src/away/voice/output.py +59 -0
  70. project_away-0.1.1/src/away/voice/vad.py +140 -0
  71. project_away-0.1.1/src/away/wifi/__init__.py +1 -0
  72. project_away-0.1.1/src/away/wifi/classifier.py +91 -0
  73. project_away-0.1.1/src/away/wifi/features.py +150 -0
  74. project_away-0.1.1/src/away/wifi/sources.py +243 -0
  75. project_away-0.1.1/src/project_away.egg-info/PKG-INFO +406 -0
  76. project_away-0.1.1/src/project_away.egg-info/SOURCES.txt +106 -0
  77. project_away-0.1.1/src/project_away.egg-info/dependency_links.txt +1 -0
  78. project_away-0.1.1/src/project_away.egg-info/entry_points.txt +5 -0
  79. project_away-0.1.1/src/project_away.egg-info/requires.txt +50 -0
  80. project_away-0.1.1/src/project_away.egg-info/top_level.txt +1 -0
  81. project_away-0.1.1/tests/test_always_on.py +394 -0
  82. project_away-0.1.1/tests/test_app.py +400 -0
  83. project_away-0.1.1/tests/test_bindings.py +323 -0
  84. project_away-0.1.1/tests/test_cleanup.py +186 -0
  85. project_away-0.1.1/tests/test_cli_controls.py +185 -0
  86. project_away-0.1.1/tests/test_config_paths.py +109 -0
  87. project_away-0.1.1/tests/test_cursor.py +112 -0
  88. project_away-0.1.1/tests/test_dispatch.py +322 -0
  89. project_away-0.1.1/tests/test_docs.py +17 -0
  90. project_away-0.1.1/tests/test_face.py +252 -0
  91. project_away-0.1.1/tests/test_filters.py +97 -0
  92. project_away-0.1.1/tests/test_frame_pairer_safety.py +66 -0
  93. project_away-0.1.1/tests/test_fusion_events.py +137 -0
  94. project_away-0.1.1/tests/test_gesture_lifecycle_safety.py +93 -0
  95. project_away-0.1.1/tests/test_gestures.py +231 -0
  96. project_away-0.1.1/tests/test_installer.py +327 -0
  97. project_away-0.1.1/tests/test_key_release_safety.py +125 -0
  98. project_away-0.1.1/tests/test_landmarks.py +99 -0
  99. project_away-0.1.1/tests/test_models_data.py +186 -0
  100. project_away-0.1.1/tests/test_multicam.py +463 -0
  101. project_away-0.1.1/tests/test_settings_model.py +255 -0
  102. project_away-0.1.1/tests/test_shutdown_safety.py +78 -0
  103. project_away-0.1.1/tests/test_spec_edges.py +458 -0
  104. project_away-0.1.1/tests/test_spec_edges2.py +210 -0
  105. project_away-0.1.1/tests/test_train_torch.py +98 -0
  106. project_away-0.1.1/tests/test_upgrades.py +437 -0
  107. project_away-0.1.1/tests/test_voice.py +241 -0
  108. project_away-0.1.1/tests/test_wifi.py +269 -0
@@ -0,0 +1,406 @@
1
+ Metadata-Version: 2.4
2
+ Name: project-away
3
+ Version: 0.1.1
4
+ Summary: Hands-free PC control: camera hand gestures, push-to-talk dictation, active-speaker gating and experimental Wi-Fi CSI sensing. Local-first.
5
+ Author-email: Fernando Garza <fernandogarzaaa@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/fernandogarzaaa/project-away
8
+ Keywords: gesture,hand-tracking,mediapipe,dictation,whisper,accessibility,wifi-csi
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Operating System :: Microsoft :: Windows
14
+ Classifier: Operating System :: MacOS
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Topic :: Desktop Environment
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+ Requires-Dist: numpy>=1.24
21
+ Requires-Dist: scipy>=1.10
22
+ Requires-Dist: PyYAML>=6.0
23
+ Provides-Extra: hands
24
+ Requires-Dist: mediapipe>=0.10.14; extra == "hands"
25
+ Requires-Dist: opencv-python>=4.8; extra == "hands"
26
+ Requires-Dist: onnxruntime>=1.17; extra == "hands"
27
+ Provides-Extra: face
28
+ Requires-Dist: mediapipe>=0.10.14; extra == "face"
29
+ Requires-Dist: opencv-python>=4.8; extra == "face"
30
+ Provides-Extra: voice
31
+ Requires-Dist: faster-whisper>=1.1; extra == "voice"
32
+ Requires-Dist: sounddevice>=0.4.6; extra == "voice"
33
+ Provides-Extra: control
34
+ Requires-Dist: pynput>=1.7.6; extra == "control"
35
+ Requires-Dist: pyperclip>=1.8; extra == "control"
36
+ Provides-Extra: tray
37
+ Requires-Dist: pystray>=0.19; extra == "tray"
38
+ Requires-Dist: Pillow>=10; extra == "tray"
39
+ Provides-Extra: train
40
+ Requires-Dist: torch>=2.1; extra == "train"
41
+ Requires-Dist: onnx>=1.15; extra == "train"
42
+ Requires-Dist: onnxruntime>=1.17; extra == "train"
43
+ Provides-Extra: parakeet
44
+ Requires-Dist: nemo_toolkit[asr]>=2.0; extra == "parakeet"
45
+ Provides-Extra: all
46
+ Requires-Dist: mediapipe>=0.10.14; extra == "all"
47
+ Requires-Dist: opencv-python>=4.8; extra == "all"
48
+ Requires-Dist: onnxruntime>=1.17; extra == "all"
49
+ Requires-Dist: faster-whisper>=1.1; extra == "all"
50
+ Requires-Dist: sounddevice>=0.4.6; extra == "all"
51
+ Requires-Dist: pynput>=1.7.6; extra == "all"
52
+ Requires-Dist: pyperclip>=1.8; extra == "all"
53
+ Requires-Dist: pystray>=0.19; extra == "all"
54
+ Requires-Dist: Pillow>=10; extra == "all"
55
+ Provides-Extra: dev
56
+ Requires-Dist: pytest>=7.4; extra == "dev"
57
+ Requires-Dist: pytest-cov>=4.1; extra == "dev"
58
+ Requires-Dist: ruff>=0.4; extra == "dev"
59
+ Requires-Dist: mypy>=1.8; extra == "dev"
60
+ Requires-Dist: types-PyYAML; extra == "dev"
61
+
62
+ # Project Away
63
+
64
+ Hands-free PC control, free and local. Webcam hand gestures drive the cursor, push-to-talk voice typing handles text, face cues gate who is dictating, and optional Wi-Fi CSI adds coarse motion confidence.
65
+
66
+ Nothing leaves your machine unless you point the optional LLM cleanup at an endpoint yourself.
67
+
68
+ ## Architecture
69
+
70
+ ```mermaid
71
+ flowchart LR
72
+ CAM[Webcam] --> HL[MediaPipe Hand Landmarker]
73
+ CAM2[Optional 2nd camera] --> PAIR[Threaded capture + timestamp pairing]
74
+ CAM --> PAIR
75
+ PAIR --> HL
76
+ HL --> MV[Multi-view fusion: confidence, triangulation, handover]
77
+ MV --> NORM[Normalize + One Euro filter]
78
+ NORM --> GSM[Gesture state machine + static/TCN models]
79
+ GSM --> STEADY[Steady click + sigmoid curve + dwell/push]
80
+ CAM --> FL[MediaPipe Face Landmarker]
81
+ FL --> ASD[Active speaker + dictation gate]
82
+ MIC[Microphone] --> VAD[Silero VAD]
83
+ VAD --> ASR[faster-whisper / Parakeet]
84
+ ASD --> ASR
85
+ ASR --> CLEAN[Rule cleanup / optional LLM]
86
+ WIFI[Wi-Fi CSI or RSSI] --> CSI[Hampel + Doppler features]
87
+ CSI --> WCLS[Coarse presence/swipe classifier]
88
+ STEADY --> FUS[Fusion + event bus]
89
+ GUARD[Physical mouse/keyboard guard] --> FUS
90
+ WCLS --> FUS
91
+ FUS --> MOUSE[Cursor / click / scroll]
92
+ CLEAN --> KB[Type into active app]
93
+ FUS --> RING[Cursor ring overlay]
94
+ ```
95
+
96
+ | Module | Role |
97
+ |---|---|
98
+ | `hands/` | landmarks, One Euro filter, cursor mapping (power or sigmoid curve, per-direction speed), gesture state machine, `stabilize.py` (steady click, dwell) |
99
+ | `multicam/` | `capture.py` (enumeration, camera threads, `FramePairer`), `geometry.py` (intrinsics, pose from hands, triangulation, checkerboard), `calibration.py`, `fusion.py` (per-view confidence, best-view/weighted fusion, 3D pinch, push) |
100
+ | `face/` | face landmarker, active speaker, brow / mouth / wink triggers |
101
+ | `voice/` | always-on + push-to-talk, ASR (vocabulary as prompt + `hotwords`), cleanup, `numbers.py`, `edit.py` (edit selected text) |
102
+ | `app/` | runner (pipeline + device threads), tray, settings GUI, overlays, `autopause.py` |
103
+ | `bindings.py` / `actions.py` | trigger -> action mapping and execution |
104
+
105
+ ## Quickstart
106
+
107
+ Full per-OS instructions: [docs/INSTALL.md](docs/INSTALL.md). Wi-Fi sensing: [docs/wifi-setup.md](docs/wifi-setup.md).
108
+
109
+ ```bash
110
+ pipx install ".[all]" # Python 3.10–3.12
111
+ away setup # downloads models, checks camera + mic
112
+ away demo # synthetic end-to-end check
113
+ away run # or: away tray ; away autostart on
114
+ ```
115
+
116
+ **One-click installers** (Releases page) bundle every feature: tracking, dictation, tray, settings
117
+ window and on-device training (CPU PyTorch). Parakeet/NeMo is the one exception (multi-GB,
118
+ NVIDIA-oriented; use `pipx install "project-away[all,parakeet]"`). Installers are ~0.4–0.8 GB; the models
119
+ (~1.6 GB, or ~0.5 GB with Whisper `small`) download during install on Windows and on first launch elsewhere.
120
+
121
+ - **Windows:** run the `setup.exe`. It's unsigned, so choose *More info → Run anyway*, then **Install**. Models,
122
+ start-at-sign-in, a desktop shortcut and launching the tray app are all on by default.
123
+ - **macOS:** open the `.dmg`, drag to Applications, then right-click → Open. The first launch downloads the
124
+ models with a notification. Allow Camera, Microphone, Accessibility and Input Monitoring.
125
+ - **Linux:** run the AppImage or install the `.deb`; the first launch downloads the models. Mouse and keyboard
126
+ control needs X11; Wayland is untested.
127
+
128
+ ## Customizing controls
129
+
130
+ Every control is a **trigger → action** binding you can change without touching code: from the
131
+ **Settings** window (tray → *Settings...*, or `away settings`), from the CLI, or in `config.yaml`
132
+ (`away config path` shows where it is). A running Away reloads the file within a second of a save.
133
+
134
+ ### Default bindings
135
+
136
+ | Trigger | Default action |
137
+ |---|---|
138
+ | `gesture:point` (index finger) | `mouse:move` (cursor) |
139
+ | `gesture:pinch` | `mouse:click` |
140
+ | `gesture:double_pinch` | `mouse:double_click` |
141
+ | `gesture:pinch_hold` | `mouse:drag` |
142
+ | `gesture:two_finger_scroll` | `mouse:scroll` |
143
+ | `gesture:open_palm` (hold) | `pause:toggle` (pause/resume gestures) |
144
+ | `gesture:swipe_left` / `swipe_right` | `key:left` / `key:right` |
145
+ | `face:brow_raise` (needs `face.brow_raise_toggle: true`) | `dictation:toggle` |
146
+ | `face:mouth_open` (needs `face.mouth_open_trigger: true`) | `none` |
147
+ | `face:wink_left` / `wink_right` (needs `face.wink_trigger: true`) | `mouse:click` / `mouse:right_click` |
148
+ | `gesture:dwell` (needs `hands.dwell.enabled: true`) | `mouse:click` |
149
+ | `gesture:push` (needs depth: two calibrated cameras, or `hands.multicam.push_monocular`) | `mouse:click` |
150
+ | `hotkey:ptt` (= `voice.hotkey`, Ctrl+Alt+Space) | `dictation:toggle` |
151
+ | `hotkey:mute` (= `voice.mute_hotkey`, Ctrl+Alt+M) | `mic:toggle` |
152
+ | `voice:"click"`, `"double click"`, `"right click"`, `"scroll up"`, `"scroll down"` | the matching `mouse:` action |
153
+ | `voice:"go back"` / `"go forward"` | `key:alt+left` / `key:alt+right` |
154
+ | `voice:"open browser"` | `launch:https://www.google.com` |
155
+ | `voice:"stop listening"` / `"start listening"` | `dictation:stop` / `dictation:start` |
156
+ | `voice:"command mode"` / `"dictation mode"` | `voice:command_mode` / `voice:dictation_mode` |
157
+ | `voice:"pause gestures"` / `"resume gestures"` | `pause:on` / `pause:off` |
158
+
159
+ **Triggers:** the gestures above plus your own (record with `away record --custom --label wave`, then
160
+ `away calibrate`), any spoken phrase (matched ignoring case and punctuation), `face:brow_raise`,
161
+ `face:mouth_open`, `face:wink_left|wink_right`, `gesture:dwell`, `gesture:push`, and any key combo
162
+ (`hotkey:ctrl+alt+k`). Two fixed hotkeys sit outside the bindings: `app.toggle_hotkey` (default F9,
163
+ gestures on/off) and `voice.edit_hotkey` (default Ctrl+Alt+E, edit the selected text by voice).
164
+
165
+ **Actions:** `none` · `mouse:click|double_click|right_click|middle_click|scroll_up|scroll_down`
166
+ (`mouse:move`, `mouse:drag`, `mouse:scroll` only on point / pinch_hold / two_finger_scroll) ·
167
+ `key:ctrl+tab` · `type:any text` · `launch:firefox` or `launch:https://…` (no shell) ·
168
+ `shell:command` (**off unless** `bindings.allow_shell: true`) · `dictation:toggle|start|stop` ·
169
+ `mic:toggle|on|off` · `voice:command_mode|dictation_mode|toggle_mode` · `pause:toggle|on|off`.
170
+
171
+ ### Per-app profiles
172
+
173
+ `default` holds the full map. Any other profile names the apps (process names, case-insensitive,
174
+ `.exe` optional) it applies to and overrides only the triggers it lists; everything else falls through
175
+ to `default`. `active_profile: auto` switches profile with the focused app (Linux needs `xdotool`);
176
+ `away profile use chrome` pins one.
177
+
178
+ ### CLI
179
+
180
+ ```bash
181
+ away bindings list [--profile chrome] [--json]
182
+ away bindings set gesture:swipe_left key:alt+left
183
+ away bindings set "voice:new tab" key:ctrl+t --profile chrome
184
+ away bindings reset [TRIGGER] [--profile NAME | --all]
185
+ away bindings export my-controls.yaml # or "-" for stdout
186
+ away bindings import my-controls.yaml [--replace] # default merges
187
+ away profile list | use NAME|auto | new NAME --apps chrome chrome.exe [--copy-from default] | remove NAME
188
+ away voice show | mode always_on|push_to_talk|toggle | wake-word away|off | intent dictation|command | sensitivity 0.6
189
+ away settings # GUI editor (tkinter); prints CLI help when there is no display
190
+ ```
191
+
192
+ Invalid configs fail with every problem listed and its YAML path, e.g.
193
+ `bindings.profiles.default.gestures.pinchh: unknown gesture 'pinchh' (did you mean 'pinch'?)`.
194
+
195
+ ### Tunables
196
+
197
+ Sliders in the Settings window, or in YAML: `hands.cursor.gain` (sensitivity), `dead_zone`,
198
+ `acceleration`, One Euro `min_cutoff` / `beta` / `d_cutoff`; `hands.gestures.pinch_on` / `pinch_off`
199
+ (pinch threshold), `scroll_gain` (scroll speed), `swipe_min_dx` + `swipe_max_time` (swipe velocity =
200
+ distance / time), `double_pinch_window`, `drag_hold`, `clutch_hold`; `voice.vad_sensitivity`,
201
+ `end_silence_ms`, `max_utterance_s`. Each is range-checked.
202
+
203
+ Pointer accuracy:
204
+
205
+ - **Steady click** (`hands.cursor.click_freeze: true`): the cursor freezes while the pinch is closing
206
+ (ratio below `pinch_on * (1 + freeze_margin)`), and the click lands where the cursor was
207
+ `click_rewind_ms` (100) before closing began, because closing thumb and index drags the index tip.
208
+ Drags unfreeze immediately.
209
+ - **Curve** (`hands.cursor.curve`): `sigmoid` (default) gives a slow plateau (`sigmoid_low`) for precise
210
+ work and a fast plateau (`sigmoid_high`) with `sigmoid_mid` / `sigmoid_slope` in between. `power`
211
+ is the original `|d|^acceleration` curve. `speed_up/down/left/right` scale each direction, for
212
+ limited reach or an angled camera. The sigmoid idea comes from Google Project Gameface (Apache-2.0)
213
+ and is re-implemented.
214
+ - **Dwell click** (`hands.dwell`): rest the cursor within `radius_px` for `time_s` and `gesture:dwell` fires.
215
+ - **Face clicks**: `face.wink_trigger` / `wink_threshold` / `wink_open_max` / `wink_hold`, plus
216
+ `brow_threshold`, `mouth_threshold` and the hold times. Thresholds are per user, so tune them in Settings.
217
+ - **Cursor ring** (`cursor_overlay: true` or `away run --cursor-overlay`): a small ring by the pointer
218
+ that fills as the pinch closes, turns solid on pinch, blue while dragging, purple while dwelling and
219
+ grey when paused.
220
+ - **Auto-pause** (`app.auto_pause: true`): touching the real mouse or keyboard pauses gesture output
221
+ until it has been idle for `app.auto_pause_idle_ms`. Away's own synthetic input is ignored.
222
+
223
+ Dictation extras: the vocabulary file is passed to faster-whisper as `initial_prompt` and as
224
+ `hotwords` (when the installed version supports it; 1.2.1 does). `voice.numbers_as_digits` turns
225
+ "twenty five" into 25 from `numbers_min_value` (10) up. `voice.edit_hotkey` works like this:
226
+ select text, press the hotkey, say an instruction ("uppercase", "replace cat with dog", "clean it up",
227
+ "make it a bullet list", or anything else when an LLM endpoint is configured), then press it again.
228
+
229
+ ### Voice input modes
230
+
231
+ `voice.mode` is `always_on` (default), `push_to_talk` (hold `voice.hold_key`) or `toggle` (press
232
+ `voice.hotkey`).
233
+
234
+ Always-on listens continuously, cuts speech into utterances with an energy VAD (end-pointing after
235
+ `end_silence_ms` of silence) and decides per utterance:
236
+
237
+ - **Wake word on** (`voice.wake_word_enabled: true`, `wake_word: away`): only utterances starting with
238
+ the wake word count ("away, scroll down"). A bare "away" opens a `wake_window` for the next utterance.
239
+ - **Wake word off:** text is typed only when the face gate agrees that you are at the screen and your
240
+ mouth is moving (active-speaker detection), so TV or background talk is ignored. With no camera,
241
+ `voice.ambient_fallback: commands_only` (default) runs bound phrases but types nothing.
242
+ - `voice.intent: command` never types, it only runs voice commands. Say "command mode" /
243
+ "dictation mode" to switch.
244
+ - "stop listening" / "start listening", the mute hotkey and the tray's **Mic on** item control it.
245
+ The tray icon shows the state (green active, red mic off, amber not listening, blue recording, grey paused).
246
+ - Audio is kept in memory only and dropped after transcription. Nothing is written to disk.
247
+
248
+ `app.always_on: true` (default) makes gestures and voice active as soon as the app starts;
249
+ `false` starts paused and muted until you resume from the tray. `app.start_minimized: false` also opens
250
+ Settings at launch.
251
+
252
+ ### Example `config.yaml`
253
+
254
+ ```yaml
255
+ app:
256
+ always_on: true
257
+ start_minimized: true
258
+ voice:
259
+ mode: always_on # always_on | push_to_talk | toggle
260
+ wake_word_enabled: false
261
+ wake_word: away
262
+ intent: dictation # dictation | command
263
+ vad_sensitivity: 0.5
264
+ mute_hotkey: <ctrl>+<alt>+m
265
+ hands:
266
+ cursor: {curve: sigmoid, gain: 2.2, dead_zone: 0.004, min_cutoff: 1.0, beta: 0.02, click_freeze: true}
267
+ gestures: {pinch_on: 0.32, pinch_off: 0.45, scroll_gain: 40.0, swipe_min_dx: 0.25, swipe_max_time: 0.45}
268
+ bindings:
269
+ active_profile: auto
270
+ allow_shell: false
271
+ custom_gestures: [wave]
272
+ profiles:
273
+ default:
274
+ gestures:
275
+ swipe_left: key:alt+left
276
+ swipe_right: key:alt+right
277
+ wave: launch:https://calendar.google.com
278
+ voice:
279
+ "screenshot": key:cmd+shift+s
280
+ hotkeys:
281
+ ctrl+alt+d: type:Best regards,
282
+ chrome:
283
+ apps: [chrome, chrome.exe, Google Chrome]
284
+ gestures:
285
+ two_finger_scroll: key:ctrl+tab
286
+ voice:
287
+ "new tab": key:ctrl+t
288
+ "close tab": key:ctrl+w
289
+ code:
290
+ apps: [code, Code.exe]
291
+ gestures:
292
+ open_palm: key:ctrl+shift+p # open palm no longer pauses in VS Code
293
+ ```
294
+
295
+ ## Multi-camera
296
+
297
+ A laptop webcam and an external camera can work together. A hand that one camera sees edge-on, or
298
+ that is occluded, is usually clear in the other.
299
+
300
+ ```bash
301
+ away cameras list # V4L2 names on Linux; probes indices on Windows/macOS
302
+ away cameras add 2 # index, /dev/video2, or part of the name ("brio")
303
+ away calibrate-cameras # wave an open hand through both views for ~20 s
304
+ away calibrate-cameras --method checkerboard --checkerboard 9x6 --square-mm 25 # accurate option
305
+ away calibrate-cameras --print-board board.png --checkerboard 9x6 # printable board
306
+ ```
307
+
308
+ The same is available in **Settings → Cameras** (device pickers, roles, calibrate button, live
309
+ per-camera confidence). In YAML:
310
+
311
+ ```yaml
312
+ hands:
313
+ cameras: # empty (default) = single-camera mode, unchanged
314
+ - {device: 0, role: primary, width: 640, height: 480, fps: 30, hfov_deg: 70}
315
+ - {device: "Logitech BRIO", role: secondary, width: 1280, height: 720, fps: 30, hfov_deg: 78}
316
+ multicam:
317
+ fusion: weighted # weighted | best_view
318
+ pair_tolerance_ms: 40
319
+ dropout_ms: 300
320
+ handover_decay_s: 1.0
321
+ use_3d_pinch: true
322
+ push_distance_m: 0.06
323
+ push_time_s: 0.35
324
+ ```
325
+
326
+ How it works:
327
+
328
+ 1. **Capture.** Each camera runs in its own thread and stamps frames on arrival. The primary camera
329
+ sets the pace. Each secondary contributes the frame nearest in time, if it falls within
330
+ `pair_tolerance_ms`. A camera that stays silent for `dropout_ms` is treated as dropped and the
331
+ others carry on.
332
+ 2. **Per-view confidence.** Each view is scored on the MediaPipe handedness score, the share of
333
+ landmarks inside the image, hand size in pixels, and sharpness (Laplacian variance of the hand
334
+ crop, which drops with motion blur).
335
+ 3. **Fusion.**
336
+ - **Uncalibrated:** `best_view` takes the most confident view. `weighted` averages the views by
337
+ confidence after mapping secondaries into the primary image with an alignment learned while both
338
+ cameras see the hand.
339
+ - **Calibrated**, with both cameras seeing the hand: the 21 landmarks are triangulated
340
+ (`cv2.triangulatePoints`). Reprojecting them into the primary view drives the cursor, and the
341
+ 3D points give true depth.
342
+ 4. **Depth uses.**
343
+ - **`gesture:push`:** the hand moves `push_distance_m` toward the screen within `push_time_s`.
344
+ The primary camera is assumed to sit at the screen. The gesture is bindable and clicks by default.
345
+ - **3D pinch:** the pinch ratio comes from 3D distances, so a thumb and index that only overlap from
346
+ one angle no longer click.
347
+ 5. **Occlusion handover.** When one view loses the hand, tracking continues from the other. The jump
348
+ between estimates is absorbed by an offset that fades over `handover_decay_s`, so the One Euro
349
+ filter sees a continuous signal and the cursor does not jump.
350
+
351
+ **Calibration.** The hand-motion path pairs MediaPipe landmarks across views and runs
352
+ `cv2.findFundamentalMat` (RANSAC, then a least-squares refit on the inliers) → `E = K2ᵀ F K1` →
353
+ `cv2.recoverPose`. Intrinsics are approximated from `hfov_deg`, and the translation is scaled so the
354
+ median palm (wrist to middle knuckle) is `palm_length_m` (9 cm). Depth is therefore approximately
355
+ metric; real palms vary by about 10–15%. The checkerboard path runs `calibrateCamera` per camera, then
356
+ `stereoCalibrate` with the real square size. Either way the extrinsics go to
357
+ `<data dir>/camera_extrinsics.json` and the reprojection error is printed.
358
+
359
+ Each view runs its own MediaPipe instance in sequence, so two cameras cost about twice the CPU of one.
360
+
361
+ ## Verified vs untested
362
+
363
+ | Area | Status |
364
+ |---|---|
365
+ | Core logic (filters, state machine, cleanup, CSI features, fusion, bindings, dispatch, always-on voice policy) | 496 tests pass (1 skipped where libtk is missing); ruff + mypy clean |
366
+ | Multi-camera geometry | Synthetic: two virtual cameras with known extrinsics. Pose from noisy (0.5 px) hand landmarks within 1.5°, palm-scaled baseline within 10%, exact noise-free triangulation, checkerboard stereo calibration under 0.5 px |
367
+ | Multi-camera fusion | Synthetic: best-view and weighted selection, timestamp pairing and dropout, occlusion handover with no jump, push gesture, 3D pinch rejecting an edge-on false pinch, Cameras-tab model |
368
+ | Real two-camera capture, calibration and fusion | **Untested** (no cameras in the build sandbox) |
369
+ | Steady click, sigmoid curve, dwell, wink, auto-pause guard, cursor ring states, numbers to digits, edit-selection rules | Logic unit-tested with synthetic input |
370
+ | faster-whisper `hotwords` | Parameter verified on the installed faster-whisper 1.2.1 by signature; **not run on real speech** |
371
+ | Cursor ring window, pynput auto-pause listeners, F9 / edit hotkeys, clipboard edit flow | **Untested** on real desktops |
372
+ | Static pose model | 99.27% acc / 0.982 macro-F1 on 4,094 HaGRID v2 samples from 520 unseen people (MediaPipe-annotated, not live webcam) |
373
+ | Temporal gesture model | Synthetic only. Off by default until `away calibrate` |
374
+ | MediaPipe hands/face on real photos | Verified |
375
+ | faster-whisper + Silero VAD on synthetic speech | Verified |
376
+ | PicoScenes `.csi` parsing on sample file | Verified |
377
+ | Linux bundle, AppImage (aarch64), `.deb` | Earlier lean bundle (without torch) built and launched in the sandbox; the full bundle is CI-verified on tag only |
378
+ | Settings window (tkinter) | Model logic unit-tested; window smoke-tested under Xvfb (Linux) only |
379
+ | Always-on voice | End-pointing, wake word and face-gate policy tested with synthetic audio; **untested with a real mic/ASR** |
380
+ | Per-app profile detection (active window process) | **Untested** on real desktops (Windows API / osascript / xdotool) |
381
+ | Real webcam, mic, mouse/keyboard, global hotkeys, tray, overlay | **Untested** |
382
+ | Installer one-click flow (non-interactive `setup --yes` with no stdin, windowed-exe streams, first-launch download, Inno options) | Logic unit-tested (`tests/test_installer.py`) |
383
+ | Frozen bundle with every feature (`away selftest`: torch, onnx, onnxruntime, MediaPipe, OpenCV, faster-whisper + Silero VAD, PortAudio, tkinter, pynput, pystray) | All 13 checks pass in the dev venv (Linux aarch64). The frozen build with the new bundle was **not completed in the sandbox** (too slow on its filesystem). Windows/macOS/Linux x86_64: **CI-verified on tag only** (the release workflow fails if any check fails) |
384
+ | Windows `.exe` installer, macOS `.dmg` | **CI-built and smoke-tested on tag only; untested on real hardware** |
385
+ | AX200/AX210 + PicoScenes live capture | **Untested** |
386
+ | Parakeet, LLM cleanup endpoint, Wayland | **Untested** |
387
+
388
+ ## Roadmap
389
+
390
+ 1. Real-hardware validation on Windows/macOS/Linux.
391
+ 2. Personal calibration data → fine-tune the temporal model on real recordings.
392
+ 3. Face recognition for the enrolled-user gate (currently largest face).
393
+ 4. TalkNet/Light-ASD active-speaker backend.
394
+ 5. Live AX210 CSI capture and a learned swipe/push-pull classifier.
395
+ 6. Phone-as-second-camera / IMU fusion.
396
+ 7. Real-hardware tuning of multi-camera fusion; parallel per-camera landmark inference.
397
+
398
+ ## Development
399
+
400
+ ```bash
401
+ pip install -e ".[dev]"
402
+ pytest -q && ruff check . && ruff format --check . && mypy src
403
+ python scripts/mutation.py && python scripts/mutation.py --summary
404
+ ```
405
+
406
+ Private project. All rights reserved by the author until a license is chosen.