project-away 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- project_away-0.1.1/PKG-INFO +406 -0
- project_away-0.1.1/README.md +345 -0
- project_away-0.1.1/pyproject.toml +114 -0
- project_away-0.1.1/setup.cfg +4 -0
- project_away-0.1.1/src/away/__init__.py +4 -0
- project_away-0.1.1/src/away/__main__.py +3 -0
- project_away-0.1.1/src/away/actions.py +329 -0
- project_away-0.1.1/src/away/app/__init__.py +1 -0
- project_away-0.1.1/src/away/app/autopause.py +119 -0
- project_away-0.1.1/src/away/app/autostart.py +83 -0
- project_away-0.1.1/src/away/app/cursor_overlay.py +110 -0
- project_away-0.1.1/src/away/app/frozen.py +175 -0
- project_away-0.1.1/src/away/app/models.py +87 -0
- project_away-0.1.1/src/away/app/overlay.py +61 -0
- project_away-0.1.1/src/away/app/runner.py +723 -0
- project_away-0.1.1/src/away/app/selftest.py +194 -0
- project_away-0.1.1/src/away/app/settings_gui.py +1202 -0
- project_away-0.1.1/src/away/app/setup_wizard.py +130 -0
- project_away-0.1.1/src/away/app/tray.py +197 -0
- project_away-0.1.1/src/away/app/uninstall.py +56 -0
- project_away-0.1.1/src/away/bindings.py +548 -0
- project_away-0.1.1/src/away/cli.py +747 -0
- project_away-0.1.1/src/away/config.py +411 -0
- project_away-0.1.1/src/away/control/__init__.py +1 -0
- project_away-0.1.1/src/away/control/keys.py +245 -0
- project_away-0.1.1/src/away/control/mouse.py +237 -0
- project_away-0.1.1/src/away/data/gesture_tcn.json +15 -0
- project_away-0.1.1/src/away/data/gesture_tcn.metrics.json +18 -0
- project_away-0.1.1/src/away/data/gesture_tcn.onnx +0 -0
- project_away-0.1.1/src/away/data/static_pose.metrics.json +77 -0
- project_away-0.1.1/src/away/data/static_pose.npz +0 -0
- project_away-0.1.1/src/away/events.py +67 -0
- project_away-0.1.1/src/away/face/__init__.py +1 -0
- project_away-0.1.1/src/away/face/commands.py +101 -0
- project_away-0.1.1/src/away/face/gate.py +65 -0
- project_away-0.1.1/src/away/face/landmarker.py +48 -0
- project_away-0.1.1/src/away/face/speaker.py +74 -0
- project_away-0.1.1/src/away/face/speaking.py +50 -0
- project_away-0.1.1/src/away/face/tracking.py +52 -0
- project_away-0.1.1/src/away/fusion.py +90 -0
- project_away-0.1.1/src/away/hands/__init__.py +1 -0
- project_away-0.1.1/src/away/hands/cursor.py +128 -0
- project_away-0.1.1/src/away/hands/dataset.py +171 -0
- project_away-0.1.1/src/away/hands/filters.py +71 -0
- project_away-0.1.1/src/away/hands/gestures.py +196 -0
- project_away-0.1.1/src/away/hands/landmarks.py +135 -0
- project_away-0.1.1/src/away/hands/model.py +158 -0
- project_away-0.1.1/src/away/hands/recorder.py +86 -0
- project_away-0.1.1/src/away/hands/runtime.py +67 -0
- project_away-0.1.1/src/away/hands/stabilize.py +110 -0
- project_away-0.1.1/src/away/hands/synthetic.py +192 -0
- project_away-0.1.1/src/away/hands/tracker.py +98 -0
- project_away-0.1.1/src/away/hands/train.py +220 -0
- project_away-0.1.1/src/away/multicam/__init__.py +1 -0
- project_away-0.1.1/src/away/multicam/calibration.py +208 -0
- project_away-0.1.1/src/away/multicam/capture.py +232 -0
- project_away-0.1.1/src/away/multicam/fusion.py +329 -0
- project_away-0.1.1/src/away/multicam/geometry.py +317 -0
- project_away-0.1.1/src/away/paths.py +54 -0
- project_away-0.1.1/src/away/py.typed +0 -0
- project_away-0.1.1/src/away/voice/__init__.py +1 -0
- project_away-0.1.1/src/away/voice/always_on.py +505 -0
- project_away-0.1.1/src/away/voice/asr.py +138 -0
- project_away-0.1.1/src/away/voice/cleanup.py +269 -0
- project_away-0.1.1/src/away/voice/context.py +89 -0
- project_away-0.1.1/src/away/voice/dictation.py +203 -0
- project_away-0.1.1/src/away/voice/edit.py +234 -0
- project_away-0.1.1/src/away/voice/numbers.py +150 -0
- project_away-0.1.1/src/away/voice/output.py +59 -0
- project_away-0.1.1/src/away/voice/vad.py +140 -0
- project_away-0.1.1/src/away/wifi/__init__.py +1 -0
- project_away-0.1.1/src/away/wifi/classifier.py +91 -0
- project_away-0.1.1/src/away/wifi/features.py +150 -0
- project_away-0.1.1/src/away/wifi/sources.py +243 -0
- project_away-0.1.1/src/project_away.egg-info/PKG-INFO +406 -0
- project_away-0.1.1/src/project_away.egg-info/SOURCES.txt +106 -0
- project_away-0.1.1/src/project_away.egg-info/dependency_links.txt +1 -0
- project_away-0.1.1/src/project_away.egg-info/entry_points.txt +5 -0
- project_away-0.1.1/src/project_away.egg-info/requires.txt +50 -0
- project_away-0.1.1/src/project_away.egg-info/top_level.txt +1 -0
- project_away-0.1.1/tests/test_always_on.py +394 -0
- project_away-0.1.1/tests/test_app.py +400 -0
- project_away-0.1.1/tests/test_bindings.py +323 -0
- project_away-0.1.1/tests/test_cleanup.py +186 -0
- project_away-0.1.1/tests/test_cli_controls.py +185 -0
- project_away-0.1.1/tests/test_config_paths.py +109 -0
- project_away-0.1.1/tests/test_cursor.py +112 -0
- project_away-0.1.1/tests/test_dispatch.py +322 -0
- project_away-0.1.1/tests/test_docs.py +17 -0
- project_away-0.1.1/tests/test_face.py +252 -0
- project_away-0.1.1/tests/test_filters.py +97 -0
- project_away-0.1.1/tests/test_frame_pairer_safety.py +66 -0
- project_away-0.1.1/tests/test_fusion_events.py +137 -0
- project_away-0.1.1/tests/test_gesture_lifecycle_safety.py +93 -0
- project_away-0.1.1/tests/test_gestures.py +231 -0
- project_away-0.1.1/tests/test_installer.py +327 -0
- project_away-0.1.1/tests/test_key_release_safety.py +125 -0
- project_away-0.1.1/tests/test_landmarks.py +99 -0
- project_away-0.1.1/tests/test_models_data.py +186 -0
- project_away-0.1.1/tests/test_multicam.py +463 -0
- project_away-0.1.1/tests/test_settings_model.py +255 -0
- project_away-0.1.1/tests/test_shutdown_safety.py +78 -0
- project_away-0.1.1/tests/test_spec_edges.py +458 -0
- project_away-0.1.1/tests/test_spec_edges2.py +210 -0
- project_away-0.1.1/tests/test_train_torch.py +98 -0
- project_away-0.1.1/tests/test_upgrades.py +437 -0
- project_away-0.1.1/tests/test_voice.py +241 -0
- project_away-0.1.1/tests/test_wifi.py +269 -0
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: project-away
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Hands-free PC control: camera hand gestures, push-to-talk dictation, active-speaker gating and experimental Wi-Fi CSI sensing. Local-first.
|
|
5
|
+
Author-email: Fernando Garza <fernandogarzaaa@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/fernandogarzaaa/project-away
|
|
8
|
+
Keywords: gesture,hand-tracking,mediapipe,dictation,whisper,accessibility,wifi-csi
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Topic :: Desktop Environment
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
Requires-Dist: numpy>=1.24
|
|
21
|
+
Requires-Dist: scipy>=1.10
|
|
22
|
+
Requires-Dist: PyYAML>=6.0
|
|
23
|
+
Provides-Extra: hands
|
|
24
|
+
Requires-Dist: mediapipe>=0.10.14; extra == "hands"
|
|
25
|
+
Requires-Dist: opencv-python>=4.8; extra == "hands"
|
|
26
|
+
Requires-Dist: onnxruntime>=1.17; extra == "hands"
|
|
27
|
+
Provides-Extra: face
|
|
28
|
+
Requires-Dist: mediapipe>=0.10.14; extra == "face"
|
|
29
|
+
Requires-Dist: opencv-python>=4.8; extra == "face"
|
|
30
|
+
Provides-Extra: voice
|
|
31
|
+
Requires-Dist: faster-whisper>=1.1; extra == "voice"
|
|
32
|
+
Requires-Dist: sounddevice>=0.4.6; extra == "voice"
|
|
33
|
+
Provides-Extra: control
|
|
34
|
+
Requires-Dist: pynput>=1.7.6; extra == "control"
|
|
35
|
+
Requires-Dist: pyperclip>=1.8; extra == "control"
|
|
36
|
+
Provides-Extra: tray
|
|
37
|
+
Requires-Dist: pystray>=0.19; extra == "tray"
|
|
38
|
+
Requires-Dist: Pillow>=10; extra == "tray"
|
|
39
|
+
Provides-Extra: train
|
|
40
|
+
Requires-Dist: torch>=2.1; extra == "train"
|
|
41
|
+
Requires-Dist: onnx>=1.15; extra == "train"
|
|
42
|
+
Requires-Dist: onnxruntime>=1.17; extra == "train"
|
|
43
|
+
Provides-Extra: parakeet
|
|
44
|
+
Requires-Dist: nemo_toolkit[asr]>=2.0; extra == "parakeet"
|
|
45
|
+
Provides-Extra: all
|
|
46
|
+
Requires-Dist: mediapipe>=0.10.14; extra == "all"
|
|
47
|
+
Requires-Dist: opencv-python>=4.8; extra == "all"
|
|
48
|
+
Requires-Dist: onnxruntime>=1.17; extra == "all"
|
|
49
|
+
Requires-Dist: faster-whisper>=1.1; extra == "all"
|
|
50
|
+
Requires-Dist: sounddevice>=0.4.6; extra == "all"
|
|
51
|
+
Requires-Dist: pynput>=1.7.6; extra == "all"
|
|
52
|
+
Requires-Dist: pyperclip>=1.8; extra == "all"
|
|
53
|
+
Requires-Dist: pystray>=0.19; extra == "all"
|
|
54
|
+
Requires-Dist: Pillow>=10; extra == "all"
|
|
55
|
+
Provides-Extra: dev
|
|
56
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
57
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
58
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
59
|
+
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
60
|
+
Requires-Dist: types-PyYAML; extra == "dev"
|
|
61
|
+
|
|
62
|
+
# Project Away
|
|
63
|
+
|
|
64
|
+
Hands-free PC control, free and local. Webcam hand gestures drive the cursor, push-to-talk voice typing handles text, face cues gate who is dictating, and optional Wi-Fi CSI adds coarse motion confidence.
|
|
65
|
+
|
|
66
|
+
Nothing leaves your machine unless you point the optional LLM cleanup at an endpoint yourself.
|
|
67
|
+
|
|
68
|
+
## Architecture
|
|
69
|
+
|
|
70
|
+
```mermaid
|
|
71
|
+
flowchart LR
|
|
72
|
+
CAM[Webcam] --> HL[MediaPipe Hand Landmarker]
|
|
73
|
+
CAM2[Optional 2nd camera] --> PAIR[Threaded capture + timestamp pairing]
|
|
74
|
+
CAM --> PAIR
|
|
75
|
+
PAIR --> HL
|
|
76
|
+
HL --> MV[Multi-view fusion: confidence, triangulation, handover]
|
|
77
|
+
MV --> NORM[Normalize + One Euro filter]
|
|
78
|
+
NORM --> GSM[Gesture state machine + static/TCN models]
|
|
79
|
+
GSM --> STEADY[Steady click + sigmoid curve + dwell/push]
|
|
80
|
+
CAM --> FL[MediaPipe Face Landmarker]
|
|
81
|
+
FL --> ASD[Active speaker + dictation gate]
|
|
82
|
+
MIC[Microphone] --> VAD[Silero VAD]
|
|
83
|
+
VAD --> ASR[faster-whisper / Parakeet]
|
|
84
|
+
ASD --> ASR
|
|
85
|
+
ASR --> CLEAN[Rule cleanup / optional LLM]
|
|
86
|
+
WIFI[Wi-Fi CSI or RSSI] --> CSI[Hampel + Doppler features]
|
|
87
|
+
CSI --> WCLS[Coarse presence/swipe classifier]
|
|
88
|
+
STEADY --> FUS[Fusion + event bus]
|
|
89
|
+
GUARD[Physical mouse/keyboard guard] --> FUS
|
|
90
|
+
WCLS --> FUS
|
|
91
|
+
FUS --> MOUSE[Cursor / click / scroll]
|
|
92
|
+
CLEAN --> KB[Type into active app]
|
|
93
|
+
FUS --> RING[Cursor ring overlay]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
| Module | Role |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `hands/` | landmarks, One Euro filter, cursor mapping (power or sigmoid curve, per-direction speed), gesture state machine, `stabilize.py` (steady click, dwell) |
|
|
99
|
+
| `multicam/` | `capture.py` (enumeration, camera threads, `FramePairer`), `geometry.py` (intrinsics, pose from hands, triangulation, checkerboard), `calibration.py`, `fusion.py` (per-view confidence, best-view/weighted fusion, 3D pinch, push) |
|
|
100
|
+
| `face/` | face landmarker, active speaker, brow / mouth / wink triggers |
|
|
101
|
+
| `voice/` | always-on + push-to-talk, ASR (vocabulary as prompt + `hotwords`), cleanup, `numbers.py`, `edit.py` (edit selected text) |
|
|
102
|
+
| `app/` | runner (pipeline + device threads), tray, settings GUI, overlays, `autopause.py` |
|
|
103
|
+
| `bindings.py` / `actions.py` | trigger -> action mapping and execution |
|
|
104
|
+
|
|
105
|
+
## Quickstart
|
|
106
|
+
|
|
107
|
+
Full per-OS instructions: [docs/INSTALL.md](docs/INSTALL.md). Wi-Fi sensing: [docs/wifi-setup.md](docs/wifi-setup.md).
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pipx install ".[all]" # Python 3.10–3.12
|
|
111
|
+
away setup # downloads models, checks camera + mic
|
|
112
|
+
away demo # synthetic end-to-end check
|
|
113
|
+
away run # or: away tray ; away autostart on
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
**One-click installers** (Releases page) bundle every feature: tracking, dictation, tray, settings
|
|
117
|
+
window and on-device training (CPU PyTorch). Parakeet/NeMo is the one exception (multi-GB,
|
|
118
|
+
NVIDIA-oriented; use `pipx install "project-away[all,parakeet]"`). Installers are ~0.4–0.8 GB; the models
|
|
119
|
+
(~1.6 GB, or ~0.5 GB with Whisper `small`) download during install on Windows and on first launch elsewhere.
|
|
120
|
+
|
|
121
|
+
- **Windows:** run the `setup.exe`. It's unsigned, so choose *More info → Run anyway*, then **Install**. Models,
|
|
122
|
+
start-at-sign-in, a desktop shortcut and launching the tray app are all on by default.
|
|
123
|
+
- **macOS:** open the `.dmg`, drag to Applications, then right-click → Open. The first launch downloads the
|
|
124
|
+
models with a notification. Allow Camera, Microphone, Accessibility and Input Monitoring.
|
|
125
|
+
- **Linux:** run the AppImage or install the `.deb`; the first launch downloads the models. Mouse and keyboard
|
|
126
|
+
control needs X11; Wayland is untested.
|
|
127
|
+
|
|
128
|
+
## Customizing controls
|
|
129
|
+
|
|
130
|
+
Every control is a **trigger → action** binding you can change without touching code: from the
|
|
131
|
+
**Settings** window (tray → *Settings...*, or `away settings`), from the CLI, or in `config.yaml`
|
|
132
|
+
(`away config path` shows where it is). A running Away reloads the file within a second of a save.
|
|
133
|
+
|
|
134
|
+
### Default bindings
|
|
135
|
+
|
|
136
|
+
| Trigger | Default action |
|
|
137
|
+
|---|---|
|
|
138
|
+
| `gesture:point` (index finger) | `mouse:move` (cursor) |
|
|
139
|
+
| `gesture:pinch` | `mouse:click` |
|
|
140
|
+
| `gesture:double_pinch` | `mouse:double_click` |
|
|
141
|
+
| `gesture:pinch_hold` | `mouse:drag` |
|
|
142
|
+
| `gesture:two_finger_scroll` | `mouse:scroll` |
|
|
143
|
+
| `gesture:open_palm` (hold) | `pause:toggle` (pause/resume gestures) |
|
|
144
|
+
| `gesture:swipe_left` / `swipe_right` | `key:left` / `key:right` |
|
|
145
|
+
| `face:brow_raise` (needs `face.brow_raise_toggle: true`) | `dictation:toggle` |
|
|
146
|
+
| `face:mouth_open` (needs `face.mouth_open_trigger: true`) | `none` |
|
|
147
|
+
| `face:wink_left` / `wink_right` (needs `face.wink_trigger: true`) | `mouse:click` / `mouse:right_click` |
|
|
148
|
+
| `gesture:dwell` (needs `hands.dwell.enabled: true`) | `mouse:click` |
|
|
149
|
+
| `gesture:push` (needs depth: two calibrated cameras, or `hands.multicam.push_monocular`) | `mouse:click` |
|
|
150
|
+
| `hotkey:ptt` (= `voice.hotkey`, Ctrl+Alt+Space) | `dictation:toggle` |
|
|
151
|
+
| `hotkey:mute` (= `voice.mute_hotkey`, Ctrl+Alt+M) | `mic:toggle` |
|
|
152
|
+
| `voice:"click"`, `"double click"`, `"right click"`, `"scroll up"`, `"scroll down"` | the matching `mouse:` action |
|
|
153
|
+
| `voice:"go back"` / `"go forward"` | `key:alt+left` / `key:alt+right` |
|
|
154
|
+
| `voice:"open browser"` | `launch:https://www.google.com` |
|
|
155
|
+
| `voice:"stop listening"` / `"start listening"` | `dictation:stop` / `dictation:start` |
|
|
156
|
+
| `voice:"command mode"` / `"dictation mode"` | `voice:command_mode` / `voice:dictation_mode` |
|
|
157
|
+
| `voice:"pause gestures"` / `"resume gestures"` | `pause:on` / `pause:off` |
|
|
158
|
+
|
|
159
|
+
**Triggers:** the gestures above plus your own (record with `away record --custom --label wave`, then
|
|
160
|
+
`away calibrate`), any spoken phrase (matched ignoring case and punctuation), `face:brow_raise`,
|
|
161
|
+
`face:mouth_open`, `face:wink_left|wink_right`, `gesture:dwell`, `gesture:push`, and any key combo
|
|
162
|
+
(`hotkey:ctrl+alt+k`). Two fixed hotkeys sit outside the bindings: `app.toggle_hotkey` (default F9,
|
|
163
|
+
gestures on/off) and `voice.edit_hotkey` (default Ctrl+Alt+E, edit the selected text by voice).
|
|
164
|
+
|
|
165
|
+
**Actions:** `none` · `mouse:click|double_click|right_click|middle_click|scroll_up|scroll_down`
|
|
166
|
+
(`mouse:move`, `mouse:drag`, `mouse:scroll` only on point / pinch_hold / two_finger_scroll) ·
|
|
167
|
+
`key:ctrl+tab` · `type:any text` · `launch:firefox` or `launch:https://…` (no shell) ·
|
|
168
|
+
`shell:command` (**off unless** `bindings.allow_shell: true`) · `dictation:toggle|start|stop` ·
|
|
169
|
+
`mic:toggle|on|off` · `voice:command_mode|dictation_mode|toggle_mode` · `pause:toggle|on|off`.
|
|
170
|
+
|
|
171
|
+
### Per-app profiles
|
|
172
|
+
|
|
173
|
+
`default` holds the full map. Any other profile names the apps (process names, case-insensitive,
|
|
174
|
+
`.exe` optional) it applies to and overrides only the triggers it lists; everything else falls through
|
|
175
|
+
to `default`. `active_profile: auto` switches profile with the focused app (Linux needs `xdotool`);
|
|
176
|
+
`away profile use chrome` pins one.
|
|
177
|
+
|
|
178
|
+
### CLI
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
away bindings list [--profile chrome] [--json]
|
|
182
|
+
away bindings set gesture:swipe_left key:alt+left
|
|
183
|
+
away bindings set "voice:new tab" key:ctrl+t --profile chrome
|
|
184
|
+
away bindings reset [TRIGGER] [--profile NAME | --all]
|
|
185
|
+
away bindings export my-controls.yaml # or "-" for stdout
|
|
186
|
+
away bindings import my-controls.yaml [--replace] # default merges
|
|
187
|
+
away profile list | use NAME|auto | new NAME --apps chrome chrome.exe [--copy-from default] | remove NAME
|
|
188
|
+
away voice show | mode always_on|push_to_talk|toggle | wake-word away|off | intent dictation|command | sensitivity 0.6
|
|
189
|
+
away settings # GUI editor (tkinter); prints CLI help when there is no display
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Invalid configs fail with every problem listed and its YAML path, e.g.
|
|
193
|
+
`bindings.profiles.default.gestures.pinchh: unknown gesture 'pinchh' (did you mean 'pinch'?)`.
|
|
194
|
+
|
|
195
|
+
### Tunables
|
|
196
|
+
|
|
197
|
+
Sliders in the Settings window, or in YAML: `hands.cursor.gain` (sensitivity), `dead_zone`,
|
|
198
|
+
`acceleration`, One Euro `min_cutoff` / `beta` / `d_cutoff`; `hands.gestures.pinch_on` / `pinch_off`
|
|
199
|
+
(pinch threshold), `scroll_gain` (scroll speed), `swipe_min_dx` + `swipe_max_time` (swipe velocity =
|
|
200
|
+
distance / time), `double_pinch_window`, `drag_hold`, `clutch_hold`; `voice.vad_sensitivity`,
|
|
201
|
+
`end_silence_ms`, `max_utterance_s`. Each is range-checked.
|
|
202
|
+
|
|
203
|
+
Pointer accuracy:
|
|
204
|
+
|
|
205
|
+
- **Steady click** (`hands.cursor.click_freeze: true`): the cursor freezes while the pinch is closing
|
|
206
|
+
(ratio below `pinch_on * (1 + freeze_margin)`), and the click lands where the cursor was
|
|
207
|
+
`click_rewind_ms` (100) before closing began, because closing thumb and index drags the index tip.
|
|
208
|
+
Drags unfreeze immediately.
|
|
209
|
+
- **Curve** (`hands.cursor.curve`): `sigmoid` (default) gives a slow plateau (`sigmoid_low`) for precise
|
|
210
|
+
work and a fast plateau (`sigmoid_high`) with `sigmoid_mid` / `sigmoid_slope` in between. `power`
|
|
211
|
+
is the original `|d|^acceleration` curve. `speed_up/down/left/right` scale each direction, for
|
|
212
|
+
limited reach or an angled camera. The sigmoid idea comes from Google Project Gameface (Apache-2.0)
|
|
213
|
+
and is re-implemented.
|
|
214
|
+
- **Dwell click** (`hands.dwell`): rest the cursor within `radius_px` for `time_s` and `gesture:dwell` fires.
|
|
215
|
+
- **Face clicks**: `face.wink_trigger` / `wink_threshold` / `wink_open_max` / `wink_hold`, plus
|
|
216
|
+
`brow_threshold`, `mouth_threshold` and the hold times. Thresholds are per user, so tune them in Settings.
|
|
217
|
+
- **Cursor ring** (`cursor_overlay: true` or `away run --cursor-overlay`): a small ring by the pointer
|
|
218
|
+
that fills as the pinch closes, turns solid on pinch, blue while dragging, purple while dwelling and
|
|
219
|
+
grey when paused.
|
|
220
|
+
- **Auto-pause** (`app.auto_pause: true`): touching the real mouse or keyboard pauses gesture output
|
|
221
|
+
until it has been idle for `app.auto_pause_idle_ms`. Away's own synthetic input is ignored.
|
|
222
|
+
|
|
223
|
+
Dictation extras: the vocabulary file is passed to faster-whisper as `initial_prompt` and as
|
|
224
|
+
`hotwords` (when the installed version supports it; 1.2.1 does). `voice.numbers_as_digits` turns
|
|
225
|
+
"twenty five" into 25 from `numbers_min_value` (10) up. `voice.edit_hotkey` works like this:
|
|
226
|
+
select text, press the hotkey, say an instruction ("uppercase", "replace cat with dog", "clean it up",
|
|
227
|
+
"make it a bullet list", or anything else when an LLM endpoint is configured), then press it again.
|
|
228
|
+
|
|
229
|
+
### Voice input modes
|
|
230
|
+
|
|
231
|
+
`voice.mode` is `always_on` (default), `push_to_talk` (hold `voice.hold_key`) or `toggle` (press
|
|
232
|
+
`voice.hotkey`).
|
|
233
|
+
|
|
234
|
+
Always-on listens continuously, cuts speech into utterances with an energy VAD (end-pointing after
|
|
235
|
+
`end_silence_ms` of silence) and decides per utterance:
|
|
236
|
+
|
|
237
|
+
- **Wake word on** (`voice.wake_word_enabled: true`, `wake_word: away`): only utterances starting with
|
|
238
|
+
the wake word count ("away, scroll down"). A bare "away" opens a `wake_window` for the next utterance.
|
|
239
|
+
- **Wake word off:** text is typed only when the face gate agrees that you are at the screen and your
|
|
240
|
+
mouth is moving (active-speaker detection), so TV or background talk is ignored. With no camera,
|
|
241
|
+
`voice.ambient_fallback: commands_only` (default) runs bound phrases but types nothing.
|
|
242
|
+
- `voice.intent: command` never types, it only runs voice commands. Say "command mode" /
|
|
243
|
+
"dictation mode" to switch.
|
|
244
|
+
- "stop listening" / "start listening", the mute hotkey and the tray's **Mic on** item control it.
|
|
245
|
+
The tray icon shows the state (green active, red mic off, amber not listening, blue recording, grey paused).
|
|
246
|
+
- Audio is kept in memory only and dropped after transcription. Nothing is written to disk.
|
|
247
|
+
|
|
248
|
+
`app.always_on: true` (default) makes gestures and voice active as soon as the app starts;
|
|
249
|
+
`false` starts paused and muted until you resume from the tray. `app.start_minimized: false` also opens
|
|
250
|
+
Settings at launch.
|
|
251
|
+
|
|
252
|
+
### Example `config.yaml`
|
|
253
|
+
|
|
254
|
+
```yaml
|
|
255
|
+
app:
|
|
256
|
+
always_on: true
|
|
257
|
+
start_minimized: true
|
|
258
|
+
voice:
|
|
259
|
+
mode: always_on # always_on | push_to_talk | toggle
|
|
260
|
+
wake_word_enabled: false
|
|
261
|
+
wake_word: away
|
|
262
|
+
intent: dictation # dictation | command
|
|
263
|
+
vad_sensitivity: 0.5
|
|
264
|
+
mute_hotkey: <ctrl>+<alt>+m
|
|
265
|
+
hands:
|
|
266
|
+
cursor: {curve: sigmoid, gain: 2.2, dead_zone: 0.004, min_cutoff: 1.0, beta: 0.02, click_freeze: true}
|
|
267
|
+
gestures: {pinch_on: 0.32, pinch_off: 0.45, scroll_gain: 40.0, swipe_min_dx: 0.25, swipe_max_time: 0.45}
|
|
268
|
+
bindings:
|
|
269
|
+
active_profile: auto
|
|
270
|
+
allow_shell: false
|
|
271
|
+
custom_gestures: [wave]
|
|
272
|
+
profiles:
|
|
273
|
+
default:
|
|
274
|
+
gestures:
|
|
275
|
+
swipe_left: key:alt+left
|
|
276
|
+
swipe_right: key:alt+right
|
|
277
|
+
wave: launch:https://calendar.google.com
|
|
278
|
+
voice:
|
|
279
|
+
"screenshot": key:cmd+shift+s
|
|
280
|
+
hotkeys:
|
|
281
|
+
ctrl+alt+d: type:Best regards,
|
|
282
|
+
chrome:
|
|
283
|
+
apps: [chrome, chrome.exe, Google Chrome]
|
|
284
|
+
gestures:
|
|
285
|
+
two_finger_scroll: key:ctrl+tab
|
|
286
|
+
voice:
|
|
287
|
+
"new tab": key:ctrl+t
|
|
288
|
+
"close tab": key:ctrl+w
|
|
289
|
+
code:
|
|
290
|
+
apps: [code, Code.exe]
|
|
291
|
+
gestures:
|
|
292
|
+
open_palm: key:ctrl+shift+p # open palm no longer pauses in VS Code
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
## Multi-camera
|
|
296
|
+
|
|
297
|
+
A laptop webcam and an external camera can work together. A hand that one camera sees edge-on, or
|
|
298
|
+
that is occluded, is usually clear in the other.
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
away cameras list # V4L2 names on Linux; probes indices on Windows/macOS
|
|
302
|
+
away cameras add 2 # index, /dev/video2, or part of the name ("brio")
|
|
303
|
+
away calibrate-cameras # wave an open hand through both views for ~20 s
|
|
304
|
+
away calibrate-cameras --method checkerboard --checkerboard 9x6 --square-mm 25 # accurate option
|
|
305
|
+
away calibrate-cameras --print-board board.png --checkerboard 9x6 # printable board
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
The same is available in **Settings → Cameras** (device pickers, roles, calibrate button, live
|
|
309
|
+
per-camera confidence). In YAML:
|
|
310
|
+
|
|
311
|
+
```yaml
|
|
312
|
+
hands:
|
|
313
|
+
cameras: # empty (default) = single-camera mode, unchanged
|
|
314
|
+
- {device: 0, role: primary, width: 640, height: 480, fps: 30, hfov_deg: 70}
|
|
315
|
+
- {device: "Logitech BRIO", role: secondary, width: 1280, height: 720, fps: 30, hfov_deg: 78}
|
|
316
|
+
multicam:
|
|
317
|
+
fusion: weighted # weighted | best_view
|
|
318
|
+
pair_tolerance_ms: 40
|
|
319
|
+
dropout_ms: 300
|
|
320
|
+
handover_decay_s: 1.0
|
|
321
|
+
use_3d_pinch: true
|
|
322
|
+
push_distance_m: 0.06
|
|
323
|
+
push_time_s: 0.35
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
How it works:
|
|
327
|
+
|
|
328
|
+
1. **Capture.** Each camera runs in its own thread and stamps frames on arrival. The primary camera
|
|
329
|
+
sets the pace. Each secondary contributes the frame nearest in time, if it falls within
|
|
330
|
+
`pair_tolerance_ms`. A camera that stays silent for `dropout_ms` is treated as dropped and the
|
|
331
|
+
others carry on.
|
|
332
|
+
2. **Per-view confidence.** Each view is scored on the MediaPipe handedness score, the share of
|
|
333
|
+
landmarks inside the image, hand size in pixels, and sharpness (Laplacian variance of the hand
|
|
334
|
+
crop, which drops with motion blur).
|
|
335
|
+
3. **Fusion.**
|
|
336
|
+
- **Uncalibrated:** `best_view` takes the most confident view. `weighted` averages the views by
|
|
337
|
+
confidence after mapping secondaries into the primary image with an alignment learned while both
|
|
338
|
+
cameras see the hand.
|
|
339
|
+
- **Calibrated**, with both cameras seeing the hand: the 21 landmarks are triangulated
|
|
340
|
+
(`cv2.triangulatePoints`). Reprojecting them into the primary view drives the cursor, and the
|
|
341
|
+
3D points give true depth.
|
|
342
|
+
4. **Depth uses.**
|
|
343
|
+
- **`gesture:push`:** the hand moves `push_distance_m` toward the screen within `push_time_s`.
|
|
344
|
+
The primary camera is assumed to sit at the screen. The gesture is bindable and clicks by default.
|
|
345
|
+
- **3D pinch:** the pinch ratio comes from 3D distances, so a thumb and index that only overlap from
|
|
346
|
+
one angle no longer click.
|
|
347
|
+
5. **Occlusion handover.** When one view loses the hand, tracking continues from the other. The jump
|
|
348
|
+
between estimates is absorbed by an offset that fades over `handover_decay_s`, so the One Euro
|
|
349
|
+
filter sees a continuous signal and the cursor does not jump.
|
|
350
|
+
|
|
351
|
+
**Calibration.** The hand-motion path pairs MediaPipe landmarks across views and runs
|
|
352
|
+
`cv2.findFundamentalMat` (RANSAC, then a least-squares refit on the inliers) → `E = K2ᵀ F K1` →
|
|
353
|
+
`cv2.recoverPose`. Intrinsics are approximated from `hfov_deg`, and the translation is scaled so the
|
|
354
|
+
median palm (wrist to middle knuckle) is `palm_length_m` (9 cm). Depth is therefore approximately
|
|
355
|
+
metric; real palms vary by about 10–15%. The checkerboard path runs `calibrateCamera` per camera, then
|
|
356
|
+
`stereoCalibrate` with the real square size. Either way the extrinsics go to
|
|
357
|
+
`<data dir>/camera_extrinsics.json` and the reprojection error is printed.
|
|
358
|
+
|
|
359
|
+
Each view runs its own MediaPipe instance in sequence, so two cameras cost about twice the CPU of one.
|
|
360
|
+
|
|
361
|
+
## Verified vs untested
|
|
362
|
+
|
|
363
|
+
| Area | Status |
|
|
364
|
+
|---|---|
|
|
365
|
+
| Core logic (filters, state machine, cleanup, CSI features, fusion, bindings, dispatch, always-on voice policy) | 496 tests pass (1 skipped where libtk is missing); ruff + mypy clean |
|
|
366
|
+
| Multi-camera geometry | Synthetic: two virtual cameras with known extrinsics. Pose from noisy (0.5 px) hand landmarks within 1.5°, palm-scaled baseline within 10%, exact noise-free triangulation, checkerboard stereo calibration under 0.5 px |
|
|
367
|
+
| Multi-camera fusion | Synthetic: best-view and weighted selection, timestamp pairing and dropout, occlusion handover with no jump, push gesture, 3D pinch rejecting an edge-on false pinch, Cameras-tab model |
|
|
368
|
+
| Real two-camera capture, calibration and fusion | **Untested** (no cameras in the build sandbox) |
|
|
369
|
+
| Steady click, sigmoid curve, dwell, wink, auto-pause guard, cursor ring states, numbers to digits, edit-selection rules | Logic unit-tested with synthetic input |
|
|
370
|
+
| faster-whisper `hotwords` | Parameter verified on the installed faster-whisper 1.2.1 by signature; **not run on real speech** |
|
|
371
|
+
| Cursor ring window, pynput auto-pause listeners, F9 / edit hotkeys, clipboard edit flow | **Untested** on real desktops |
|
|
372
|
+
| Static pose model | 99.27% acc / 0.982 macro-F1 on 4,094 HaGRID v2 samples from 520 unseen people (MediaPipe-annotated, not live webcam) |
|
|
373
|
+
| Temporal gesture model | Synthetic only. Off by default until `away calibrate` |
|
|
374
|
+
| MediaPipe hands/face on real photos | Verified |
|
|
375
|
+
| faster-whisper + Silero VAD on synthetic speech | Verified |
|
|
376
|
+
| PicoScenes `.csi` parsing on sample file | Verified |
|
|
377
|
+
| Linux bundle, AppImage (aarch64), `.deb` | Earlier lean bundle (without torch) built and launched in the sandbox; the full bundle is CI-verified on tag only |
|
|
378
|
+
| Settings window (tkinter) | Model logic unit-tested; window smoke-tested under Xvfb (Linux) only |
|
|
379
|
+
| Always-on voice | End-pointing, wake word and face-gate policy tested with synthetic audio; **untested with a real mic/ASR** |
|
|
380
|
+
| Per-app profile detection (active window process) | **Untested** on real desktops (Windows API / osascript / xdotool) |
|
|
381
|
+
| Real webcam, mic, mouse/keyboard, global hotkeys, tray, overlay | **Untested** |
|
|
382
|
+
| Installer one-click flow (non-interactive `setup --yes` with no stdin, windowed-exe streams, first-launch download, Inno options) | Logic unit-tested (`tests/test_installer.py`) |
|
|
383
|
+
| Frozen bundle with every feature (`away selftest`: torch, onnx, onnxruntime, MediaPipe, OpenCV, faster-whisper + Silero VAD, PortAudio, tkinter, pynput, pystray) | All 13 checks pass in the dev venv (Linux aarch64). The frozen build with the new bundle was **not completed in the sandbox** (too slow on its filesystem). Windows/macOS/Linux x86_64: **CI-verified on tag only** (the release workflow fails if any check fails) |
|
|
384
|
+
| Windows `.exe` installer, macOS `.dmg` | **CI-built and smoke-tested on tag only; untested on real hardware** |
|
|
385
|
+
| AX200/AX210 + PicoScenes live capture | **Untested** |
|
|
386
|
+
| Parakeet, LLM cleanup endpoint, Wayland | **Untested** |
|
|
387
|
+
|
|
388
|
+
## Roadmap
|
|
389
|
+
|
|
390
|
+
1. Real-hardware validation on Windows/macOS/Linux.
|
|
391
|
+
2. Personal calibration data → fine-tune the temporal model on real recordings.
|
|
392
|
+
3. Face recognition for the enrolled-user gate (currently largest face).
|
|
393
|
+
4. TalkNet/Light-ASD active-speaker backend.
|
|
394
|
+
5. Live AX210 CSI capture and a learned swipe/push-pull classifier.
|
|
395
|
+
6. Phone-as-second-camera / IMU fusion.
|
|
396
|
+
7. Real-hardware tuning of multi-camera fusion; parallel per-camera landmark inference.
|
|
397
|
+
|
|
398
|
+
## Development
|
|
399
|
+
|
|
400
|
+
```bash
|
|
401
|
+
pip install -e ".[dev]"
|
|
402
|
+
pytest -q && ruff check . && ruff format --check . && mypy src
|
|
403
|
+
python scripts/mutation.py && python scripts/mutation.py --summary
|
|
404
|
+
```
|
|
405
|
+
|
|
406
|
+
Private project. All rights reserved by the author until a license is chosen.
|