mctrl 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mctrl-0.1.0.dist-info/METADATA +692 -0
- mctrl-0.1.0.dist-info/RECORD +33 -0
- mctrl-0.1.0.dist-info/WHEEL +4 -0
- mctrl-0.1.0.dist-info/entry_points.txt +3 -0
- mindcontrol/__init__.py +10 -0
- mindcontrol/__main__.py +4 -0
- mindcontrol/app.py +432 -0
- mindcontrol/autotune.py +493 -0
- mindcontrol/calibrate.py +199 -0
- mindcontrol/capture.py +159 -0
- mindcontrol/config.py +243 -0
- mindcontrol/control/__init__.py +1 -0
- mindcontrol/control/bridge.py +360 -0
- mindcontrol/control/events.py +34 -0
- mindcontrol/control/keyboard.py +101 -0
- mindcontrol/control/modes.py +194 -0
- mindcontrol/control/mouse.py +256 -0
- mindcontrol/debug_view.py +188 -0
- mindcontrol/devices.py +222 -0
- mindcontrol/filters.py +95 -0
- mindcontrol/fusion.py +240 -0
- mindcontrol/geometry.py +200 -0
- mindcontrol/gestures/__init__.py +1 -0
- mindcontrol/gestures/engine.py +465 -0
- mindcontrol/logs.py +87 -0
- mindcontrol/models.py +59 -0
- mindcontrol/pipeline.py +376 -0
- mindcontrol/record.py +454 -0
- mindcontrol/replay.py +193 -0
- mindcontrol/session.py +328 -0
- mindcontrol/tracking/__init__.py +1 -0
- mindcontrol/tracking/gaze.py +272 -0
- mindcontrol/tracking/hands.py +102 -0
mindcontrol/autotune.py
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
"""Fitting thresholds to the hands that were actually recorded.
|
|
2
|
+
|
|
3
|
+
Every threshold in this system separates two clusters of a measurement -- pinched
|
|
4
|
+
from open, curled from extended. Guessing where the boundary goes is what the
|
|
5
|
+
shipped defaults do. Given a labelled recording the boundary can instead be
|
|
6
|
+
*measured*, because the label says which cluster each sample belongs to.
|
|
7
|
+
|
|
8
|
+
The method is the same in every case:
|
|
9
|
+
|
|
10
|
+
1. Collect the measurement under labels where it should be low, and under labels
|
|
11
|
+
where it should be high.
|
|
12
|
+
2. Look at the gap between the two clusters, using percentiles rather than
|
|
13
|
+
min/max so one bad frame cannot define the boundary.
|
|
14
|
+
3. Place the threshold inside that gap.
|
|
15
|
+
4. If there is no gap, refuse. Overlapping clusters mean the pose is genuinely
|
|
16
|
+
not separable this way, and a fabricated number would only hide that.
|
|
17
|
+
|
|
18
|
+
Refusing is the important part. A tuner that always emits a value is
|
|
19
|
+
indistinguishable from one that emits noise.
|
|
20
|
+
|
|
21
|
+
One sample is taken per frame, from the hand that was *performing* the prompt.
|
|
22
|
+
Recordings routinely show both hands while only one does the work -- the other
|
|
23
|
+
rests in view -- and pooling them merges two different shapes into one cluster.
|
|
24
|
+
Observed in practice: a fist segment where the working hand measured 0.85 and the
|
|
25
|
+
resting hand 1.18 looked like a single smear from 0.68 to 1.28, and the thumb was
|
|
26
|
+
declared unseparable. Picking the frame's extreme value in the direction the
|
|
27
|
+
prompt implies recovers the performing hand without needing to be told which it
|
|
28
|
+
was.
|
|
29
|
+
|
|
30
|
+
That choice is made *after* fusion, never across cameras. Picking a frame's
|
|
31
|
+
extreme across viewpoints would be a different operation wearing the same clothes:
|
|
32
|
+
it takes the lowest of three cameras for a low cluster and the highest for a high
|
|
33
|
+
one, inventing a gap that no camera measured. A three-camera recording was fitted
|
|
34
|
+
that way and put `pinch_close` at 0.498 when the closed pinches actually sat
|
|
35
|
+
around 0.604 -- below every frame it was meant to catch. Fitting the fused hand
|
|
36
|
+
also means the number is fitted to what the engine will compare it against.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import math
|
|
42
|
+
import re
|
|
43
|
+
from dataclasses import dataclass
|
|
44
|
+
from itertools import pairwise
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
|
|
47
|
+
import numpy as np
|
|
48
|
+
|
|
49
|
+
from .config import GestureConfig, TrackingConfig
|
|
50
|
+
from .fusion import FusedHand, fuse_session
|
|
51
|
+
from .geometry import PALM_POINTS, palm_span
|
|
52
|
+
from .session import RecordedFrame, Session
|
|
53
|
+
|
|
54
|
+
PALM_POINTS_IDX = list(PALM_POINTS)
|
|
55
|
+
|
|
56
|
+
# One frame of a recording after fusion: the prompt it was captured under, and the
|
|
57
|
+
# merged hands the engine would have seen.
|
|
58
|
+
Fused = list[tuple[RecordedFrame, list[FusedHand]]]
|
|
59
|
+
|
|
60
|
+
# Percentiles used as cluster edges: the point below which almost all of the low
|
|
61
|
+
# cluster sits, and above which almost all of the high cluster sits.
|
|
62
|
+
LOW_EDGE = 90.0
|
|
63
|
+
HIGH_EDGE = 10.0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class Suggestion:
|
|
68
|
+
"""One proposed threshold, with the evidence behind it."""
|
|
69
|
+
|
|
70
|
+
section: str
|
|
71
|
+
key: str
|
|
72
|
+
current: float
|
|
73
|
+
proposed: float | None
|
|
74
|
+
reason: str
|
|
75
|
+
samples: int = 0
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def actionable(self) -> bool:
|
|
79
|
+
if self.proposed is None:
|
|
80
|
+
return False
|
|
81
|
+
return abs(self.proposed - self.current) > 1e-4
|
|
82
|
+
|
|
83
|
+
def describe(self) -> str:
|
|
84
|
+
if self.proposed is None:
|
|
85
|
+
return f" {self.key:<20} keep {self.current:<7.3f} {self.reason}"
|
|
86
|
+
arrow = "->" if self.actionable else "=="
|
|
87
|
+
return (
|
|
88
|
+
f" {self.key:<20} {self.current:<7.3f} {arrow} {self.proposed:<7.3f}"
|
|
89
|
+
f" {self.reason}"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _percentile(values: list[float], q: float) -> float:
|
|
94
|
+
return float(np.percentile(np.asarray(values, dtype=np.float64), q))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _split_threshold(
|
|
98
|
+
low: list[float], high: list[float], position: float, name: str
|
|
99
|
+
) -> tuple[float | None, str]:
|
|
100
|
+
"""Place a boundary in the gap between a low and a high cluster.
|
|
101
|
+
|
|
102
|
+
``position`` slides the result across the gap: 0.0 hugs the low cluster, 1.0
|
|
103
|
+
hugs the high one.
|
|
104
|
+
"""
|
|
105
|
+
if len(low) < 20 or len(high) < 20:
|
|
106
|
+
return None, f"not enough samples ({len(low)} low, {len(high)} high)"
|
|
107
|
+
low_edge = _percentile(low, LOW_EDGE)
|
|
108
|
+
high_edge = _percentile(high, HIGH_EDGE)
|
|
109
|
+
if high_edge <= low_edge:
|
|
110
|
+
return None, (
|
|
111
|
+
f"clusters overlap (low p{LOW_EDGE:.0f}={low_edge:.3f} >= "
|
|
112
|
+
f"high p{HIGH_EDGE:.0f}={high_edge:.3f}); {name} not separable"
|
|
113
|
+
)
|
|
114
|
+
value = low_edge + position * (high_edge - low_edge)
|
|
115
|
+
return round(value, 3), f"gap {low_edge:.3f}..{high_edge:.3f}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _segment(fused: Fused, labels: tuple[str, ...]) -> Fused:
|
|
119
|
+
return [entry for entry in fused if entry[0].label in labels]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _per_frame(fused: Fused, labels: tuple[str, ...], metric, low: bool) -> list[float]:
|
|
123
|
+
"""One sample per frame, from whichever fused hand was performing the prompt.
|
|
124
|
+
|
|
125
|
+
``low`` says which direction the prompt implies: a fist means the lowest
|
|
126
|
+
thumb-to-palm distance in the frame, an open palm the highest. The choice is
|
|
127
|
+
between the user's two hands, not between cameras -- those are already merged.
|
|
128
|
+
"""
|
|
129
|
+
samples: list[float] = []
|
|
130
|
+
for _, hands in _segment(fused, labels):
|
|
131
|
+
values = [metric(hand.features) for hand in hands]
|
|
132
|
+
finite = [v for v in values if v is not None and math.isfinite(v)]
|
|
133
|
+
if finite:
|
|
134
|
+
samples.append(min(finite) if low else max(finite))
|
|
135
|
+
return samples
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _finger_ratio(hand, finger: int) -> float:
|
|
139
|
+
"""Tip-to-wrist over knuckle-to-wrist, what `finger_extended` compares against."""
|
|
140
|
+
from .geometry import FINGERS, WRIST
|
|
141
|
+
|
|
142
|
+
tip, pip = FINGERS[finger]
|
|
143
|
+
points = hand.world
|
|
144
|
+
reach = float(np.linalg.norm(points[pip] - points[WRIST]))
|
|
145
|
+
if reach < 1e-6:
|
|
146
|
+
return float("nan")
|
|
147
|
+
return float(np.linalg.norm(points[tip] - points[WRIST])) / reach
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _finger_ratios(fused: Fused, labels: tuple[str, ...], low: bool) -> list[float]:
|
|
151
|
+
"""All four finger ratios, per frame, from the hand performing the prompt.
|
|
152
|
+
|
|
153
|
+
The hand is chosen once per frame by total extension, so all four ratios come
|
|
154
|
+
from the same hand -- picking per finger could mix a curled finger from one
|
|
155
|
+
hand with an extended one from the other.
|
|
156
|
+
"""
|
|
157
|
+
values: list[float] = []
|
|
158
|
+
for _, hands in _segment(fused, labels):
|
|
159
|
+
best: list[float] | None = None
|
|
160
|
+
best_total: float | None = None
|
|
161
|
+
for hand in hands:
|
|
162
|
+
ratios = [_finger_ratio(hand.features, finger) for finger in range(4)]
|
|
163
|
+
if not all(math.isfinite(r) for r in ratios):
|
|
164
|
+
continue
|
|
165
|
+
total = sum(ratios)
|
|
166
|
+
if best_total is None or (total < best_total if low else total > best_total):
|
|
167
|
+
best, best_total = ratios, total
|
|
168
|
+
if best is not None:
|
|
169
|
+
values += best
|
|
170
|
+
return values
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _thumb_distance(hand) -> float:
|
|
174
|
+
from .geometry import PINKY_MCP, THUMB_TIP
|
|
175
|
+
|
|
176
|
+
points = hand.world
|
|
177
|
+
return float(np.linalg.norm(points[THUMB_TIP] - points[PINKY_MCP])) / palm_span(points)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _measure(fused: Fused, labels: tuple[str, ...], pick, low: bool) -> list[float]:
|
|
181
|
+
return _per_frame(fused, labels, pick, low)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def analyse(
|
|
185
|
+
session: Session, cfg: GestureConfig, tracking: TrackingConfig | None = None
|
|
186
|
+
) -> list[Suggestion]:
|
|
187
|
+
"""Work out every threshold this recording has something to say about."""
|
|
188
|
+
out: list[Suggestion] = []
|
|
189
|
+
# Fused once, under the current thresholds, and reused by every fit below.
|
|
190
|
+
fused: Fused = list(fuse_session(session, cfg, tracking))
|
|
191
|
+
|
|
192
|
+
# --- pinch -------------------------------------------------------------
|
|
193
|
+
# Closed while pinching, open while holding ready or an open palm. The
|
|
194
|
+
# pinch_cycle segment spans both states, so it is left out of the clusters
|
|
195
|
+
# and used later by the replay test instead.
|
|
196
|
+
# A fist belongs in the open cluster even though it is not an open hand. Its
|
|
197
|
+
# thumb sits alongside the curled fingers, so it measures as pinched -- and a
|
|
198
|
+
# pinch that closes on the way into a fist latches PINCHED, which outranks the
|
|
199
|
+
# scroll branch and silently costs every scroll after it. Fitting against ready
|
|
200
|
+
# and open palms alone once proposed 0.676 on a hand whose fists sat at 0.537,
|
|
201
|
+
# which replayed to zero scrolls. If the two genuinely overlap there is no safe
|
|
202
|
+
# threshold, and declining is the right answer.
|
|
203
|
+
closed = _measure(fused, ("pinch_closed",), lambda h: h.pinch_index, low=True)
|
|
204
|
+
opened = _measure(
|
|
205
|
+
fused, ("ready", "open_palm", "fist"), lambda h: h.pinch_index, low=False
|
|
206
|
+
)
|
|
207
|
+
close_value, close_reason = _split_threshold(closed, opened, 0.30, "pinch")
|
|
208
|
+
open_value, open_reason = _split_threshold(closed, opened, 0.60, "pinch")
|
|
209
|
+
out.append(
|
|
210
|
+
Suggestion(
|
|
211
|
+
"gestures", "pinch_close", cfg.pinch_close, close_value, close_reason,
|
|
212
|
+
len(closed) + len(opened),
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
out.append(
|
|
216
|
+
Suggestion(
|
|
217
|
+
"gestures", "pinch_open", cfg.pinch_open, open_value, open_reason,
|
|
218
|
+
len(closed) + len(opened),
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# --- finger extension --------------------------------------------------
|
|
223
|
+
# A fist is the cleanest "everything curled" and an open palm the cleanest
|
|
224
|
+
# "everything extended", so the boundary is fitted across all four fingers
|
|
225
|
+
# pooled. Pooling matters: a per-finger threshold would drift with finger
|
|
226
|
+
# length, which is exactly what the ratio is designed to cancel out.
|
|
227
|
+
curled = _finger_ratios(fused, ("fist",), low=True)
|
|
228
|
+
extended = _finger_ratios(fused, ("open_palm",), low=False)
|
|
229
|
+
# Placed nearer the extended cluster than the midpoint, because the two
|
|
230
|
+
# reference poses are the extremes: a fist is fully curled, an open palm fully
|
|
231
|
+
# straight, and the fingers that matter most sit between them. In a relaxed
|
|
232
|
+
# pinching hand the ring and little finger are only half folded, and a midpoint
|
|
233
|
+
# threshold reads them as extended -- which stops the hand being `ready` at all.
|
|
234
|
+
value, reason = _split_threshold(curled, extended, 0.7, "finger extension")
|
|
235
|
+
out.append(
|
|
236
|
+
Suggestion(
|
|
237
|
+
"gestures", "finger_extended", cfg.finger_extended, value, reason,
|
|
238
|
+
len(curled) + len(extended),
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# --- thumb -------------------------------------------------------------
|
|
243
|
+
thumb_in = _per_frame(fused, ("fist",), _thumb_distance, low=True)
|
|
244
|
+
thumb_out = _per_frame(fused, ("telephone", "open_palm"), _thumb_distance, low=False)
|
|
245
|
+
value, reason = _split_threshold(thumb_in, thumb_out, 0.5, "thumb")
|
|
246
|
+
out.append(
|
|
247
|
+
Suggestion(
|
|
248
|
+
"gestures", "thumb_extended", cfg.thumb_extended, value, reason,
|
|
249
|
+
len(thumb_in) + len(thumb_out),
|
|
250
|
+
)
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# --- open palm ---------------------------------------------------------
|
|
254
|
+
# One-sided: there is no "should be low" cluster, so the threshold simply
|
|
255
|
+
# sits below what open palms actually measured, with headroom.
|
|
256
|
+
spreads = _measure(fused, ("open_palm",), lambda h: h.spread, low=False)
|
|
257
|
+
if len(spreads) >= 20:
|
|
258
|
+
floor = _percentile(spreads, HIGH_EDGE)
|
|
259
|
+
out.append(
|
|
260
|
+
Suggestion(
|
|
261
|
+
"gestures", "palm_spread", cfg.palm_spread, round(floor * 0.8, 3),
|
|
262
|
+
f"open palms measured p{HIGH_EDGE:.0f}={floor:.3f}", len(spreads),
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
else:
|
|
266
|
+
out.append(
|
|
267
|
+
Suggestion("gestures", "palm_spread", cfg.palm_spread, None, "no open-palm samples")
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
facings = _measure(fused, ("open_palm",), lambda h: h.facing, low=False)
|
|
271
|
+
if len(facings) >= 20:
|
|
272
|
+
floor = _percentile(facings, HIGH_EDGE)
|
|
273
|
+
# Never propose a negative gate; that would accept a palm facing away.
|
|
274
|
+
out.append(
|
|
275
|
+
Suggestion(
|
|
276
|
+
"gestures", "palm_facing", cfg.palm_facing, round(max(floor * 0.7, 0.0), 3),
|
|
277
|
+
f"open palms measured p{HIGH_EDGE:.0f}={floor:.3f}", len(facings),
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
# --- holding still -----------------------------------------------------
|
|
282
|
+
drift = _hold_drift(fused, ("open_palm", "telephone"))
|
|
283
|
+
if drift:
|
|
284
|
+
worst = _percentile(drift, 95.0)
|
|
285
|
+
out.append(
|
|
286
|
+
Suggestion(
|
|
287
|
+
"gestures", "hold_max_travel", cfg.hold_max_travel,
|
|
288
|
+
round(max(worst * 1.3, 0.02), 3),
|
|
289
|
+
f"you drift up to {worst:.3f} while holding still", len(drift),
|
|
290
|
+
)
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# --- swipes ------------------------------------------------------------
|
|
294
|
+
speeds = _swipe_speeds(fused)
|
|
295
|
+
if len(speeds) >= 5:
|
|
296
|
+
gentlest = _percentile(speeds, 25.0)
|
|
297
|
+
out.append(
|
|
298
|
+
Suggestion(
|
|
299
|
+
"gestures", "swipe_min_speed", cfg.swipe_min_speed,
|
|
300
|
+
round(gentlest * 0.6, 3),
|
|
301
|
+
f"your swipes peaked at p25={gentlest:.2f} units/s", len(speeds),
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
else:
|
|
305
|
+
out.append(
|
|
306
|
+
Suggestion(
|
|
307
|
+
"gestures", "swipe_min_speed", cfg.swipe_min_speed, None,
|
|
308
|
+
"too few swipes detected to fit",
|
|
309
|
+
)
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
return out
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _tracks(fused: Fused, labels: tuple[str, ...]) -> list[list[tuple[float, tuple[float, float]]]]:
|
|
316
|
+
"""Timed anchor paths, one per unbroken run of a single hand on one camera.
|
|
317
|
+
|
|
318
|
+
Split by handedness, because pooling both hands' anchors measures the distance
|
|
319
|
+
*between* the hands rather than the motion of either.
|
|
320
|
+
|
|
321
|
+
Split again at every rebase. Fused position comes from whichever camera
|
|
322
|
+
currently leads, so a change of leader moves the anchor by the parallax
|
|
323
|
+
between two viewpoints while the hand itself has not moved. Carried into a
|
|
324
|
+
drift figure that reads as wander; into a speed, as a hand that teleported.
|
|
325
|
+
The live pointer rebases at exactly these moments for the same reason.
|
|
326
|
+
"""
|
|
327
|
+
runs: dict[str, list[list[tuple[float, tuple[float, float]]]]] = {}
|
|
328
|
+
for frame, hands in _segment(fused, labels):
|
|
329
|
+
for hand in hands:
|
|
330
|
+
side = hand.features.handedness
|
|
331
|
+
chain = runs.setdefault(side, [[]])
|
|
332
|
+
if hand.rebased and chain[-1]:
|
|
333
|
+
chain.append([])
|
|
334
|
+
chain[-1].append((frame.time, hand.features.anchor))
|
|
335
|
+
return [run for chain in runs.values() for run in chain if run]
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _hold_drift(fused: Fused, labels: tuple[str, ...]) -> list[float]:
|
|
339
|
+
"""How far a palm wanders during poses meant to be held still.
|
|
340
|
+
|
|
341
|
+
Measured on the steadiest run in each segment, because the gesture that
|
|
342
|
+
cares about this -- holding a palm up to engage -- only asks one hand to be
|
|
343
|
+
still. The other hand shifting about is not the user failing to hold a pose.
|
|
344
|
+
"""
|
|
345
|
+
values: list[float] = []
|
|
346
|
+
for label in labels:
|
|
347
|
+
candidates: list[list[float]] = []
|
|
348
|
+
for run in _tracks(fused, (label,)):
|
|
349
|
+
if len(run) < 5:
|
|
350
|
+
continue
|
|
351
|
+
anchors = [point for _, point in run]
|
|
352
|
+
centre_x = float(np.mean([a[0] for a in anchors]))
|
|
353
|
+
centre_y = float(np.mean([a[1] for a in anchors]))
|
|
354
|
+
candidates.append([math.hypot(a[0] - centre_x, a[1] - centre_y) for a in anchors])
|
|
355
|
+
if candidates:
|
|
356
|
+
values += min(candidates, key=lambda drift: float(np.median(drift)))
|
|
357
|
+
return values
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _swipe_speeds(fused: Fused) -> list[float]:
|
|
361
|
+
"""Peak anchor speeds during the swipe prompt, in units per second."""
|
|
362
|
+
speeds: list[float] = []
|
|
363
|
+
# Timestamps run alongside the anchors so each pair divides by its own
|
|
364
|
+
# interval; a dropped frame otherwise reads as an impossibly fast hand.
|
|
365
|
+
for run in _tracks(fused, ("swipe",)):
|
|
366
|
+
for (t0, a), (t1, b) in pairwise(run):
|
|
367
|
+
dt = t1 - t0
|
|
368
|
+
if dt <= 1e-4:
|
|
369
|
+
continue
|
|
370
|
+
speeds.append(math.hypot(b[0] - a[0], b[1] - a[1]) / dt)
|
|
371
|
+
|
|
372
|
+
# Only the fast part of a sweep is the swipe; the turnarounds at each end are
|
|
373
|
+
# slow by definition and would drag the estimate down.
|
|
374
|
+
if not speeds:
|
|
375
|
+
return []
|
|
376
|
+
fast = _percentile(speeds, 70.0)
|
|
377
|
+
return [s for s in speeds if s >= fast]
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# Thresholds that only mean anything as a pair: the first must stay below the
|
|
381
|
+
# second. `pinch_close`/`pinch_open` is a hysteresis band -- close the pinch below
|
|
382
|
+
# one, release it above the other -- and the gap between them is what stops a hand
|
|
383
|
+
# hovering at the boundary from chattering.
|
|
384
|
+
ORDERED_PAIRS: tuple[tuple[str, str], ...] = (("pinch_close", "pinch_open"),)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _broken_pairs(cfg: GestureConfig, suggestions: list[Suggestion]) -> list[str]:
|
|
388
|
+
"""Complaints about any pair a partial write would invert.
|
|
389
|
+
|
|
390
|
+
Worth checking because writing half a pair is an easy and quiet mistake. Fit
|
|
391
|
+
both and the ordering is preserved; take only the lower one and it can land
|
|
392
|
+
above the upper, at which point a single steady hand reads as closed *and*
|
|
393
|
+
open and emits a click every frame. Observed: six clicks in one second.
|
|
394
|
+
"""
|
|
395
|
+
proposed = {s.key: s.proposed for s in suggestions if s.proposed is not None}
|
|
396
|
+
complaints: list[str] = []
|
|
397
|
+
for lower, upper in ORDERED_PAIRS:
|
|
398
|
+
low = proposed.get(lower, getattr(cfg, lower))
|
|
399
|
+
high = proposed.get(upper, getattr(cfg, upper))
|
|
400
|
+
if low >= high:
|
|
401
|
+
complaints.append(
|
|
402
|
+
f"{lower}={low} would sit at or above {upper}={high}, "
|
|
403
|
+
f"which inverts the hysteresis and makes a steady hand chatter. "
|
|
404
|
+
f"Write both, or neither."
|
|
405
|
+
)
|
|
406
|
+
return complaints
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def patch_config(path: Path, suggestions: list[Suggestion]) -> list[str]:
|
|
410
|
+
"""Rewrite thresholds in place, preserving comments and layout.
|
|
411
|
+
|
|
412
|
+
A TOML round-trip would strip the commentary that makes this file worth
|
|
413
|
+
reading, so the assignments are edited line by line instead.
|
|
414
|
+
"""
|
|
415
|
+
applied: list[str] = []
|
|
416
|
+
lines = path.read_text().splitlines(keepends=True)
|
|
417
|
+
wanted = {(s.section, s.key): s for s in suggestions if s.actionable}
|
|
418
|
+
section = ""
|
|
419
|
+
|
|
420
|
+
for index, line in enumerate(lines):
|
|
421
|
+
header = re.match(r"\s*\[([^\]]+)\]", line)
|
|
422
|
+
if header:
|
|
423
|
+
section = header.group(1)
|
|
424
|
+
continue
|
|
425
|
+
match = re.match(r"(\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*=\s*)(.+?)(\s*(?:#.*)?)$", line)
|
|
426
|
+
if not match:
|
|
427
|
+
continue
|
|
428
|
+
suggestion = wanted.get((section, match.group(2)))
|
|
429
|
+
if suggestion is None or suggestion.proposed is None:
|
|
430
|
+
continue
|
|
431
|
+
indent, key, sep, _old, trailing = match.groups()
|
|
432
|
+
lines[index] = f"{indent}{key}{sep}{suggestion.proposed}{trailing.rstrip()}\n"
|
|
433
|
+
applied.append(f"{section}.{key} = {suggestion.proposed}")
|
|
434
|
+
|
|
435
|
+
if applied:
|
|
436
|
+
path.write_text("".join(lines))
|
|
437
|
+
return applied
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def run(
|
|
441
|
+
session_path: Path,
|
|
442
|
+
cfg: GestureConfig,
|
|
443
|
+
config_path: Path | None,
|
|
444
|
+
apply: bool,
|
|
445
|
+
only: set[str] | None = None,
|
|
446
|
+
tracking: TrackingConfig | None = None,
|
|
447
|
+
) -> int:
|
|
448
|
+
session = Session.load(session_path)
|
|
449
|
+
print(f"[autotune] {session_path.name}: {session.summary()}\n")
|
|
450
|
+
|
|
451
|
+
problems = session.problems()
|
|
452
|
+
if problems:
|
|
453
|
+
print("recording quality — these limit what can be fitted:")
|
|
454
|
+
for problem in problems:
|
|
455
|
+
print(f" ! {problem}")
|
|
456
|
+
print()
|
|
457
|
+
|
|
458
|
+
suggestions = analyse(session, cfg, tracking)
|
|
459
|
+
print("threshold current proposed evidence")
|
|
460
|
+
for suggestion in suggestions:
|
|
461
|
+
print(suggestion.describe())
|
|
462
|
+
|
|
463
|
+
changes = [s for s in suggestions if s.actionable]
|
|
464
|
+
refused = [s for s in suggestions if s.proposed is None]
|
|
465
|
+
print(f"\n{len(changes)} change(s) proposed, {len(refused)} declined")
|
|
466
|
+
|
|
467
|
+
if only is not None:
|
|
468
|
+
unknown = only - {s.key for s in suggestions}
|
|
469
|
+
if unknown:
|
|
470
|
+
print(f"[autotune] no such threshold: {', '.join(sorted(unknown))}")
|
|
471
|
+
return 2
|
|
472
|
+
# A fit can be sound and still be a regression -- several of these
|
|
473
|
+
# thresholds trade one pose against another. Replaying a candidate is the
|
|
474
|
+
# only way to tell, so writing a chosen subset has to be possible.
|
|
475
|
+
suggestions = [s for s in suggestions if s.key in only]
|
|
476
|
+
broken = _broken_pairs(cfg, suggestions)
|
|
477
|
+
if broken:
|
|
478
|
+
for complaint in broken:
|
|
479
|
+
print(f"[autotune] refusing: {complaint}")
|
|
480
|
+
return 2
|
|
481
|
+
print(f"[autotune] writing only: {', '.join(sorted(only))}")
|
|
482
|
+
|
|
483
|
+
if not apply:
|
|
484
|
+
print("[autotune] dry run; pass --apply to write these into config.toml")
|
|
485
|
+
return 0
|
|
486
|
+
if config_path is None:
|
|
487
|
+
print("[autotune] no config.toml found to write to")
|
|
488
|
+
return 2
|
|
489
|
+
applied = patch_config(config_path, suggestions)
|
|
490
|
+
for entry in applied:
|
|
491
|
+
print(f"[autotune] set {entry}")
|
|
492
|
+
print(f"[autotune] updated {config_path}" if applied else "[autotune] nothing to change")
|
|
493
|
+
return 0
|
mindcontrol/calibrate.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Nine-point gaze calibration.
|
|
2
|
+
|
|
3
|
+
Runs as its own process, for two reasons: a fullscreen OpenCV window has to own
|
|
4
|
+
the main thread, and the camera can only be held by one process at a time, so the
|
|
5
|
+
menu-bar app releases its cameras and hands them over for the duration.
|
|
6
|
+
|
|
7
|
+
You look at each dot; the app records what your eyes and head look like while you
|
|
8
|
+
do. Fitting those pairs gives the mapping from appearance to screen position that
|
|
9
|
+
`GazeModel` uses from then on. The result is written to disk, and the app reloads
|
|
10
|
+
it when it takes the camera back.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
import cv2
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
from .capture import CameraWorker
|
|
22
|
+
from .config import GAZE_MODEL_PATH, Config, load
|
|
23
|
+
from .control.mouse import main_display_bounds
|
|
24
|
+
from .tracking.gaze import GazeModel, GazeTracker
|
|
25
|
+
|
|
26
|
+
WINDOW = "mindcontrol calibration"
|
|
27
|
+
# Inset from the edges: a dot in the very corner is uncomfortable to fixate and
|
|
28
|
+
# tends to be tracked with the head rather than the eyes.
|
|
29
|
+
GRID = (0.08, 0.5, 0.92)
|
|
30
|
+
TARGETS = [(x, y) for y in GRID for x in GRID]
|
|
31
|
+
|
|
32
|
+
SETTLE_S = 1.0
|
|
33
|
+
SAMPLES_PER_TARGET = 30
|
|
34
|
+
SAMPLE_TIMEOUT_S = 4.0
|
|
35
|
+
MIN_SAMPLES_PER_TARGET = 8
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _canvas(width: int, height: int) -> np.ndarray:
|
|
39
|
+
return np.zeros((height, width, 3), dtype=np.uint8)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _draw_target(
|
|
43
|
+
frame: np.ndarray, target: tuple[float, float], progress: float, phase: str
|
|
44
|
+
) -> None:
|
|
45
|
+
height, width = frame.shape[:2]
|
|
46
|
+
cx, cy = int(target[0] * width), int(target[1] * height)
|
|
47
|
+
|
|
48
|
+
# The ring closing in on the dot shows how much longer to hold still.
|
|
49
|
+
radius = int(46 - 26 * progress)
|
|
50
|
+
colour = (90, 210, 120) if phase == "recording" else (120, 120, 120)
|
|
51
|
+
cv2.circle(frame, (cx, cy), max(radius, 12), colour, 2, cv2.LINE_AA)
|
|
52
|
+
cv2.circle(frame, (cx, cy), 6, (255, 255, 255), -1, cv2.LINE_AA)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _draw_caption(frame: np.ndarray, lines: list[str]) -> None:
|
|
56
|
+
height, width = frame.shape[:2]
|
|
57
|
+
for index, text in enumerate(lines):
|
|
58
|
+
size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 1)[0]
|
|
59
|
+
cv2.putText(
|
|
60
|
+
frame,
|
|
61
|
+
text,
|
|
62
|
+
((width - size[0]) // 2, int(height * 0.86) + index * 30),
|
|
63
|
+
cv2.FONT_HERSHEY_SIMPLEX,
|
|
64
|
+
0.7,
|
|
65
|
+
(170, 170, 170),
|
|
66
|
+
1,
|
|
67
|
+
cv2.LINE_AA,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def run(cfg: Config | None = None) -> int:
|
|
72
|
+
"""Drive the calibration. Returns a process exit code."""
|
|
73
|
+
cfg = cfg or load()
|
|
74
|
+
# Sized to the main display, matching the fractions `Mouse.move_to_fraction`
|
|
75
|
+
# will later interpret. Calibrating against the desktop union would record
|
|
76
|
+
# targets for a coordinate space the fullscreen window never covered.
|
|
77
|
+
left, top, right, bottom = main_display_bounds()
|
|
78
|
+
width, height = int(right - left), int(bottom - top)
|
|
79
|
+
|
|
80
|
+
camera = CameraWorker(cfg.cameras.primary_gaze, cfg.cameras)
|
|
81
|
+
if not camera.start():
|
|
82
|
+
print(f"[calibrate] {camera.error}", file=sys.stderr)
|
|
83
|
+
return 2
|
|
84
|
+
|
|
85
|
+
tracker = GazeTracker(cfg.tracking)
|
|
86
|
+
cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL)
|
|
87
|
+
cv2.setWindowProperty(WINDOW, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
|
|
88
|
+
|
|
89
|
+
features: list[np.ndarray] = []
|
|
90
|
+
targets: list[tuple[float, float]] = []
|
|
91
|
+
aborted = False
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
# Let auto-exposure settle before the first dot, or the first point is
|
|
95
|
+
# collected from a half-dark frame.
|
|
96
|
+
_wait_for_camera(camera, tracker, width, height)
|
|
97
|
+
|
|
98
|
+
for index, target in enumerate(TARGETS):
|
|
99
|
+
collected = _collect_point(camera, tracker, cfg, target, index, width, height)
|
|
100
|
+
if collected is None:
|
|
101
|
+
aborted = True
|
|
102
|
+
break
|
|
103
|
+
if len(collected) < MIN_SAMPLES_PER_TARGET:
|
|
104
|
+
print(
|
|
105
|
+
f"[calibrate] only {len(collected)} usable samples for point "
|
|
106
|
+
f"{index + 1}; keeping them but the fit will be weaker"
|
|
107
|
+
)
|
|
108
|
+
features.extend(collected)
|
|
109
|
+
targets.extend([target] * len(collected))
|
|
110
|
+
finally:
|
|
111
|
+
cv2.destroyAllWindows()
|
|
112
|
+
# macOS needs a few event-loop turns to actually tear the window down.
|
|
113
|
+
for _ in range(4):
|
|
114
|
+
cv2.waitKey(1)
|
|
115
|
+
tracker.close()
|
|
116
|
+
camera.stop()
|
|
117
|
+
|
|
118
|
+
if aborted:
|
|
119
|
+
print("[calibrate] cancelled; existing calibration left untouched")
|
|
120
|
+
return 1
|
|
121
|
+
if len(features) < len(TARGETS) * MIN_SAMPLES_PER_TARGET:
|
|
122
|
+
print("[calibrate] not enough usable samples; nothing saved", file=sys.stderr)
|
|
123
|
+
return 3
|
|
124
|
+
|
|
125
|
+
model = GazeModel.fit(np.vstack(features), np.array(targets, dtype=np.float64))
|
|
126
|
+
model.save(GAZE_MODEL_PATH)
|
|
127
|
+
print(
|
|
128
|
+
f"[calibrate] saved {GAZE_MODEL_PATH} from {len(features)} samples; "
|
|
129
|
+
f"mean error {model.quality * 100:.1f}% of screen"
|
|
130
|
+
)
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _wait_for_camera(camera: CameraWorker, tracker: GazeTracker, width: int, height: int) -> None:
|
|
135
|
+
deadline = time.monotonic() + 2.5
|
|
136
|
+
while time.monotonic() < deadline:
|
|
137
|
+
frame = camera.latest()
|
|
138
|
+
if frame is not None:
|
|
139
|
+
tracker.process(frame)
|
|
140
|
+
canvas = _canvas(width, height)
|
|
141
|
+
_draw_caption(
|
|
142
|
+
canvas,
|
|
143
|
+
[
|
|
144
|
+
"Look at each dot until its ring closes.",
|
|
145
|
+
"Keep your head still and comfortable. Esc cancels.",
|
|
146
|
+
],
|
|
147
|
+
)
|
|
148
|
+
cv2.imshow(WINDOW, canvas)
|
|
149
|
+
if cv2.waitKey(30) & 0xFF == 27:
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _collect_point(
|
|
154
|
+
camera: CameraWorker,
|
|
155
|
+
tracker: GazeTracker,
|
|
156
|
+
cfg: Config,
|
|
157
|
+
target: tuple[float, float],
|
|
158
|
+
index: int,
|
|
159
|
+
width: int,
|
|
160
|
+
height: int,
|
|
161
|
+
) -> list[np.ndarray] | None:
|
|
162
|
+
"""Show one dot and gather samples. None means the user pressed Escape."""
|
|
163
|
+
samples: list[np.ndarray] = []
|
|
164
|
+
started = time.monotonic()
|
|
165
|
+
last_sequence = -1
|
|
166
|
+
|
|
167
|
+
while True:
|
|
168
|
+
elapsed = time.monotonic() - started
|
|
169
|
+
recording = elapsed >= SETTLE_S
|
|
170
|
+
if recording:
|
|
171
|
+
frame = camera.latest()
|
|
172
|
+
if frame is not None and frame.sequence != last_sequence:
|
|
173
|
+
last_sequence = frame.sequence
|
|
174
|
+
observation = tracker.process(frame)
|
|
175
|
+
# Blinks and lost faces are skipped rather than averaged in;
|
|
176
|
+
# a closed eye says nothing about where you are looking.
|
|
177
|
+
if observation.usable and observation.openness >= cfg.gaze.blink_ear:
|
|
178
|
+
assert observation.features is not None
|
|
179
|
+
samples.append(observation.features)
|
|
180
|
+
if len(samples) >= SAMPLES_PER_TARGET or elapsed > SETTLE_S + SAMPLE_TIMEOUT_S:
|
|
181
|
+
return samples
|
|
182
|
+
progress = len(samples) / SAMPLES_PER_TARGET
|
|
183
|
+
else:
|
|
184
|
+
progress = elapsed / SETTLE_S
|
|
185
|
+
|
|
186
|
+
canvas = _canvas(width, height)
|
|
187
|
+
_draw_target(canvas, target, min(progress, 1.0), "recording" if recording else "settle")
|
|
188
|
+
_draw_caption(canvas, [f"Point {index + 1} of {len(TARGETS)}"])
|
|
189
|
+
cv2.imshow(WINDOW, canvas)
|
|
190
|
+
if cv2.waitKey(15) & 0xFF == 27:
|
|
191
|
+
return None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def main() -> int:
|
|
195
|
+
return run()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if __name__ == "__main__":
|
|
199
|
+
raise SystemExit(main())
|