cheta 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cheta/__init__.py +9 -0
- cheta/calibrate.py +134 -0
- cheta/cli.py +262 -0
- cheta/face/__init__.py +11 -0
- cheta/face/capture.py +33 -0
- cheta/face/embedder.py +95 -0
- cheta/face/matcher.py +30 -0
- cheta/fusion.py +54 -0
- cheta/identity.py +552 -0
- cheta/liveness.py +55 -0
- cheta/memory/__init__.py +6 -0
- cheta/memory/embedder.py +42 -0
- cheta/memory/types.py +17 -0
- cheta/models.py +164 -0
- cheta/person.py +14 -0
- cheta/presence.py +48 -0
- cheta/py.typed +0 -0
- cheta/robot.py +133 -0
- cheta/speech.py +108 -0
- cheta/storage/__init__.py +5 -0
- cheta/storage/crypto.py +147 -0
- cheta/storage/db.py +269 -0
- cheta/voice/__init__.py +33 -0
- cheta/voice/audio.py +35 -0
- cheta/voice/embedder.py +41 -0
- cheta/voice/vad.py +67 -0
- cheta-0.1.0.dist-info/METADATA +161 -0
- cheta-0.1.0.dist-info/RECORD +31 -0
- cheta-0.1.0.dist-info/WHEEL +4 -0
- cheta-0.1.0.dist-info/entry_points.txt +2 -0
- cheta-0.1.0.dist-info/licenses/LICENSE +202 -0
cheta/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""cheta: on-device face and voice recognition, memory and forgetting for Reachy Mini apps."""
|
|
2
|
+
|
|
3
|
+
from cheta.identity import Identity
|
|
4
|
+
from cheta.memory import Memory
|
|
5
|
+
from cheta.person import Person
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = ["Identity", "Memory", "Person", "__version__"]
|
cheta/calibrate.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Pick a voice threshold from your own recordings.
|
|
2
|
+
|
|
3
|
+
Speaker-embedding scores depend on the microphone, the room and the people, so no single default
|
|
4
|
+
fits everyone. Record a few clips of each person, then measure how high *genuine* scores (a clip
|
|
5
|
+
against the same person) and *impostor* scores (a clip against everyone else) actually are.
|
|
6
|
+
``recommend_threshold`` picks the lowest threshold that keeps false accepts under your target.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
from numpy.typing import NDArray
|
|
18
|
+
|
|
19
|
+
from cheta.robot import resample
|
|
20
|
+
from cheta.speech import read_wav
|
|
21
|
+
from cheta.voice import (
|
|
22
|
+
SAMPLE_RATE,
|
|
23
|
+
Audio,
|
|
24
|
+
VoiceActivityDetector,
|
|
25
|
+
VoiceEmbedder,
|
|
26
|
+
split_windows,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
MIN_SPEECH_S = 1.5
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Recommendation:
|
|
34
|
+
"""A suggested threshold and what it would have done on the recorded data."""
|
|
35
|
+
|
|
36
|
+
threshold: float
|
|
37
|
+
false_accept_rate: float # impostor probes scoring at or above the threshold
|
|
38
|
+
false_reject_rate: float # genuine probes scoring below it
|
|
39
|
+
n_genuine: int
|
|
40
|
+
n_impostor: int
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def recommend_threshold(
|
|
44
|
+
genuine: Sequence[float], impostor: Sequence[float], max_far: float = 0.01
|
|
45
|
+
) -> Recommendation:
|
|
46
|
+
"""Lowest threshold whose false-accept rate on ``impostor`` is at most ``max_far``.
|
|
47
|
+
|
|
48
|
+
Lower thresholds reject fewer genuine users, so the lowest one that meets the false-accept
|
|
49
|
+
target is the best. With few impostor scores the rate is coarse; collect more clips.
|
|
50
|
+
"""
|
|
51
|
+
if not genuine or not impostor:
|
|
52
|
+
raise ValueError("need at least one genuine and one impostor score")
|
|
53
|
+
if not 0.0 <= max_far < 1.0:
|
|
54
|
+
raise ValueError("max_far must be in [0, 1)")
|
|
55
|
+
gen = np.sort(np.asarray(genuine, dtype=np.float64))
|
|
56
|
+
imp = np.sort(np.asarray(impostor, dtype=np.float64))
|
|
57
|
+
# Candidates: just above each impostor score, so "score >= threshold" excludes it.
|
|
58
|
+
candidates = np.unique(imp + 1e-6)
|
|
59
|
+
chosen = next(float(t) for t in candidates if float(np.mean(imp >= t)) <= max_far)
|
|
60
|
+
chosen = math.ceil(chosen * 1000) / 1000 # round *up* so the false-accept target still holds
|
|
61
|
+
return Recommendation(
|
|
62
|
+
threshold=chosen,
|
|
63
|
+
false_accept_rate=float(np.mean(imp >= chosen)),
|
|
64
|
+
false_reject_rate=float(np.mean(gen < chosen)),
|
|
65
|
+
n_genuine=len(gen),
|
|
66
|
+
n_impostor=len(imp),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _n(x: NDArray[np.float32]) -> NDArray[np.float32]:
|
|
71
|
+
return x / (np.linalg.norm(x) or 1.0)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def score_voices(
|
|
75
|
+
enrol: Mapping[str, Audio],
|
|
76
|
+
probes: Mapping[str, Sequence[Audio]],
|
|
77
|
+
vad: VoiceActivityDetector,
|
|
78
|
+
embedder: VoiceEmbedder,
|
|
79
|
+
) -> tuple[list[float], list[float]]:
|
|
80
|
+
"""Return ``(genuine, impostor)`` scores the way ``Identity`` would produce them.
|
|
81
|
+
|
|
82
|
+
Each person is enrolled from their ``enrol`` clip (windows plus the whole utterance, as in
|
|
83
|
+
``enroll()``). Every probe clip is split into windows; a window's score for a person is its
|
|
84
|
+
best cosine similarity to that person's samples. The genuine score is the window against its
|
|
85
|
+
own person, the impostor score is the *best* other person (the open-set worst case).
|
|
86
|
+
"""
|
|
87
|
+
if len(enrol) < 2:
|
|
88
|
+
raise ValueError("need at least two people to measure impostor scores")
|
|
89
|
+
templates: dict[str, list[NDArray[np.float32]]] = {}
|
|
90
|
+
for name, clip in enrol.items():
|
|
91
|
+
speech = vad.speech_only(clip)
|
|
92
|
+
if len(speech) < MIN_SPEECH_S * SAMPLE_RATE:
|
|
93
|
+
raise ValueError(f"not enough speech in the enrolment clip for {name!r}")
|
|
94
|
+
embs = [_n(embedder.embed(w)) for w in split_windows(speech)]
|
|
95
|
+
if len(embs) > 1:
|
|
96
|
+
embs.append(_n(embedder.embed(speech)))
|
|
97
|
+
templates[name] = embs
|
|
98
|
+
genuine: list[float] = []
|
|
99
|
+
impostor: list[float] = []
|
|
100
|
+
for name, clips in probes.items():
|
|
101
|
+
if name not in templates:
|
|
102
|
+
raise ValueError(f"probe clips for {name!r} have no enrolment clip")
|
|
103
|
+
for clip in clips:
|
|
104
|
+
speech = vad.speech_only(clip)
|
|
105
|
+
if len(speech) < MIN_SPEECH_S * SAMPLE_RATE:
|
|
106
|
+
continue
|
|
107
|
+
for window in split_windows(speech):
|
|
108
|
+
q = _n(embedder.embed(window))
|
|
109
|
+
scores = {n: max(float(q @ t) for t in ts) for n, ts in templates.items()}
|
|
110
|
+
genuine.append(scores[name])
|
|
111
|
+
impostor.append(max(s for n, s in scores.items() if n != name))
|
|
112
|
+
if not genuine:
|
|
113
|
+
raise ValueError("no probe clip had enough speech")
|
|
114
|
+
return genuine, impostor
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def load_voice_dir(path: Path) -> tuple[dict[str, Audio], dict[str, list[Audio]]]:
|
|
118
|
+
"""Read ``<name>_<n>.wav`` files: a person's lowest ``n`` enrols them, the rest are probes."""
|
|
119
|
+
clips: dict[str, list[tuple[int, Audio]]] = {}
|
|
120
|
+
for wav in sorted(Path(path).glob("*.wav")):
|
|
121
|
+
name, _, number = wav.stem.rpartition("_")
|
|
122
|
+
if not name or not number.isdigit():
|
|
123
|
+
raise ValueError(f"{wav.name}: expected a file name like <name>_<number>.wav")
|
|
124
|
+
samples, rate = read_wav(wav)
|
|
125
|
+
clips.setdefault(name, []).append((int(number), resample(samples, rate, SAMPLE_RATE)))
|
|
126
|
+
enrol: dict[str, Audio] = {}
|
|
127
|
+
probes: dict[str, list[Audio]] = {}
|
|
128
|
+
for name, items in clips.items():
|
|
129
|
+
items.sort(key=lambda item: item[0])
|
|
130
|
+
if len(items) < 2:
|
|
131
|
+
raise ValueError(f"{name!r} needs at least two clips: one to enrol, one to test")
|
|
132
|
+
enrol[name] = items[0][1]
|
|
133
|
+
probes[name] = [audio for _, audio in items[1:]]
|
|
134
|
+
return enrol, probes
|
cheta/cli.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""Command line interface: ``cheta enroll|list|forget|demo``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from cheta.calibrate import load_voice_dir, recommend_threshold, score_voices
|
|
11
|
+
from cheta.face.matcher import DEFAULT_THRESHOLD
|
|
12
|
+
from cheta.identity import DEFAULT_VOICE_THRESHOLD, Identity
|
|
13
|
+
from cheta.person import Person
|
|
14
|
+
from cheta.voice import EcapaEmbedder, SileroVAD
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _identity(args: argparse.Namespace) -> Identity:
|
|
18
|
+
return Identity(
|
|
19
|
+
app=args.app,
|
|
20
|
+
camera=args.camera,
|
|
21
|
+
data_dir=args.data_dir,
|
|
22
|
+
threshold=getattr(args, "threshold", DEFAULT_THRESHOLD),
|
|
23
|
+
liveness=not getattr(args, "no_liveness", False),
|
|
24
|
+
listen_voice=getattr(args, "voice", False),
|
|
25
|
+
on_prompt=print,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def cmd_enroll(args: argparse.Namespace) -> int:
|
|
30
|
+
"""Enroll a person from the camera."""
|
|
31
|
+
if not args.yes:
|
|
32
|
+
answer = input(
|
|
33
|
+
f"Store a face template for {args.name!r} on this device? "
|
|
34
|
+
"Raw images are never saved. [y/N] "
|
|
35
|
+
)
|
|
36
|
+
if answer.strip().lower() not in ("y", "yes"):
|
|
37
|
+
print("Cancelled: consent not given.")
|
|
38
|
+
return 1
|
|
39
|
+
idt = _identity(args)
|
|
40
|
+
try:
|
|
41
|
+
person = idt.enroll(args.name, consent=True, on_prompt=print, voice=not args.no_voice)
|
|
42
|
+
finally:
|
|
43
|
+
idt.close()
|
|
44
|
+
what = "face samples" if args.no_voice else "face and voice samples"
|
|
45
|
+
print(f"Enrolled {person.name} ({person.id}) with encrypted {what}.")
|
|
46
|
+
return 0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cmd_list(args: argparse.Namespace) -> int:
|
|
50
|
+
"""List enrolled people."""
|
|
51
|
+
idt = _identity(args)
|
|
52
|
+
try:
|
|
53
|
+
people = idt.store.list_people()
|
|
54
|
+
finally:
|
|
55
|
+
idt.close()
|
|
56
|
+
for p in people:
|
|
57
|
+
print(f"{p.id} {p.name}")
|
|
58
|
+
if not people:
|
|
59
|
+
print("No one enrolled.")
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def cmd_demo(args: argparse.Namespace) -> int:
|
|
64
|
+
"""Live recognition: print who is in view and arrival/left events until Ctrl-C."""
|
|
65
|
+
idt = _identity(args)
|
|
66
|
+
idt.on_person_arrived(lambda p: print(f"[arrived] {p.name} ({p.confidence:.2f})"))
|
|
67
|
+
idt.on_person_left(lambda p: print(f"[left] {p.name}"))
|
|
68
|
+
idt.start()
|
|
69
|
+
print("Watching the camera. Press Ctrl-C to stop.")
|
|
70
|
+
last = ""
|
|
71
|
+
try:
|
|
72
|
+
while True:
|
|
73
|
+
who = idt.who_is_here()
|
|
74
|
+
line = f"{who.name} ({who.confidence:.2f})" if who else "nobody known"
|
|
75
|
+
if line != last:
|
|
76
|
+
print(f"here: {line}")
|
|
77
|
+
last = line
|
|
78
|
+
time.sleep(0.2)
|
|
79
|
+
except KeyboardInterrupt:
|
|
80
|
+
pass
|
|
81
|
+
finally:
|
|
82
|
+
idt.close()
|
|
83
|
+
return 0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def cmd_forget(args: argparse.Namespace) -> int:
|
|
87
|
+
"""Hard-delete a person: face samples and memories."""
|
|
88
|
+
idt = _identity(args)
|
|
89
|
+
try:
|
|
90
|
+
matches = idt.store.find_by_name(args.name) if args.name else []
|
|
91
|
+
if args.id:
|
|
92
|
+
person = idt.store.get_person(args.id)
|
|
93
|
+
matches = [person] if person else []
|
|
94
|
+
if not matches:
|
|
95
|
+
print("No such person.")
|
|
96
|
+
return 1
|
|
97
|
+
if len(matches) > 1:
|
|
98
|
+
print("Several people match; pass --id:")
|
|
99
|
+
for p in matches:
|
|
100
|
+
print(f" {p.id} {p.name}")
|
|
101
|
+
return 1
|
|
102
|
+
person = matches[0]
|
|
103
|
+
if not args.yes:
|
|
104
|
+
answer = input(f"Permanently delete {person.name} and all their data? [y/N] ")
|
|
105
|
+
if answer.strip().lower() not in ("y", "yes"):
|
|
106
|
+
print("Cancelled.")
|
|
107
|
+
return 1
|
|
108
|
+
idt.forget(person)
|
|
109
|
+
finally:
|
|
110
|
+
idt.close()
|
|
111
|
+
print(f"Forgot {person.name}.")
|
|
112
|
+
return 0
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def cmd_listen(args: argparse.Namespace) -> int:
|
|
116
|
+
"""Record a few seconds from the microphone and say who is speaking."""
|
|
117
|
+
idt = Identity(
|
|
118
|
+
app=args.app,
|
|
119
|
+
camera=args.camera,
|
|
120
|
+
data_dir=args.data_dir,
|
|
121
|
+
voice_threshold=args.threshold,
|
|
122
|
+
)
|
|
123
|
+
try:
|
|
124
|
+
print(f"Speak for {args.seconds:.0f} seconds...")
|
|
125
|
+
who = idt.identify_voice(seconds=args.seconds)
|
|
126
|
+
finally:
|
|
127
|
+
idt.close()
|
|
128
|
+
print(f"speaker: {who.name} ({who.confidence:.2f})" if who else "speaker: unknown")
|
|
129
|
+
return 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def cmd_calibrate(args: argparse.Namespace) -> int:
|
|
133
|
+
"""Recommend a voice threshold from recordings in a folder (nothing is enrolled or stored)."""
|
|
134
|
+
try:
|
|
135
|
+
enrol, probes = load_voice_dir(Path(args.dir))
|
|
136
|
+
genuine, impostor = score_voices(enrol, probes, SileroVAD(), EcapaEmbedder())
|
|
137
|
+
rec = recommend_threshold(genuine, impostor, max_far=args.max_far)
|
|
138
|
+
except ValueError as exc:
|
|
139
|
+
print(f"Cannot calibrate: {exc}")
|
|
140
|
+
return 1
|
|
141
|
+
print(f"{len(enrol)} people, {rec.n_genuine} genuine and {rec.n_impostor} impostor scores.")
|
|
142
|
+
print(f"genuine : median {sorted(genuine)[len(genuine) // 2]:.2f}, lowest {min(genuine):.2f}")
|
|
143
|
+
print(
|
|
144
|
+
f"impostor : median {sorted(impostor)[len(impostor) // 2]:.2f}, highest {max(impostor):.2f}"
|
|
145
|
+
)
|
|
146
|
+
print(
|
|
147
|
+
f"Recommended voice_threshold = {rec.threshold:.3f} "
|
|
148
|
+
f"(false accepts {rec.false_accept_rate:.1%}, false rejects {rec.false_reject_rate:.1%})"
|
|
149
|
+
)
|
|
150
|
+
if rec.n_impostor < 100:
|
|
151
|
+
print("Note: few impostor scores; the false-accept estimate is coarse. Use more clips.")
|
|
152
|
+
return 0
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _one_person(idt: Identity, name: str) -> Person | None:
|
|
156
|
+
matches = idt.store.find_by_name(name)
|
|
157
|
+
if not matches:
|
|
158
|
+
print(f"No one called {name!r} is enrolled.")
|
|
159
|
+
return None
|
|
160
|
+
if len(matches) > 1:
|
|
161
|
+
print(f"Several people are called {name!r}; use `cheta list` to tell them apart.")
|
|
162
|
+
return None
|
|
163
|
+
return matches[0]
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def cmd_remember(args: argparse.Namespace) -> int:
|
|
167
|
+
"""Store a fact about an enrolled person."""
|
|
168
|
+
idt = _identity(args)
|
|
169
|
+
try:
|
|
170
|
+
person = _one_person(idt, args.name)
|
|
171
|
+
if person is None:
|
|
172
|
+
return 1
|
|
173
|
+
idt.remember(person, args.text, expires_in_days=args.expires_days)
|
|
174
|
+
finally:
|
|
175
|
+
idt.close()
|
|
176
|
+
print(f"Remembered for {person.name}: {args.text}")
|
|
177
|
+
return 0
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def cmd_recall(args: argparse.Namespace) -> int:
|
|
181
|
+
"""Look up what is remembered about a person."""
|
|
182
|
+
idt = _identity(args)
|
|
183
|
+
try:
|
|
184
|
+
person = _one_person(idt, args.name)
|
|
185
|
+
if person is None:
|
|
186
|
+
return 1
|
|
187
|
+
memories = idt.recall(person, args.query, k=args.k)
|
|
188
|
+
finally:
|
|
189
|
+
idt.close()
|
|
190
|
+
for m in memories:
|
|
191
|
+
print(f"{m.score:.2f} {m.text}")
|
|
192
|
+
if not memories:
|
|
193
|
+
print("Nothing remembered.")
|
|
194
|
+
return 0
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
198
|
+
"""Build the argument parser."""
|
|
199
|
+
parser = argparse.ArgumentParser(prog="cheta", description=__doc__)
|
|
200
|
+
parser.add_argument("--app", default="cheta", help="app namespace (default: cheta)")
|
|
201
|
+
parser.add_argument("--data-dir", default="~/.cheta")
|
|
202
|
+
parser.add_argument("--camera", choices=["webcam", "reachy"], default="webcam")
|
|
203
|
+
sub = parser.add_subparsers(dest="command")
|
|
204
|
+
|
|
205
|
+
enroll = sub.add_parser("enroll", help="enroll a person from the camera")
|
|
206
|
+
enroll.add_argument("--name", required=True)
|
|
207
|
+
enroll.add_argument("--no-voice", action="store_true", help="enroll by face only")
|
|
208
|
+
enroll.add_argument("--yes", action="store_true", help="confirm consent without prompting")
|
|
209
|
+
enroll.set_defaults(func=cmd_enroll)
|
|
210
|
+
|
|
211
|
+
demo = sub.add_parser("demo", help="live recognition from the camera")
|
|
212
|
+
demo.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
|
|
213
|
+
demo.add_argument("--voice", action="store_true", help="also listen to the mic and fuse voice")
|
|
214
|
+
demo.add_argument(
|
|
215
|
+
"--no-liveness", action="store_true", help="skip the head-turn check (photos will pass!)"
|
|
216
|
+
)
|
|
217
|
+
demo.set_defaults(func=cmd_demo)
|
|
218
|
+
|
|
219
|
+
forget = sub.add_parser("forget", help="delete a person and all their data")
|
|
220
|
+
who = forget.add_mutually_exclusive_group(required=True)
|
|
221
|
+
who.add_argument("--name")
|
|
222
|
+
who.add_argument("--id")
|
|
223
|
+
forget.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
|
|
224
|
+
forget.set_defaults(func=cmd_forget)
|
|
225
|
+
|
|
226
|
+
listen = sub.add_parser("listen", help="identify the speaker from the microphone")
|
|
227
|
+
listen.add_argument("--seconds", type=float, default=4.0)
|
|
228
|
+
listen.add_argument("--threshold", type=float, default=DEFAULT_VOICE_THRESHOLD)
|
|
229
|
+
listen.set_defaults(func=cmd_listen)
|
|
230
|
+
|
|
231
|
+
calibrate = sub.add_parser(
|
|
232
|
+
"calibrate", help="recommend a voice threshold from recordings (<name>_<n>.wav)"
|
|
233
|
+
)
|
|
234
|
+
calibrate.add_argument("dir", help="folder of <name>_1.wav (enrol), <name>_2.wav ... (tests)")
|
|
235
|
+
calibrate.add_argument("--max-far", type=float, default=0.01, help="false-accept target")
|
|
236
|
+
calibrate.set_defaults(func=cmd_calibrate)
|
|
237
|
+
|
|
238
|
+
remember = sub.add_parser("remember", help="store a fact about an enrolled person")
|
|
239
|
+
remember.add_argument("--name", required=True)
|
|
240
|
+
remember.add_argument("text")
|
|
241
|
+
remember.add_argument("--expires-days", type=int, default=None)
|
|
242
|
+
remember.set_defaults(func=cmd_remember)
|
|
243
|
+
|
|
244
|
+
recall = sub.add_parser("recall", help="look up facts about an enrolled person")
|
|
245
|
+
recall.add_argument("--name", required=True)
|
|
246
|
+
recall.add_argument("query")
|
|
247
|
+
recall.add_argument("-k", type=int, default=5)
|
|
248
|
+
recall.set_defaults(func=cmd_recall)
|
|
249
|
+
|
|
250
|
+
lst = sub.add_parser("list", help="list enrolled people")
|
|
251
|
+
lst.set_defaults(func=cmd_list)
|
|
252
|
+
return parser
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
256
|
+
"""CLI entry point."""
|
|
257
|
+
parser = build_parser()
|
|
258
|
+
args = parser.parse_args(argv)
|
|
259
|
+
if not hasattr(args, "func"):
|
|
260
|
+
parser.print_help()
|
|
261
|
+
return 0
|
|
262
|
+
return int(args.func(args))
|
cheta/face/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Face detection and embedding."""
|
|
2
|
+
|
|
3
|
+
from cheta.face.embedder import (
|
|
4
|
+
FaceEmbedder,
|
|
5
|
+
FaceObservation,
|
|
6
|
+
SFaceEmbedder,
|
|
7
|
+
cosine_similarity,
|
|
8
|
+
head_yaw,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = ["FaceEmbedder", "FaceObservation", "SFaceEmbedder", "cosine_similarity", "head_yaw"]
|
cheta/face/capture.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Frame sources. Frames are held in memory only long enough to embed them."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol, cast
|
|
6
|
+
|
|
7
|
+
import cv2
|
|
8
|
+
|
|
9
|
+
from cheta.face.embedder import Frame
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FrameSource(Protocol):
|
|
13
|
+
"""Anything that yields BGR frames."""
|
|
14
|
+
|
|
15
|
+
def read(self) -> Frame | None: ...
|
|
16
|
+
|
|
17
|
+
def close(self) -> None: ...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class WebcamSource:
|
|
21
|
+
"""Laptop webcam via OpenCV."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, index: int = 0) -> None:
|
|
24
|
+
self._cap = cv2.VideoCapture(index)
|
|
25
|
+
if not self._cap.isOpened():
|
|
26
|
+
raise RuntimeError(f"could not open webcam {index}")
|
|
27
|
+
|
|
28
|
+
def read(self) -> Frame | None:
|
|
29
|
+
ok, frame = self._cap.read()
|
|
30
|
+
return cast(Frame, frame) if ok else None
|
|
31
|
+
|
|
32
|
+
def close(self) -> None:
|
|
33
|
+
self._cap.release()
|
cheta/face/embedder.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Face detection (YuNet) and embedding (SFace) via OpenCV. Raw frames are never stored."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Protocol
|
|
8
|
+
|
|
9
|
+
import cv2
|
|
10
|
+
import numpy as np
|
|
11
|
+
from numpy.typing import NDArray
|
|
12
|
+
|
|
13
|
+
from cheta.models import download_model
|
|
14
|
+
|
|
15
|
+
Frame = NDArray[np.uint8]
|
|
16
|
+
MIN_DETECTION_SCORE = 0.85
|
|
17
|
+
MIN_FACE_SIZE = 80 # pixels, shortest side of the detected box
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class FaceObservation:
|
|
22
|
+
"""What one frame shows: the face embedding and a head-turn (yaw) proxy.
|
|
23
|
+
|
|
24
|
+
``yaw`` is the nose's horizontal offset from the face centre in units of eye distance:
|
|
25
|
+
about 0 facing the camera, growing in magnitude as the head turns.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
embedding: NDArray[np.float32]
|
|
29
|
+
yaw: float
|
|
30
|
+
center: tuple[float, float] | None = None # face centre in frame pixels (u right, v down)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class FaceEmbedder(Protocol):
|
|
34
|
+
"""Looks at one frame containing a single face."""
|
|
35
|
+
|
|
36
|
+
def observe(self, frame: Frame) -> FaceObservation | None:
|
|
37
|
+
"""Return the observation, or None when there is not exactly one usable face."""
|
|
38
|
+
...
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def head_yaw(face: NDArray[np.float32]) -> float:
|
|
42
|
+
"""Yaw proxy from a YuNet detection row (landmarks: eyes, nose, mouth corners).
|
|
43
|
+
|
|
44
|
+
A real head turning moves the nose (which sits in front of the eye plane) sideways relative
|
|
45
|
+
to the eyes and mouth. A flat photo, even tilted or moved, keeps this ratio almost constant.
|
|
46
|
+
"""
|
|
47
|
+
r_eye, l_eye, nose, r_mouth, l_mouth = (face[4 + 2 * i : 6 + 2 * i] for i in range(5))
|
|
48
|
+
eye_dist = float(abs(l_eye[0] - r_eye[0]))
|
|
49
|
+
if eye_dist < 1e-6:
|
|
50
|
+
return 0.0
|
|
51
|
+
centre_x = (float(r_eye[0] + l_eye[0]) / 2 + float(r_mouth[0] + l_mouth[0]) / 2) / 2
|
|
52
|
+
return (float(nose[0]) - centre_x) / eye_dist
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def cosine_similarity(a: NDArray[np.float32], b: NDArray[np.float32]) -> float:
|
|
56
|
+
"""Cosine similarity of two vectors."""
|
|
57
|
+
denom = float(np.linalg.norm(a) * np.linalg.norm(b))
|
|
58
|
+
return float(np.dot(a, b) / denom) if denom else 0.0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SFaceEmbedder:
|
|
62
|
+
"""YuNet detector + SFace recognizer."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, yunet_path: str | Path | None = None, sface_path: str | Path | None = None):
|
|
65
|
+
self._yunet_path = str(yunet_path or download_model("yunet"))
|
|
66
|
+
self._sface_path = str(sface_path or download_model("sface"))
|
|
67
|
+
self._detector = cv2.FaceDetectorYN.create(self._yunet_path, "", (320, 320))
|
|
68
|
+
self._recognizer = cv2.FaceRecognizerSF.create(self._sface_path, "")
|
|
69
|
+
|
|
70
|
+
def observe(self, frame: Frame) -> FaceObservation | None:
|
|
71
|
+
h, w = frame.shape[:2]
|
|
72
|
+
self._detector.setInputSize((w, h))
|
|
73
|
+
_, detected = self._detector.detect(frame)
|
|
74
|
+
if detected is None:
|
|
75
|
+
return None
|
|
76
|
+
faces: list[NDArray[np.float32]] = list(detected)
|
|
77
|
+
good = [
|
|
78
|
+
f for f in faces if f[14] >= MIN_DETECTION_SCORE and min(f[2], f[3]) >= MIN_FACE_SIZE
|
|
79
|
+
]
|
|
80
|
+
if len(good) != 1:
|
|
81
|
+
return None
|
|
82
|
+
aligned = self._recognizer.alignCrop(frame, good[0])
|
|
83
|
+
feat = self._recognizer.feature(aligned).reshape(-1).astype(np.float32)
|
|
84
|
+
norm = float(np.linalg.norm(feat))
|
|
85
|
+
if not norm:
|
|
86
|
+
return None
|
|
87
|
+
x, y, bw, bh = (float(v) for v in good[0][:4])
|
|
88
|
+
return FaceObservation(
|
|
89
|
+
embedding=feat / norm, yaw=head_yaw(good[0]), center=(x + bw / 2, y + bh / 2)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def embed(self, frame: Frame) -> NDArray[np.float32] | None:
|
|
93
|
+
"""Embedding only (no yaw)."""
|
|
94
|
+
obs = self.observe(frame)
|
|
95
|
+
return obs.embedding if obs else None
|
cheta/face/matcher.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Match a face embedding against enrolled galleries by cosine similarity."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from numpy.typing import NDArray
|
|
7
|
+
|
|
8
|
+
DEFAULT_THRESHOLD = 0.4 # SFace's documented cosine threshold is 0.363; slightly stricter here
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def match(
|
|
12
|
+
query: NDArray[np.float32],
|
|
13
|
+
gallery: dict[str, list[NDArray[np.float32]]],
|
|
14
|
+
threshold: float = DEFAULT_THRESHOLD,
|
|
15
|
+
) -> tuple[str, float] | None:
|
|
16
|
+
"""Return ``(person_id, similarity)`` of the best match at or above ``threshold``, else None.
|
|
17
|
+
|
|
18
|
+
A person's score is the highest cosine similarity across their stored embeddings.
|
|
19
|
+
"""
|
|
20
|
+
q = query / (np.linalg.norm(query) or 1.0)
|
|
21
|
+
best_id: str | None = None
|
|
22
|
+
best = -1.0
|
|
23
|
+
for person_id, embeddings in gallery.items():
|
|
24
|
+
for emb in embeddings:
|
|
25
|
+
score = float(np.dot(q, emb / (np.linalg.norm(emb) or 1.0)))
|
|
26
|
+
if score > best:
|
|
27
|
+
best_id, best = person_id, score
|
|
28
|
+
if best_id is None or best < threshold:
|
|
29
|
+
return None
|
|
30
|
+
return best_id, best
|
cheta/fusion.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Combine face and voice evidence into one identity decision.
|
|
2
|
+
|
|
3
|
+
Each modality's cosine similarity is first normalised so its own threshold maps to 0 and a
|
|
4
|
+
perfect match to 1. Then:
|
|
5
|
+
|
|
6
|
+
* face and voice agree on the same person: confidence = 1 - (1 - face) * (1 - voice), so two
|
|
7
|
+
independent signals are stronger than either alone;
|
|
8
|
+
* only one modality is available: its normalised score times ``single_factor`` (default
|
|
9
|
+
``SINGLE_MODALITY_FACTOR``, a lower confidence since there is nothing to corroborate it;
|
|
10
|
+
pass 1.0 for no penalty);
|
|
11
|
+
* they name different people: no decision (None), because a conflict is safer than a guess.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
SINGLE_MODALITY_FACTOR = 0.8
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class Evidence:
|
|
23
|
+
"""One modality's best match: who, the raw cosine similarity, and that modality's threshold."""
|
|
24
|
+
|
|
25
|
+
person_id: str
|
|
26
|
+
score: float
|
|
27
|
+
threshold: float
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def normalised(self) -> float:
|
|
31
|
+
"""Score rescaled so the threshold is 0 and a perfect match is 1."""
|
|
32
|
+
if self.threshold >= 1.0:
|
|
33
|
+
return 0.0
|
|
34
|
+
return min(1.0, max(0.0, (self.score - self.threshold) / (1.0 - self.threshold)))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def fuse(
|
|
38
|
+
face: Evidence | None,
|
|
39
|
+
voice: Evidence | None,
|
|
40
|
+
single_factor: float = SINGLE_MODALITY_FACTOR,
|
|
41
|
+
) -> tuple[str, float] | None:
|
|
42
|
+
"""Return ``(person_id, confidence)``, or None if there is no evidence or it conflicts."""
|
|
43
|
+
if not 0.0 < single_factor <= 1.0:
|
|
44
|
+
raise ValueError("single_factor must be in (0, 1]")
|
|
45
|
+
if face is None and voice is None:
|
|
46
|
+
return None
|
|
47
|
+
if face is not None and voice is not None:
|
|
48
|
+
if face.person_id != voice.person_id:
|
|
49
|
+
return None
|
|
50
|
+
confidence = 1.0 - (1.0 - face.normalised) * (1.0 - voice.normalised)
|
|
51
|
+
return face.person_id, confidence
|
|
52
|
+
only = face if face is not None else voice
|
|
53
|
+
assert only is not None
|
|
54
|
+
return only.person_id, only.normalised * single_factor
|