cheta 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cheta/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """cheta: on-device face and voice recognition, memory and forgetting for Reachy Mini apps."""
2
+
3
+ from cheta.identity import Identity
4
+ from cheta.memory import Memory
5
+ from cheta.person import Person
6
+
7
+ __version__ = "0.1.0"
8
+
9
+ __all__ = ["Identity", "Memory", "Person", "__version__"]
cheta/calibrate.py ADDED
@@ -0,0 +1,134 @@
1
+ """Pick a voice threshold from your own recordings.
2
+
3
+ Speaker-embedding scores depend on the microphone, the room and the people, so no single default
4
+ fits everyone. Record a few clips of each person, then measure how high *genuine* scores (a clip
5
+ against the same person) and *impostor* scores (a clip against everyone else) actually are.
6
+ ``recommend_threshold`` picks the lowest threshold that keeps false accepts under your target.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ from collections.abc import Mapping, Sequence
13
+ from dataclasses import dataclass
14
+ from pathlib import Path
15
+
16
+ import numpy as np
17
+ from numpy.typing import NDArray
18
+
19
+ from cheta.robot import resample
20
+ from cheta.speech import read_wav
21
+ from cheta.voice import (
22
+ SAMPLE_RATE,
23
+ Audio,
24
+ VoiceActivityDetector,
25
+ VoiceEmbedder,
26
+ split_windows,
27
+ )
28
+
29
+ MIN_SPEECH_S = 1.5
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class Recommendation:
34
+ """A suggested threshold and what it would have done on the recorded data."""
35
+
36
+ threshold: float
37
+ false_accept_rate: float # impostor probes scoring at or above the threshold
38
+ false_reject_rate: float # genuine probes scoring below it
39
+ n_genuine: int
40
+ n_impostor: int
41
+
42
+
43
+ def recommend_threshold(
44
+ genuine: Sequence[float], impostor: Sequence[float], max_far: float = 0.01
45
+ ) -> Recommendation:
46
+ """Lowest threshold whose false-accept rate on ``impostor`` is at most ``max_far``.
47
+
48
+ Lower thresholds reject fewer genuine users, so the lowest one that meets the false-accept
49
+ target is the best. With few impostor scores the rate is coarse; collect more clips.
50
+ """
51
+ if not genuine or not impostor:
52
+ raise ValueError("need at least one genuine and one impostor score")
53
+ if not 0.0 <= max_far < 1.0:
54
+ raise ValueError("max_far must be in [0, 1)")
55
+ gen = np.sort(np.asarray(genuine, dtype=np.float64))
56
+ imp = np.sort(np.asarray(impostor, dtype=np.float64))
57
+ # Candidates: just above each impostor score, so "score >= threshold" excludes it.
58
+ candidates = np.unique(imp + 1e-6)
59
+ chosen = next(float(t) for t in candidates if float(np.mean(imp >= t)) <= max_far)
60
+ chosen = math.ceil(chosen * 1000) / 1000 # round *up* so the false-accept target still holds
61
+ return Recommendation(
62
+ threshold=chosen,
63
+ false_accept_rate=float(np.mean(imp >= chosen)),
64
+ false_reject_rate=float(np.mean(gen < chosen)),
65
+ n_genuine=len(gen),
66
+ n_impostor=len(imp),
67
+ )
68
+
69
+
70
+ def _n(x: NDArray[np.float32]) -> NDArray[np.float32]:
71
+ return x / (np.linalg.norm(x) or 1.0)
72
+
73
+
74
+ def score_voices(
75
+ enrol: Mapping[str, Audio],
76
+ probes: Mapping[str, Sequence[Audio]],
77
+ vad: VoiceActivityDetector,
78
+ embedder: VoiceEmbedder,
79
+ ) -> tuple[list[float], list[float]]:
80
+ """Return ``(genuine, impostor)`` scores the way ``Identity`` would produce them.
81
+
82
+ Each person is enrolled from their ``enrol`` clip (windows plus the whole utterance, as in
83
+ ``enroll()``). Every probe clip is split into windows; a window's score for a person is its
84
+ best cosine similarity to that person's samples. The genuine score is the window against its
85
+ own person, the impostor score is the *best* other person (the open-set worst case).
86
+ """
87
+ if len(enrol) < 2:
88
+ raise ValueError("need at least two people to measure impostor scores")
89
+ templates: dict[str, list[NDArray[np.float32]]] = {}
90
+ for name, clip in enrol.items():
91
+ speech = vad.speech_only(clip)
92
+ if len(speech) < MIN_SPEECH_S * SAMPLE_RATE:
93
+ raise ValueError(f"not enough speech in the enrolment clip for {name!r}")
94
+ embs = [_n(embedder.embed(w)) for w in split_windows(speech)]
95
+ if len(embs) > 1:
96
+ embs.append(_n(embedder.embed(speech)))
97
+ templates[name] = embs
98
+ genuine: list[float] = []
99
+ impostor: list[float] = []
100
+ for name, clips in probes.items():
101
+ if name not in templates:
102
+ raise ValueError(f"probe clips for {name!r} have no enrolment clip")
103
+ for clip in clips:
104
+ speech = vad.speech_only(clip)
105
+ if len(speech) < MIN_SPEECH_S * SAMPLE_RATE:
106
+ continue
107
+ for window in split_windows(speech):
108
+ q = _n(embedder.embed(window))
109
+ scores = {n: max(float(q @ t) for t in ts) for n, ts in templates.items()}
110
+ genuine.append(scores[name])
111
+ impostor.append(max(s for n, s in scores.items() if n != name))
112
+ if not genuine:
113
+ raise ValueError("no probe clip had enough speech")
114
+ return genuine, impostor
115
+
116
+
117
+ def load_voice_dir(path: Path) -> tuple[dict[str, Audio], dict[str, list[Audio]]]:
118
+ """Read ``<name>_<n>.wav`` files: a person's lowest ``n`` enrols them, the rest are probes."""
119
+ clips: dict[str, list[tuple[int, Audio]]] = {}
120
+ for wav in sorted(Path(path).glob("*.wav")):
121
+ name, _, number = wav.stem.rpartition("_")
122
+ if not name or not number.isdigit():
123
+ raise ValueError(f"{wav.name}: expected a file name like <name>_<number>.wav")
124
+ samples, rate = read_wav(wav)
125
+ clips.setdefault(name, []).append((int(number), resample(samples, rate, SAMPLE_RATE)))
126
+ enrol: dict[str, Audio] = {}
127
+ probes: dict[str, list[Audio]] = {}
128
+ for name, items in clips.items():
129
+ items.sort(key=lambda item: item[0])
130
+ if len(items) < 2:
131
+ raise ValueError(f"{name!r} needs at least two clips: one to enrol, one to test")
132
+ enrol[name] = items[0][1]
133
+ probes[name] = [audio for _, audio in items[1:]]
134
+ return enrol, probes
cheta/cli.py ADDED
@@ -0,0 +1,262 @@
1
+ """Command line interface: ``cheta enroll|list|forget|demo``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import time
7
+ from collections.abc import Sequence
8
+ from pathlib import Path
9
+
10
+ from cheta.calibrate import load_voice_dir, recommend_threshold, score_voices
11
+ from cheta.face.matcher import DEFAULT_THRESHOLD
12
+ from cheta.identity import DEFAULT_VOICE_THRESHOLD, Identity
13
+ from cheta.person import Person
14
+ from cheta.voice import EcapaEmbedder, SileroVAD
15
+
16
+
17
+ def _identity(args: argparse.Namespace) -> Identity:
18
+ return Identity(
19
+ app=args.app,
20
+ camera=args.camera,
21
+ data_dir=args.data_dir,
22
+ threshold=getattr(args, "threshold", DEFAULT_THRESHOLD),
23
+ liveness=not getattr(args, "no_liveness", False),
24
+ listen_voice=getattr(args, "voice", False),
25
+ on_prompt=print,
26
+ )
27
+
28
+
29
+ def cmd_enroll(args: argparse.Namespace) -> int:
30
+ """Enroll a person from the camera."""
31
+ if not args.yes:
32
+ answer = input(
33
+ f"Store a face template for {args.name!r} on this device? "
34
+ "Raw images are never saved. [y/N] "
35
+ )
36
+ if answer.strip().lower() not in ("y", "yes"):
37
+ print("Cancelled: consent not given.")
38
+ return 1
39
+ idt = _identity(args)
40
+ try:
41
+ person = idt.enroll(args.name, consent=True, on_prompt=print, voice=not args.no_voice)
42
+ finally:
43
+ idt.close()
44
+ what = "face samples" if args.no_voice else "face and voice samples"
45
+ print(f"Enrolled {person.name} ({person.id}) with encrypted {what}.")
46
+ return 0
47
+
48
+
49
+ def cmd_list(args: argparse.Namespace) -> int:
50
+ """List enrolled people."""
51
+ idt = _identity(args)
52
+ try:
53
+ people = idt.store.list_people()
54
+ finally:
55
+ idt.close()
56
+ for p in people:
57
+ print(f"{p.id} {p.name}")
58
+ if not people:
59
+ print("No one enrolled.")
60
+ return 0
61
+
62
+
63
+ def cmd_demo(args: argparse.Namespace) -> int:
64
+ """Live recognition: print who is in view and arrival/left events until Ctrl-C."""
65
+ idt = _identity(args)
66
+ idt.on_person_arrived(lambda p: print(f"[arrived] {p.name} ({p.confidence:.2f})"))
67
+ idt.on_person_left(lambda p: print(f"[left] {p.name}"))
68
+ idt.start()
69
+ print("Watching the camera. Press Ctrl-C to stop.")
70
+ last = ""
71
+ try:
72
+ while True:
73
+ who = idt.who_is_here()
74
+ line = f"{who.name} ({who.confidence:.2f})" if who else "nobody known"
75
+ if line != last:
76
+ print(f"here: {line}")
77
+ last = line
78
+ time.sleep(0.2)
79
+ except KeyboardInterrupt:
80
+ pass
81
+ finally:
82
+ idt.close()
83
+ return 0
84
+
85
+
86
+ def cmd_forget(args: argparse.Namespace) -> int:
87
+ """Hard-delete a person: face samples and memories."""
88
+ idt = _identity(args)
89
+ try:
90
+ matches = idt.store.find_by_name(args.name) if args.name else []
91
+ if args.id:
92
+ person = idt.store.get_person(args.id)
93
+ matches = [person] if person else []
94
+ if not matches:
95
+ print("No such person.")
96
+ return 1
97
+ if len(matches) > 1:
98
+ print("Several people match; pass --id:")
99
+ for p in matches:
100
+ print(f" {p.id} {p.name}")
101
+ return 1
102
+ person = matches[0]
103
+ if not args.yes:
104
+ answer = input(f"Permanently delete {person.name} and all their data? [y/N] ")
105
+ if answer.strip().lower() not in ("y", "yes"):
106
+ print("Cancelled.")
107
+ return 1
108
+ idt.forget(person)
109
+ finally:
110
+ idt.close()
111
+ print(f"Forgot {person.name}.")
112
+ return 0
113
+
114
+
115
+ def cmd_listen(args: argparse.Namespace) -> int:
116
+ """Record a few seconds from the microphone and say who is speaking."""
117
+ idt = Identity(
118
+ app=args.app,
119
+ camera=args.camera,
120
+ data_dir=args.data_dir,
121
+ voice_threshold=args.threshold,
122
+ )
123
+ try:
124
+ print(f"Speak for {args.seconds:.0f} seconds...")
125
+ who = idt.identify_voice(seconds=args.seconds)
126
+ finally:
127
+ idt.close()
128
+ print(f"speaker: {who.name} ({who.confidence:.2f})" if who else "speaker: unknown")
129
+ return 0
130
+
131
+
132
+ def cmd_calibrate(args: argparse.Namespace) -> int:
133
+ """Recommend a voice threshold from recordings in a folder (nothing is enrolled or stored)."""
134
+ try:
135
+ enrol, probes = load_voice_dir(Path(args.dir))
136
+ genuine, impostor = score_voices(enrol, probes, SileroVAD(), EcapaEmbedder())
137
+ rec = recommend_threshold(genuine, impostor, max_far=args.max_far)
138
+ except ValueError as exc:
139
+ print(f"Cannot calibrate: {exc}")
140
+ return 1
141
+ print(f"{len(enrol)} people, {rec.n_genuine} genuine and {rec.n_impostor} impostor scores.")
142
+ print(f"genuine : median {sorted(genuine)[len(genuine) // 2]:.2f}, lowest {min(genuine):.2f}")
143
+ print(
144
+ f"impostor : median {sorted(impostor)[len(impostor) // 2]:.2f}, highest {max(impostor):.2f}"
145
+ )
146
+ print(
147
+ f"Recommended voice_threshold = {rec.threshold:.3f} "
148
+ f"(false accepts {rec.false_accept_rate:.1%}, false rejects {rec.false_reject_rate:.1%})"
149
+ )
150
+ if rec.n_impostor < 100:
151
+ print("Note: few impostor scores; the false-accept estimate is coarse. Use more clips.")
152
+ return 0
153
+
154
+
155
+ def _one_person(idt: Identity, name: str) -> Person | None:
156
+ matches = idt.store.find_by_name(name)
157
+ if not matches:
158
+ print(f"No one called {name!r} is enrolled.")
159
+ return None
160
+ if len(matches) > 1:
161
+ print(f"Several people are called {name!r}; use `cheta list` to tell them apart.")
162
+ return None
163
+ return matches[0]
164
+
165
+
166
+ def cmd_remember(args: argparse.Namespace) -> int:
167
+ """Store a fact about an enrolled person."""
168
+ idt = _identity(args)
169
+ try:
170
+ person = _one_person(idt, args.name)
171
+ if person is None:
172
+ return 1
173
+ idt.remember(person, args.text, expires_in_days=args.expires_days)
174
+ finally:
175
+ idt.close()
176
+ print(f"Remembered for {person.name}: {args.text}")
177
+ return 0
178
+
179
+
180
+ def cmd_recall(args: argparse.Namespace) -> int:
181
+ """Look up what is remembered about a person."""
182
+ idt = _identity(args)
183
+ try:
184
+ person = _one_person(idt, args.name)
185
+ if person is None:
186
+ return 1
187
+ memories = idt.recall(person, args.query, k=args.k)
188
+ finally:
189
+ idt.close()
190
+ for m in memories:
191
+ print(f"{m.score:.2f} {m.text}")
192
+ if not memories:
193
+ print("Nothing remembered.")
194
+ return 0
195
+
196
+
197
+ def build_parser() -> argparse.ArgumentParser:
198
+ """Build the argument parser."""
199
+ parser = argparse.ArgumentParser(prog="cheta", description=__doc__)
200
+ parser.add_argument("--app", default="cheta", help="app namespace (default: cheta)")
201
+ parser.add_argument("--data-dir", default="~/.cheta")
202
+ parser.add_argument("--camera", choices=["webcam", "reachy"], default="webcam")
203
+ sub = parser.add_subparsers(dest="command")
204
+
205
+ enroll = sub.add_parser("enroll", help="enroll a person from the camera")
206
+ enroll.add_argument("--name", required=True)
207
+ enroll.add_argument("--no-voice", action="store_true", help="enroll by face only")
208
+ enroll.add_argument("--yes", action="store_true", help="confirm consent without prompting")
209
+ enroll.set_defaults(func=cmd_enroll)
210
+
211
+ demo = sub.add_parser("demo", help="live recognition from the camera")
212
+ demo.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
213
+ demo.add_argument("--voice", action="store_true", help="also listen to the mic and fuse voice")
214
+ demo.add_argument(
215
+ "--no-liveness", action="store_true", help="skip the head-turn check (photos will pass!)"
216
+ )
217
+ demo.set_defaults(func=cmd_demo)
218
+
219
+ forget = sub.add_parser("forget", help="delete a person and all their data")
220
+ who = forget.add_mutually_exclusive_group(required=True)
221
+ who.add_argument("--name")
222
+ who.add_argument("--id")
223
+ forget.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
224
+ forget.set_defaults(func=cmd_forget)
225
+
226
+ listen = sub.add_parser("listen", help="identify the speaker from the microphone")
227
+ listen.add_argument("--seconds", type=float, default=4.0)
228
+ listen.add_argument("--threshold", type=float, default=DEFAULT_VOICE_THRESHOLD)
229
+ listen.set_defaults(func=cmd_listen)
230
+
231
+ calibrate = sub.add_parser(
232
+ "calibrate", help="recommend a voice threshold from recordings (<name>_<n>.wav)"
233
+ )
234
+ calibrate.add_argument("dir", help="folder of <name>_1.wav (enrol), <name>_2.wav ... (tests)")
235
+ calibrate.add_argument("--max-far", type=float, default=0.01, help="false-accept target")
236
+ calibrate.set_defaults(func=cmd_calibrate)
237
+
238
+ remember = sub.add_parser("remember", help="store a fact about an enrolled person")
239
+ remember.add_argument("--name", required=True)
240
+ remember.add_argument("text")
241
+ remember.add_argument("--expires-days", type=int, default=None)
242
+ remember.set_defaults(func=cmd_remember)
243
+
244
+ recall = sub.add_parser("recall", help="look up facts about an enrolled person")
245
+ recall.add_argument("--name", required=True)
246
+ recall.add_argument("query")
247
+ recall.add_argument("-k", type=int, default=5)
248
+ recall.set_defaults(func=cmd_recall)
249
+
250
+ lst = sub.add_parser("list", help="list enrolled people")
251
+ lst.set_defaults(func=cmd_list)
252
+ return parser
253
+
254
+
255
+ def main(argv: Sequence[str] | None = None) -> int:
256
+ """CLI entry point."""
257
+ parser = build_parser()
258
+ args = parser.parse_args(argv)
259
+ if not hasattr(args, "func"):
260
+ parser.print_help()
261
+ return 0
262
+ return int(args.func(args))
cheta/face/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """Face detection and embedding."""
2
+
3
+ from cheta.face.embedder import (
4
+ FaceEmbedder,
5
+ FaceObservation,
6
+ SFaceEmbedder,
7
+ cosine_similarity,
8
+ head_yaw,
9
+ )
10
+
11
+ __all__ = ["FaceEmbedder", "FaceObservation", "SFaceEmbedder", "cosine_similarity", "head_yaw"]
cheta/face/capture.py ADDED
@@ -0,0 +1,33 @@
1
+ """Frame sources. Frames are held in memory only long enough to embed them."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Protocol, cast
6
+
7
+ import cv2
8
+
9
+ from cheta.face.embedder import Frame
10
+
11
+
12
+ class FrameSource(Protocol):
13
+ """Anything that yields BGR frames."""
14
+
15
+ def read(self) -> Frame | None: ...
16
+
17
+ def close(self) -> None: ...
18
+
19
+
20
+ class WebcamSource:
21
+ """Laptop webcam via OpenCV."""
22
+
23
+ def __init__(self, index: int = 0) -> None:
24
+ self._cap = cv2.VideoCapture(index)
25
+ if not self._cap.isOpened():
26
+ raise RuntimeError(f"could not open webcam {index}")
27
+
28
+ def read(self) -> Frame | None:
29
+ ok, frame = self._cap.read()
30
+ return cast(Frame, frame) if ok else None
31
+
32
+ def close(self) -> None:
33
+ self._cap.release()
cheta/face/embedder.py ADDED
@@ -0,0 +1,95 @@
1
+ """Face detection (YuNet) and embedding (SFace) via OpenCV. Raw frames are never stored."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Protocol
8
+
9
+ import cv2
10
+ import numpy as np
11
+ from numpy.typing import NDArray
12
+
13
+ from cheta.models import download_model
14
+
15
+ Frame = NDArray[np.uint8]
16
+ MIN_DETECTION_SCORE = 0.85
17
+ MIN_FACE_SIZE = 80 # pixels, shortest side of the detected box
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class FaceObservation:
22
+ """What one frame shows: the face embedding and a head-turn (yaw) proxy.
23
+
24
+ ``yaw`` is the nose's horizontal offset from the face centre in units of eye distance:
25
+ about 0 facing the camera, growing in magnitude as the head turns.
26
+ """
27
+
28
+ embedding: NDArray[np.float32]
29
+ yaw: float
30
+ center: tuple[float, float] | None = None # face centre in frame pixels (u right, v down)
31
+
32
+
33
+ class FaceEmbedder(Protocol):
34
+ """Looks at one frame containing a single face."""
35
+
36
+ def observe(self, frame: Frame) -> FaceObservation | None:
37
+ """Return the observation, or None when there is not exactly one usable face."""
38
+ ...
39
+
40
+
41
+ def head_yaw(face: NDArray[np.float32]) -> float:
42
+ """Yaw proxy from a YuNet detection row (landmarks: eyes, nose, mouth corners).
43
+
44
+ A real head turning moves the nose (which sits in front of the eye plane) sideways relative
45
+ to the eyes and mouth. A flat photo, even tilted or moved, keeps this ratio almost constant.
46
+ """
47
+ r_eye, l_eye, nose, r_mouth, l_mouth = (face[4 + 2 * i : 6 + 2 * i] for i in range(5))
48
+ eye_dist = float(abs(l_eye[0] - r_eye[0]))
49
+ if eye_dist < 1e-6:
50
+ return 0.0
51
+ centre_x = (float(r_eye[0] + l_eye[0]) / 2 + float(r_mouth[0] + l_mouth[0]) / 2) / 2
52
+ return (float(nose[0]) - centre_x) / eye_dist
53
+
54
+
55
+ def cosine_similarity(a: NDArray[np.float32], b: NDArray[np.float32]) -> float:
56
+ """Cosine similarity of two vectors."""
57
+ denom = float(np.linalg.norm(a) * np.linalg.norm(b))
58
+ return float(np.dot(a, b) / denom) if denom else 0.0
59
+
60
+
61
+ class SFaceEmbedder:
62
+ """YuNet detector + SFace recognizer."""
63
+
64
+ def __init__(self, yunet_path: str | Path | None = None, sface_path: str | Path | None = None):
65
+ self._yunet_path = str(yunet_path or download_model("yunet"))
66
+ self._sface_path = str(sface_path or download_model("sface"))
67
+ self._detector = cv2.FaceDetectorYN.create(self._yunet_path, "", (320, 320))
68
+ self._recognizer = cv2.FaceRecognizerSF.create(self._sface_path, "")
69
+
70
+ def observe(self, frame: Frame) -> FaceObservation | None:
71
+ h, w = frame.shape[:2]
72
+ self._detector.setInputSize((w, h))
73
+ _, detected = self._detector.detect(frame)
74
+ if detected is None:
75
+ return None
76
+ faces: list[NDArray[np.float32]] = list(detected)
77
+ good = [
78
+ f for f in faces if f[14] >= MIN_DETECTION_SCORE and min(f[2], f[3]) >= MIN_FACE_SIZE
79
+ ]
80
+ if len(good) != 1:
81
+ return None
82
+ aligned = self._recognizer.alignCrop(frame, good[0])
83
+ feat = self._recognizer.feature(aligned).reshape(-1).astype(np.float32)
84
+ norm = float(np.linalg.norm(feat))
85
+ if not norm:
86
+ return None
87
+ x, y, bw, bh = (float(v) for v in good[0][:4])
88
+ return FaceObservation(
89
+ embedding=feat / norm, yaw=head_yaw(good[0]), center=(x + bw / 2, y + bh / 2)
90
+ )
91
+
92
+ def embed(self, frame: Frame) -> NDArray[np.float32] | None:
93
+ """Embedding only (no yaw)."""
94
+ obs = self.observe(frame)
95
+ return obs.embedding if obs else None
cheta/face/matcher.py ADDED
@@ -0,0 +1,30 @@
1
+ """Match a face embedding against enrolled galleries by cosine similarity."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import numpy as np
6
+ from numpy.typing import NDArray
7
+
8
+ DEFAULT_THRESHOLD = 0.4 # SFace's documented cosine threshold is 0.363; slightly stricter here
9
+
10
+
11
+ def match(
12
+ query: NDArray[np.float32],
13
+ gallery: dict[str, list[NDArray[np.float32]]],
14
+ threshold: float = DEFAULT_THRESHOLD,
15
+ ) -> tuple[str, float] | None:
16
+ """Return ``(person_id, similarity)`` of the best match at or above ``threshold``, else None.
17
+
18
+ A person's score is the highest cosine similarity across their stored embeddings.
19
+ """
20
+ q = query / (np.linalg.norm(query) or 1.0)
21
+ best_id: str | None = None
22
+ best = -1.0
23
+ for person_id, embeddings in gallery.items():
24
+ for emb in embeddings:
25
+ score = float(np.dot(q, emb / (np.linalg.norm(emb) or 1.0)))
26
+ if score > best:
27
+ best_id, best = person_id, score
28
+ if best_id is None or best < threshold:
29
+ return None
30
+ return best_id, best
cheta/fusion.py ADDED
@@ -0,0 +1,54 @@
1
+ """Combine face and voice evidence into one identity decision.
2
+
3
+ Each modality's cosine similarity is first normalised so its own threshold maps to 0 and a
4
+ perfect match to 1. Then:
5
+
6
+ * face and voice agree on the same person: confidence = 1 - (1 - face) * (1 - voice), so two
7
+ independent signals are stronger than either alone;
8
+ * only one modality is available: its normalised score times ``single_factor`` (default
9
+ ``SINGLE_MODALITY_FACTOR``, a lower confidence since there is nothing to corroborate it;
10
+ pass 1.0 for no penalty);
11
+ * they name different people: no decision (None), because a conflict is safer than a guess.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass
17
+
18
+ SINGLE_MODALITY_FACTOR = 0.8
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class Evidence:
23
+ """One modality's best match: who, the raw cosine similarity, and that modality's threshold."""
24
+
25
+ person_id: str
26
+ score: float
27
+ threshold: float
28
+
29
+ @property
30
+ def normalised(self) -> float:
31
+ """Score rescaled so the threshold is 0 and a perfect match is 1."""
32
+ if self.threshold >= 1.0:
33
+ return 0.0
34
+ return min(1.0, max(0.0, (self.score - self.threshold) / (1.0 - self.threshold)))
35
+
36
+
37
+ def fuse(
38
+ face: Evidence | None,
39
+ voice: Evidence | None,
40
+ single_factor: float = SINGLE_MODALITY_FACTOR,
41
+ ) -> tuple[str, float] | None:
42
+ """Return ``(person_id, confidence)``, or None if there is no evidence or it conflicts."""
43
+ if not 0.0 < single_factor <= 1.0:
44
+ raise ValueError("single_factor must be in (0, 1]")
45
+ if face is None and voice is None:
46
+ return None
47
+ if face is not None and voice is not None:
48
+ if face.person_id != voice.person_id:
49
+ return None
50
+ confidence = 1.0 - (1.0 - face.normalised) * (1.0 - voice.normalised)
51
+ return face.person_id, confidence
52
+ only = face if face is not None else voice
53
+ assert only is not None
54
+ return only.person_id, only.normalised * single_factor