islkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- islkit/__init__.py +88 -0
- islkit/adapters.py +237 -0
- islkit/baseline.py +287 -0
- islkit/data.py +320 -0
- islkit/device.py +38 -0
- islkit/domain.py +187 -0
- islkit/features.py +323 -0
- islkit/infer.py +911 -0
- islkit/labels.py +213 -0
- islkit/metrics.py +81 -0
- islkit/model.py +623 -0
- islkit/pipeline.py +717 -0
- islkit/plotting.py +131 -0
- islkit/seeding.py +19 -0
- islkit/server.py +246 -0
- islkit/view.py +287 -0
- islkit/viz.py +435 -0
- islkit-0.1.0.dist-info/METADATA +200 -0
- islkit-0.1.0.dist-info/RECORD +21 -0
- islkit-0.1.0.dist-info/WHEEL +4 -0
- islkit-0.1.0.dist-info/licenses/LICENSE +21 -0
islkit/plotting.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Training curves, saved to disk rather than shown.
|
|
2
|
+
|
|
3
|
+
Experiments run from a terminal, so there is no window to pop up. Everything
|
|
4
|
+
lands in runs/<experiment>/ under the current working directory, or under
|
|
5
|
+
$ISLKIT_RUNS_DIR when set. Keep figures beside any write-up that argues from
|
|
6
|
+
them, so the evidence stays checkable.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import matplotlib
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
matplotlib.use("Agg") # no display; must be set before pyplot is imported
|
|
17
|
+
import matplotlib.pyplot as plt # noqa: E402
|
|
18
|
+
|
|
19
|
+
RUNS_DIR = Path(os.environ.get("ISLKIT_RUNS_DIR", "runs")).resolve()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def plot_curves(
|
|
23
|
+
series: Mapping[str, Sequence[float]],
|
|
24
|
+
title: str,
|
|
25
|
+
experiment: str,
|
|
26
|
+
filename: str = "curves.png",
|
|
27
|
+
xlabel: str = "epoch",
|
|
28
|
+
) -> Path:
|
|
29
|
+
"""Plot one line per entry in `series` and save it under runs/<experiment>/.
|
|
30
|
+
|
|
31
|
+
Returns the path written, so callers can print it.
|
|
32
|
+
"""
|
|
33
|
+
out_dir = RUNS_DIR / experiment
|
|
34
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
out_path = out_dir / filename
|
|
36
|
+
|
|
37
|
+
fig, ax = plt.subplots(figsize=(7, 4))
|
|
38
|
+
for label, values in series.items():
|
|
39
|
+
ax.plot(range(1, len(values) + 1), values, label=label, marker="o", markersize=3)
|
|
40
|
+
ax.set_title(title)
|
|
41
|
+
ax.set_xlabel(xlabel)
|
|
42
|
+
ax.grid(alpha=0.3)
|
|
43
|
+
ax.legend()
|
|
44
|
+
fig.tight_layout()
|
|
45
|
+
fig.savefig(out_path, dpi=120)
|
|
46
|
+
plt.close(fig)
|
|
47
|
+
return out_path
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def plot_confusion(
|
|
51
|
+
cm,
|
|
52
|
+
title: str,
|
|
53
|
+
experiment: str,
|
|
54
|
+
filename: str = "confusion.png",
|
|
55
|
+
names: Sequence[str] | None = None,
|
|
56
|
+
max_ticks: int = 45,
|
|
57
|
+
) -> Path:
|
|
58
|
+
"""Row-normalised confusion matrix as an image, saved under runs/<experiment>/.
|
|
59
|
+
|
|
60
|
+
Rows are the true class, columns the prediction, so a bright diagonal is a
|
|
61
|
+
working classifier and a bright column is a class the model falls back on.
|
|
62
|
+
Rows are normalised by support because INCLUDE is 6.25x imbalanced and raw
|
|
63
|
+
counts would just redraw the class distribution.
|
|
64
|
+
|
|
65
|
+
Tick labels are drawn only when there are few enough classes to read them;
|
|
66
|
+
past that the image shows structure and `top_confusions` carries the detail.
|
|
67
|
+
"""
|
|
68
|
+
cm = np.asarray(cm, dtype=np.float64)
|
|
69
|
+
out_dir = RUNS_DIR / experiment
|
|
70
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
out_path = out_dir / filename
|
|
72
|
+
|
|
73
|
+
support = cm.sum(axis=1, keepdims=True)
|
|
74
|
+
norm = np.divide(cm, support, out=np.zeros_like(cm), where=support > 0)
|
|
75
|
+
|
|
76
|
+
n = len(cm)
|
|
77
|
+
size = max(6.0, min(16.0, n * 0.22))
|
|
78
|
+
fig, ax = plt.subplots(figsize=(size, size * 0.92))
|
|
79
|
+
im = ax.imshow(norm, cmap="magma", vmin=0.0, vmax=1.0, interpolation="nearest")
|
|
80
|
+
|
|
81
|
+
ax.set_title(title)
|
|
82
|
+
ax.set_xlabel("predicted")
|
|
83
|
+
ax.set_ylabel("true")
|
|
84
|
+
if names is not None and n <= max_ticks:
|
|
85
|
+
ax.set_xticks(range(n), names, rotation=90, fontsize=7)
|
|
86
|
+
ax.set_yticks(range(n), names, fontsize=7)
|
|
87
|
+
else:
|
|
88
|
+
ax.set_xticks([0, n - 1])
|
|
89
|
+
ax.set_yticks([0, n - 1])
|
|
90
|
+
fig.colorbar(im, ax=ax, fraction=0.046, label="share of true class")
|
|
91
|
+
fig.tight_layout()
|
|
92
|
+
fig.savefig(out_path, dpi=140)
|
|
93
|
+
plt.close(fig)
|
|
94
|
+
return out_path
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def plot_f1_distribution(
|
|
98
|
+
scores: Sequence[float],
|
|
99
|
+
title: str,
|
|
100
|
+
experiment: str,
|
|
101
|
+
filename: str = "f1_distribution.png",
|
|
102
|
+
) -> Path:
|
|
103
|
+
"""Sorted per-class F1 plus its histogram, saved under runs/<experiment>/.
|
|
104
|
+
|
|
105
|
+
Report per-class F1: an aggregate number hides a class that never works. The
|
|
106
|
+
left panel is where you see how many classes are at zero.
|
|
107
|
+
"""
|
|
108
|
+
scores = np.sort(np.asarray(scores, dtype=np.float64))
|
|
109
|
+
out_dir = RUNS_DIR / experiment
|
|
110
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
out_path = out_dir / filename
|
|
112
|
+
|
|
113
|
+
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4))
|
|
114
|
+
ax1.plot(range(1, len(scores) + 1), scores, lw=1.6)
|
|
115
|
+
ax1.axhline(float(np.median(scores)), ls="--", c="crimson", lw=1, label="median")
|
|
116
|
+
ax1.set_xlabel("class, worst to best")
|
|
117
|
+
ax1.set_ylabel("F1")
|
|
118
|
+
ax1.set_ylim(-0.02, 1.02)
|
|
119
|
+
ax1.grid(alpha=0.3)
|
|
120
|
+
ax1.legend()
|
|
121
|
+
|
|
122
|
+
ax2.hist(scores, bins=20, range=(0.0, 1.0), color="#4c72b0")
|
|
123
|
+
ax2.set_xlabel("F1")
|
|
124
|
+
ax2.set_ylabel("classes")
|
|
125
|
+
ax2.grid(alpha=0.3)
|
|
126
|
+
|
|
127
|
+
fig.suptitle(title)
|
|
128
|
+
fig.tight_layout()
|
|
129
|
+
fig.savefig(out_path, dpi=130)
|
|
130
|
+
plt.close(fig)
|
|
131
|
+
return out_path
|
islkit/seeding.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Reproducibility.
|
|
2
|
+
|
|
3
|
+
Seeding all three RNGs matters more than it looks: numpy generates the data,
|
|
4
|
+
torch initialises the weights and shuffles the batches, and `random` sneaks in
|
|
5
|
+
through library code. Miss one and a "deterministic" run still drifts.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import random
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import torch
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def set_seed(seed: int = 0) -> None:
|
|
15
|
+
random.seed(seed)
|
|
16
|
+
np.random.seed(seed)
|
|
17
|
+
torch.manual_seed(seed)
|
|
18
|
+
if torch.cuda.is_available():
|
|
19
|
+
torch.cuda.manual_seed_all(seed)
|
islkit/server.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""The recognition service's HTTP layer. No camera, no torch, no MediaPipe.
|
|
2
|
+
|
|
3
|
+
Camera in, glosses out — this half is only the "out". It knows nothing about
|
|
4
|
+
what a screen is, how the consuming service numbers its messages, or what
|
|
5
|
+
language the device speaks; those belong to the orchestrator. If a change to
|
|
6
|
+
the display protocol would require touching this file, the boundary has been
|
|
7
|
+
drawn wrong.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import queue
|
|
15
|
+
import threading
|
|
16
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
17
|
+
|
|
18
|
+
MAX_SUBSCRIBERS = 4
|
|
19
|
+
QUEUE_MAXSIZE = 256
|
|
20
|
+
HEARTBEAT_SECONDS = 2.0
|
|
21
|
+
|
|
22
|
+
# Which events are worth replaying to a subscriber that connects late, and under
|
|
23
|
+
# which slot. Two slots, not one: framing and take state move independently, so a
|
|
24
|
+
# reconnecting orchestrator needs both to draw a correct screen. `classifying`
|
|
25
|
+
# lasts ~30 ms and `recognised`/`unclear`/`fault` are one-shot answers — retaining
|
|
26
|
+
# any of them would strand a reconnect in a state that has long since passed.
|
|
27
|
+
_RETAIN_SLOT = {"armed": "take", "recording": "take", "tracking": "tracking"}
|
|
28
|
+
_REPLAY_ORDER = ("tracking", "take") # framing first, then what we are doing
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class EventHub:
|
|
32
|
+
"""Fan-out from one capture thread to N SSE subscribers.
|
|
33
|
+
|
|
34
|
+
`publish` never blocks and never raises. That is the whole point: the capture
|
|
35
|
+
thread is holding the camera and MediaPipe, and a slow or dead orchestrator
|
|
36
|
+
must not be able to stall it. A full queue loses its oldest event and says so
|
|
37
|
+
in `dropped_events`, which `/health` reports.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS, maxsize: int = QUEUE_MAXSIZE):
|
|
41
|
+
self._max_subscribers = max_subscribers
|
|
42
|
+
self._maxsize = maxsize
|
|
43
|
+
self._lock = threading.Lock()
|
|
44
|
+
self._queues: list[queue.Queue] = []
|
|
45
|
+
self._retained: dict[str, dict] = {}
|
|
46
|
+
self.dropped_events = 0
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def subscribers(self) -> int:
|
|
50
|
+
with self._lock:
|
|
51
|
+
return len(self._queues)
|
|
52
|
+
|
|
53
|
+
def subscribe(self) -> queue.Queue | None:
|
|
54
|
+
"""A queue pre-seeded with the retained state, or None if we are full."""
|
|
55
|
+
with self._lock:
|
|
56
|
+
if len(self._queues) >= self._max_subscribers:
|
|
57
|
+
return None
|
|
58
|
+
q: queue.Queue = queue.Queue(maxsize=self._maxsize)
|
|
59
|
+
for slot in _REPLAY_ORDER:
|
|
60
|
+
event = self._retained.get(slot)
|
|
61
|
+
if event is not None:
|
|
62
|
+
q.put_nowait(event)
|
|
63
|
+
self._queues.append(q)
|
|
64
|
+
return q
|
|
65
|
+
|
|
66
|
+
def unsubscribe(self, q: queue.Queue) -> None:
|
|
67
|
+
"""Idempotent: a handler thread can unwind through more than one path."""
|
|
68
|
+
with self._lock:
|
|
69
|
+
if q in self._queues:
|
|
70
|
+
self._queues.remove(q)
|
|
71
|
+
|
|
72
|
+
def clear_take(self) -> None:
|
|
73
|
+
"""Drop the retained take-state event, leaving `tracking` retained.
|
|
74
|
+
|
|
75
|
+
Pausing destroys any in-flight take (`RecognitionPipeline.set_capture`
|
|
76
|
+
disarms it), but without this the hub would go on replaying the last
|
|
77
|
+
`armed`/`recording` to every new subscriber, when while paused only
|
|
78
|
+
`tracking` should be sent. Idempotent: popping an already-empty slot is a
|
|
79
|
+
no-op, so it is safe to call on every pause, not just a transition.
|
|
80
|
+
"""
|
|
81
|
+
with self._lock:
|
|
82
|
+
self._retained.pop("take", None)
|
|
83
|
+
|
|
84
|
+
def publish(self, event: dict) -> None:
|
|
85
|
+
with self._lock:
|
|
86
|
+
slot = _RETAIN_SLOT.get(event.get("e"))
|
|
87
|
+
if slot is not None:
|
|
88
|
+
self._retained[slot] = event
|
|
89
|
+
for q in self._queues:
|
|
90
|
+
try:
|
|
91
|
+
q.put_nowait(event)
|
|
92
|
+
except queue.Full:
|
|
93
|
+
try:
|
|
94
|
+
q.get_nowait() # drop the oldest, keep the newest
|
|
95
|
+
except queue.Empty: # pragma: no cover - drained concurrently
|
|
96
|
+
pass
|
|
97
|
+
self.dropped_events += 1
|
|
98
|
+
try:
|
|
99
|
+
q.put_nowait(event)
|
|
100
|
+
except queue.Full: # pragma: no cover - drained concurrently
|
|
101
|
+
pass
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
DEFAULT_PORT = 9978 # the orchestrator takes 9977
|
|
105
|
+
MAX_BODY_BYTES = 4096
|
|
106
|
+
|
|
107
|
+
log = logging.getLogger("islkit.server")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
111
|
+
protocol_version = "HTTP/1.1"
|
|
112
|
+
server_version = "islkit-recognition/1"
|
|
113
|
+
|
|
114
|
+
# -- helpers -----------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
def _json(self, status: int, payload: dict) -> None:
|
|
117
|
+
body = json.dumps(payload).encode()
|
|
118
|
+
self.send_response(status)
|
|
119
|
+
self.send_header("Content-Type", "application/json")
|
|
120
|
+
self.send_header("Content-Length", str(len(body)))
|
|
121
|
+
self.send_header("Connection", "close")
|
|
122
|
+
self.end_headers()
|
|
123
|
+
self.wfile.write(body)
|
|
124
|
+
|
|
125
|
+
def log_message(self, fmt: str, *args) -> None:
|
|
126
|
+
log.debug("%s - %s", self.address_string(), fmt % args)
|
|
127
|
+
|
|
128
|
+
# -- routes ------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's spelling
|
|
131
|
+
path = self.path.split("?", 1)[0]
|
|
132
|
+
if path == "/results":
|
|
133
|
+
self._results()
|
|
134
|
+
elif path == "/health":
|
|
135
|
+
try:
|
|
136
|
+
payload = self.server.pipeline.health()
|
|
137
|
+
except Exception:
|
|
138
|
+
# /health is what a watchdog polls; a bare disconnect on our
|
|
139
|
+
# side is the least useful failure it could see.
|
|
140
|
+
log.exception("pipeline.health() failed")
|
|
141
|
+
self._json(500, {"error": "health check failed"})
|
|
142
|
+
return
|
|
143
|
+
# The hub's own counters, not the pipeline's: /health reports both,
|
|
144
|
+
# and without this merge they are only ever exercised by the hub's
|
|
145
|
+
# own tests — slow-subscriber observability never actually reaches
|
|
146
|
+
# the wire.
|
|
147
|
+
payload = {
|
|
148
|
+
**payload,
|
|
149
|
+
"subscribers": self.server.hub.subscribers,
|
|
150
|
+
"dropped_events": self.server.hub.dropped_events,
|
|
151
|
+
}
|
|
152
|
+
self._json(200, payload)
|
|
153
|
+
else:
|
|
154
|
+
self._json(404, {"error": "not found"})
|
|
155
|
+
|
|
156
|
+
def do_POST(self) -> None: # noqa: N802
|
|
157
|
+
if self.path.split("?", 1)[0] != "/capture":
|
|
158
|
+
self._json(404, {"error": "not found"})
|
|
159
|
+
return
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
length = int(self.headers.get("Content-Length") or 0)
|
|
163
|
+
except ValueError:
|
|
164
|
+
self._json(400, {"error": "invalid Content-Length"})
|
|
165
|
+
return
|
|
166
|
+
if length > MAX_BODY_BYTES:
|
|
167
|
+
self._json(413, {"error": f"body must be under {MAX_BODY_BYTES} bytes"})
|
|
168
|
+
return
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
payload = json.loads(self.rfile.read(length) or b"")
|
|
172
|
+
active = payload["active"]
|
|
173
|
+
except (ValueError, KeyError, TypeError):
|
|
174
|
+
self._json(400, {"error": 'body must be {"active": true|false}'})
|
|
175
|
+
return
|
|
176
|
+
if not isinstance(active, bool):
|
|
177
|
+
self._json(400, {"error": '"active" must be true or false'})
|
|
178
|
+
return
|
|
179
|
+
|
|
180
|
+
self._json(200, {"capture": self.server.pipeline.set_capture(active)})
|
|
181
|
+
|
|
182
|
+
def _results(self) -> None:
|
|
183
|
+
"""SSE. No Content-Length: the body is delimited by the close, which is
|
|
184
|
+
what lets this stream indefinitely."""
|
|
185
|
+
q = self.server.hub.subscribe()
|
|
186
|
+
if q is None:
|
|
187
|
+
self._json(503, {"error": "too many subscribers"})
|
|
188
|
+
return
|
|
189
|
+
|
|
190
|
+
# Everything from here on can raise if the peer is already gone (a
|
|
191
|
+
# reset between accept and first byte hits end_headers itself) — the
|
|
192
|
+
# try must cover the header writes too, or a vanished client leaks
|
|
193
|
+
# its hub slot forever.
|
|
194
|
+
try:
|
|
195
|
+
self.send_response(200)
|
|
196
|
+
self.send_header("Content-Type", "text/event-stream")
|
|
197
|
+
self.send_header("Cache-Control", "no-cache")
|
|
198
|
+
self.send_header("Connection", "close")
|
|
199
|
+
self.end_headers()
|
|
200
|
+
|
|
201
|
+
while True:
|
|
202
|
+
try:
|
|
203
|
+
event = q.get(timeout=self.server.heartbeat)
|
|
204
|
+
payload = f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
|
|
205
|
+
except queue.Empty:
|
|
206
|
+
# A comment, not an event: the orchestrator's 5 s offline
|
|
207
|
+
# timer must reset on any received line, and an idle signer
|
|
208
|
+
# produces no events for minutes.
|
|
209
|
+
payload = ": ping\n\n"
|
|
210
|
+
self.wfile.write(payload.encode())
|
|
211
|
+
self.wfile.flush()
|
|
212
|
+
except (BrokenPipeError, ConnectionResetError, OSError):
|
|
213
|
+
pass # the orchestrator went away; that is its right
|
|
214
|
+
finally:
|
|
215
|
+
self.server.hub.unsubscribe(q)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class RecognitionServer(ThreadingHTTPServer):
|
|
219
|
+
daemon_threads = True
|
|
220
|
+
allow_reuse_address = True
|
|
221
|
+
|
|
222
|
+
def __init__(self, address, pipeline, hub, heartbeat):
|
|
223
|
+
super().__init__(address, _Handler)
|
|
224
|
+
self.pipeline = pipeline
|
|
225
|
+
self.hub = hub
|
|
226
|
+
self.heartbeat = heartbeat
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def make_server(
|
|
230
|
+
pipeline,
|
|
231
|
+
hub: EventHub,
|
|
232
|
+
host: str = "127.0.0.1",
|
|
233
|
+
port: int = DEFAULT_PORT,
|
|
234
|
+
heartbeat: float = HEARTBEAT_SECONDS,
|
|
235
|
+
) -> RecognitionServer:
|
|
236
|
+
"""Bound to loopback by default. Nothing outside this machine — and nothing
|
|
237
|
+
inside an App Lab container — talks to this service directly."""
|
|
238
|
+
return RecognitionServer((host, port), pipeline, hub, heartbeat)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def serve(pipeline, hub: EventHub, host: str = "127.0.0.1", port: int = DEFAULT_PORT) -> None:
|
|
242
|
+
server = make_server(pipeline, hub, host, port)
|
|
243
|
+
try:
|
|
244
|
+
server.serve_forever()
|
|
245
|
+
finally:
|
|
246
|
+
server.server_close()
|
islkit/view.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""The debug view: annotated camera frames over MJPEG, for aiming the camera.
|
|
2
|
+
|
|
3
|
+
This exists so a tester can see what the model sees — where the landmarks
|
|
4
|
+
actually land, whether they are clipped, whether MediaPipe has lost the hands —
|
|
5
|
+
without which "it did not recognise my sign" is unfalsifiable.
|
|
6
|
+
|
|
7
|
+
Two properties matter more than anything else here:
|
|
8
|
+
|
|
9
|
+
**It costs nothing when nobody is watching.** `ViewSink.watching` is False until
|
|
10
|
+
a browser connects, and the pipeline checks it before it copies, draws or
|
|
11
|
+
encodes a single frame. A service running unwatched pays one boolean read per
|
|
12
|
+
frame.
|
|
13
|
+
|
|
14
|
+
**It reuses the landmarks recognition already computed.** `HolisticExtractor`
|
|
15
|
+
returns the frame and the raw MediaPipe results together, and already has a
|
|
16
|
+
`draw()` built for this. Nothing here runs inference; a second Holistic pass on
|
|
17
|
+
a board managing ~2 fps would halve the thing it is meant to debug.
|
|
18
|
+
|
|
19
|
+
The frame published here is **unmirrored**, exactly as the model sees it. The
|
|
20
|
+
page flips it with a CSS transform so positioning yourself feels natural, which
|
|
21
|
+
costs nothing and lets each viewer toggle independently — mirroring server-side
|
|
22
|
+
would mean a second encode per orientation for no gain.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import logging
|
|
28
|
+
import threading
|
|
29
|
+
import time
|
|
30
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
31
|
+
|
|
32
|
+
log = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
DEFAULT_VIEW_PORT = 9979
|
|
35
|
+
JPEG_QUALITY = 70
|
|
36
|
+
IDLE_SLEEP_S = 0.02
|
|
37
|
+
CLIENT_TIMEOUT_S = 30.0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ViewSink:
|
|
41
|
+
"""One latest-frame slot, plus the viewer count that gates producing it.
|
|
42
|
+
|
|
43
|
+
The pipeline publishes; HTTP handlers consume. Deliberately one slot and not
|
|
44
|
+
a queue: a viewer that falls behind should see the newest frame, never a
|
|
45
|
+
backlog of stale ones, and a debug view must never apply back-pressure to
|
|
46
|
+
the capture loop.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self):
|
|
50
|
+
self._lock = threading.Lock()
|
|
51
|
+
self._new = threading.Condition(self._lock)
|
|
52
|
+
self._jpeg: bytes | None = None
|
|
53
|
+
self._seq = 0
|
|
54
|
+
self._viewers = 0
|
|
55
|
+
self._published = 0
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def watching(self) -> bool:
|
|
59
|
+
"""True when at least one browser is attached. The pipeline's gate."""
|
|
60
|
+
with self._lock:
|
|
61
|
+
return self._viewers > 0
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def viewers(self) -> int:
|
|
65
|
+
with self._lock:
|
|
66
|
+
return self._viewers
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def published(self) -> int:
|
|
70
|
+
with self._lock:
|
|
71
|
+
return self._published
|
|
72
|
+
|
|
73
|
+
def attach(self) -> None:
|
|
74
|
+
with self._lock:
|
|
75
|
+
self._viewers += 1
|
|
76
|
+
|
|
77
|
+
def detach(self) -> None:
|
|
78
|
+
with self._lock:
|
|
79
|
+
self._viewers = max(0, self._viewers - 1)
|
|
80
|
+
|
|
81
|
+
def publish(self, jpeg: bytes) -> None:
|
|
82
|
+
"""Replace the current frame and wake every waiting handler."""
|
|
83
|
+
with self._new:
|
|
84
|
+
self._jpeg = jpeg
|
|
85
|
+
self._seq += 1
|
|
86
|
+
self._published += 1
|
|
87
|
+
self._new.notify_all()
|
|
88
|
+
|
|
89
|
+
def wait_for(self, last_seq: int, timeout: float = 1.0):
|
|
90
|
+
"""Block until a frame newer than `last_seq`, or time out.
|
|
91
|
+
|
|
92
|
+
Returns (seq, jpeg), or (last_seq, None) if nothing new arrived. The
|
|
93
|
+
timeout is what lets a handler notice a client that has gone away.
|
|
94
|
+
"""
|
|
95
|
+
with self._new:
|
|
96
|
+
if self._seq == last_seq or self._jpeg is None:
|
|
97
|
+
self._new.wait(timeout)
|
|
98
|
+
if self._jpeg is None or self._seq == last_seq:
|
|
99
|
+
return last_seq, None
|
|
100
|
+
return self._seq, self._jpeg
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def annotate(cv2, extractor, frame, results, overlay: dict) -> bytes | None:
|
|
104
|
+
"""Draw landmarks and status onto a copy of `frame`, return JPEG bytes.
|
|
105
|
+
|
|
106
|
+
The copy is not optional. `extractor.draw()` writes into the array it is
|
|
107
|
+
given, and the array it would be given here comes straight from the camera
|
|
108
|
+
source — which may reuse its buffer. Drawing into it would corrupt whatever
|
|
109
|
+
the capture loop does with that frame next.
|
|
110
|
+
|
|
111
|
+
Returns None if the encode fails, which the caller treats as "skip this
|
|
112
|
+
frame": a debug view is never worth an exception in the capture loop.
|
|
113
|
+
"""
|
|
114
|
+
try:
|
|
115
|
+
canvas = frame.copy()
|
|
116
|
+
extractor.draw(canvas, results)
|
|
117
|
+
_draw_overlay(cv2, canvas, overlay)
|
|
118
|
+
ok, buf = cv2.imencode(".jpg", canvas, [int(cv2.IMWRITE_JPEG_QUALITY), JPEG_QUALITY])
|
|
119
|
+
if not ok:
|
|
120
|
+
return None
|
|
121
|
+
return buf.tobytes()
|
|
122
|
+
except Exception: # noqa: BLE001 - the view must not break capture
|
|
123
|
+
log.exception("view frame failed; skipping")
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _draw_overlay(cv2, canvas, overlay: dict) -> None:
|
|
128
|
+
"""Status text, so the picture answers questions on its own.
|
|
129
|
+
|
|
130
|
+
Tracking status is the one that matters: `clipped` and `absent` explain a
|
|
131
|
+
take that never fired, and they are invisible in the landmarks alone.
|
|
132
|
+
"""
|
|
133
|
+
lines = [
|
|
134
|
+
f"tracking: {overlay.get('tracking')}",
|
|
135
|
+
f"take: {overlay.get('take_state')} capture: {'on' if overlay.get('capture') else 'off'}",
|
|
136
|
+
f"fps: {overlay.get('fps'):.2f}" if overlay.get("fps") is not None else "fps: -",
|
|
137
|
+
]
|
|
138
|
+
tracking = overlay.get("tracking")
|
|
139
|
+
# BGR. Green when the framing is usable, amber when it is not.
|
|
140
|
+
colour = (120, 220, 120) if tracking == "ok" else (60, 190, 250)
|
|
141
|
+
|
|
142
|
+
y = 22
|
|
143
|
+
for text in lines:
|
|
144
|
+
cv2.putText(canvas, text, (8, y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 3)
|
|
145
|
+
cv2.putText(canvas, text, (8, y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, colour, 1)
|
|
146
|
+
y += 20
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
PAGE = """<!doctype html>
|
|
150
|
+
<html lang="en">
|
|
151
|
+
<head>
|
|
152
|
+
<meta charset="utf-8">
|
|
153
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
154
|
+
<title>islkit recognition view</title>
|
|
155
|
+
<style>
|
|
156
|
+
:root { color-scheme: dark; }
|
|
157
|
+
body { margin: 0; background: #10141a; color: #e6e8ea;
|
|
158
|
+
font: 14px/1.5 ui-sans-serif, system-ui, sans-serif; }
|
|
159
|
+
.wrap { max-width: 720px; margin: 0 auto; padding: 20px 16px 40px; }
|
|
160
|
+
h1 { font-size: 17px; margin: 0 0 4px; font-weight: 600; }
|
|
161
|
+
p.sub { margin: 0 0 16px; color: #98a2ad; font-size: 13px; }
|
|
162
|
+
.shot { background: #000; border: 1px solid #263040; border-radius: 6px;
|
|
163
|
+
overflow: hidden; line-height: 0; }
|
|
164
|
+
img { width: 100%; height: auto; display: block; }
|
|
165
|
+
img.mirror { transform: scaleX(-1); }
|
|
166
|
+
.bar { display: flex; gap: 10px; align-items: center; margin-top: 12px; flex-wrap: wrap; }
|
|
167
|
+
button { background: #1b2431; color: #e6e8ea; border: 1px solid #2f3a4a;
|
|
168
|
+
border-radius: 5px; padding: 7px 12px; font: inherit; cursor: pointer; }
|
|
169
|
+
button:hover { background: #223044; }
|
|
170
|
+
.hint { color: #98a2ad; font-size: 12px; }
|
|
171
|
+
code { background: #1b2431; padding: 1px 5px; border-radius: 3px; font-size: 12px; }
|
|
172
|
+
</style>
|
|
173
|
+
</head>
|
|
174
|
+
<body>
|
|
175
|
+
<div class="wrap">
|
|
176
|
+
<h1>Recognition view</h1>
|
|
177
|
+
<p class="sub">The frames the classifier is actually seeing, with the landmarks it extracted.
|
|
178
|
+
Streaming costs nothing while this page is closed.</p>
|
|
179
|
+
|
|
180
|
+
<div class="shot">
|
|
181
|
+
<img id="feed" class="mirror" src="/view.mjpg" alt="annotated camera stream">
|
|
182
|
+
</div>
|
|
183
|
+
|
|
184
|
+
<div class="bar">
|
|
185
|
+
<button id="flip">Show the model’s view</button>
|
|
186
|
+
<span class="hint" id="hint">Mirrored, so moving left moves the image left.</span>
|
|
187
|
+
</div>
|
|
188
|
+
|
|
189
|
+
<p class="hint" style="margin-top:16px">
|
|
190
|
+
Landmarks are drawn <em>before</em> any mirroring, so they line up in both views.
|
|
191
|
+
The model always receives the unmirrored frame — that is pinned at
|
|
192
|
+
<code>100.0%</code> unmirrored against <code>28.3%</code> mirrored.
|
|
193
|
+
</p>
|
|
194
|
+
</div>
|
|
195
|
+
<script>
|
|
196
|
+
const feed = document.getElementById('feed');
|
|
197
|
+
const flip = document.getElementById('flip');
|
|
198
|
+
const hint = document.getElementById('hint');
|
|
199
|
+
flip.addEventListener('click', () => {
|
|
200
|
+
const mirrored = feed.classList.toggle('mirror');
|
|
201
|
+
flip.textContent = mirrored ? "Show the model\\u2019s view" : 'Show the mirrored view';
|
|
202
|
+
hint.textContent = mirrored
|
|
203
|
+
? 'Mirrored, so moving left moves the image left.'
|
|
204
|
+
: 'Unmirrored \\u2014 exactly the geometry the classifier receives.';
|
|
205
|
+
});
|
|
206
|
+
</script>
|
|
207
|
+
</body>
|
|
208
|
+
</html>
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
213
|
+
protocol_version = "HTTP/1.0" # one response per connection; MJPEG closes it
|
|
214
|
+
|
|
215
|
+
def log_message(self, fmt, *args):
|
|
216
|
+
log.debug("view %s - %s", self.address_string(), fmt % args)
|
|
217
|
+
|
|
218
|
+
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's spelling
|
|
219
|
+
path = self.path.split("?", 1)[0]
|
|
220
|
+
if path in ("/", "/view", "/index.html"):
|
|
221
|
+
self._page()
|
|
222
|
+
elif path == "/view.mjpg":
|
|
223
|
+
self._stream()
|
|
224
|
+
else:
|
|
225
|
+
self.send_error(404)
|
|
226
|
+
|
|
227
|
+
def _page(self) -> None:
|
|
228
|
+
body = PAGE.encode("utf-8")
|
|
229
|
+
self.send_response(200)
|
|
230
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
231
|
+
self.send_header("Content-Length", str(len(body)))
|
|
232
|
+
self.end_headers()
|
|
233
|
+
self.wfile.write(body)
|
|
234
|
+
|
|
235
|
+
def _stream(self) -> None:
|
|
236
|
+
sink = self.server.sink
|
|
237
|
+
sink.attach()
|
|
238
|
+
try:
|
|
239
|
+
self.send_response(200)
|
|
240
|
+
self.send_header("Age", "0")
|
|
241
|
+
self.send_header("Cache-Control", "no-cache, private")
|
|
242
|
+
self.send_header("Pragma", "no-cache")
|
|
243
|
+
self.send_header("Content-Type", "multipart/x-mixed-replace; boundary=frame")
|
|
244
|
+
self.end_headers()
|
|
245
|
+
|
|
246
|
+
last = 0
|
|
247
|
+
deadline = time.monotonic() + CLIENT_TIMEOUT_S
|
|
248
|
+
while True:
|
|
249
|
+
seq, jpeg = sink.wait_for(last)
|
|
250
|
+
if jpeg is None:
|
|
251
|
+
# Nothing new. The capture loop may simply be slow; only give
|
|
252
|
+
# up once nothing has arrived for a long time.
|
|
253
|
+
if time.monotonic() > deadline:
|
|
254
|
+
return
|
|
255
|
+
continue
|
|
256
|
+
last = seq
|
|
257
|
+
deadline = time.monotonic() + CLIENT_TIMEOUT_S
|
|
258
|
+
self.wfile.write(b"--frame\r\n")
|
|
259
|
+
self.wfile.write(b"Content-Type: image/jpeg\r\n")
|
|
260
|
+
self.wfile.write(f"Content-Length: {len(jpeg)}\r\n\r\n".encode())
|
|
261
|
+
self.wfile.write(jpeg)
|
|
262
|
+
self.wfile.write(b"\r\n")
|
|
263
|
+
self.wfile.flush()
|
|
264
|
+
except (BrokenPipeError, ConnectionResetError, OSError):
|
|
265
|
+
log.debug("view client went away")
|
|
266
|
+
finally:
|
|
267
|
+
sink.detach()
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class ViewServer(ThreadingHTTPServer):
|
|
271
|
+
daemon_threads = True
|
|
272
|
+
allow_reuse_address = True
|
|
273
|
+
|
|
274
|
+
def __init__(self, address, sink: ViewSink):
|
|
275
|
+
self.sink = sink
|
|
276
|
+
super().__init__(address, _Handler)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def make_view_server(sink: ViewSink, host: str = "0.0.0.0", port: int = DEFAULT_VIEW_PORT):
|
|
280
|
+
"""A debug-view server on its own port.
|
|
281
|
+
|
|
282
|
+
Separate from the recognition API on purpose. That API stays on loopback —
|
|
283
|
+
nothing outside the board should be able to start the camera — while this
|
|
284
|
+
binds where a laptop can reach it. The trade is deliberate and documented:
|
|
285
|
+
anyone who can reach this port can watch the camera.
|
|
286
|
+
"""
|
|
287
|
+
return ViewServer((host, port), sink)
|