hyperstudy-perception 0.1.0__py3-none-any.whl

perception/_pipeline/nodes/vision/face-track/frames.py

@@ -0,0 +1,88 @@
+"""Face crops, cut with PyAV: its wheels carry FFmpeg's libraries, so the node needs no ffmpeg on the machine (an app
+started from the Finder has only /usr/bin:/bin:/usr/sbin:/sbin on its PATH). Each crop is what the ffmpeg command it
+replaced wrote (`ffmpeg -ss {t:.3f} -i video -frames:v 1 -vf crop=...,scale=...`), at the frame the detector saw."""
+
+from fractions import Fraction
+from pathlib import Path
+
+import av
+import numpy as np
+
+from perception_node import NodeError
+
+
+def frame_at(container, t: float) -> av.VideoFrame:
+    """The frame nearest `t` seconds after the video stream's first frame. Callers' times are frame times counted from
+    that frame (face-track's frame index / fps, decode's shots from scenedetect's frame counts), rounded to the
+    millisecond: on either side of the frame's own time at 29.97 fps, so neither the first frame at or after t nor the
+    one shown at t is always it. (`ffmpeg -ss` counted from the container's start instead: when the video starts after
+    the audio, it cut crops and keyframes early.) The decode node has the same function."""
+    v = container.streams.video[0]
+    target = Fraction(v.start_time or 0) * v.time_base + Fraction(round(t * 1000), 1000)
+    container.seek(int(target / v.time_base), stream=v, backward=True)
+    before = None
+    for frame in container.decode(v):
+        if frame.pts is None:
+            continue
+        at = frame.pts * v.time_base
+        if at >= target:
+            return before if before is not None and target - before.pts * v.time_base < at - target else frame
+        before = frame
+    if before is not None and target - before.pts * v.time_base < 1 / v.average_rate:   # within the last frame
+        return before
+    raise NodeError("input", f"the video has no frame at {t:.3f} s")
+
+
+def upright(frame: av.VideoFrame) -> np.ndarray:
+    """The frame as RGB, turned the way players show it. Phone recordings store the picture sideways with a rotation
+    to apply: ffmpeg applied it, and so does torchcodec, which reads the frames face-track's detector sees; PyAV only
+    reports it. The decode node has the same function."""
+    return np.ascontiguousarray(np.rot90(frame.to_ndarray(format="rgb24"), round((frame.rotation or 0) / 90) % 4))
+
+
+def crop_box(box, pad: float, frame_w: int, frame_h: int) -> tuple[int, int, int, int]:
+    """x, y, width, height of the padded box within the frame; even, as ffmpeg's crop makes them on 4:2:0 video."""
+    x, y, w, h = box
+    x0, y0 = max(0, x - pad * w), max(0, y - pad * h)
+    x1, y1 = min(frame_w, x + (1 + pad) * w), min(frame_h, y + (1 + pad) * h)
+    return int(x0) & ~1, int(y0) & ~1, int(x1 - x0) & ~1, int(y1 - y0) & ~1
+
+
+def even(n: Fraction) -> int:
+    """ffmpeg's -2 in a scale: the nearest even number."""
+    return max(2, round(n / 2) * 2)
+
+
+def crop(video: Path, t: float, box, pad: float, size) -> av.VideoFrame:
+    """The padded box at `t` (in the upright frame, as the detector saw it, and within it), scaled (bicubic, as
+    ffmpeg's scale) to size(width, height) -> (width, height)."""
+    with av.open(str(video)) as container:
+        rgb = upright(frame_at(container, t))
+    x, y, w, h = crop_box(box, pad, rgb.shape[1], rgb.shape[0])
+    out_w, out_h = size(w, h)
+    return av.VideoFrame.from_ndarray(np.ascontiguousarray(rgb[y:y + h, x:x + w]), format="rgb24").reformat(
+        width=out_w, height=out_h, interpolation="BICUBIC")
+
+
+def longest_side(px: int):
+    """scale='if(gt(iw,ih),px,-2)':'if(gt(iw,ih),-2,px)'"""
+    return lambda w, h: (px, even(Fraction(px * h, w))) if w > h else (even(Fraction(px * w, h)), px)
+
+
+def width(px: int):
+    """scale=px:-2"""
+    return lambda w, h: (px, even(Fraction(px * h, w)))
+
+
+def square(px: int):
+    """scale=px:px"""
+    return lambda w, h: (px, px)
+
+
+def save_crop(video: Path, t: float, box, pad: float, out: Path, size) -> None:
+    crop(video, t, box, pad, size).to_image().save(out, quality=90)
+
+
+def crop_pixels(video: Path, t: float, box, pad: float, px: int) -> np.ndarray:
+    """The crop as px x px x 3 RGB, for scoring."""
+    return crop(video, t, box, pad, square(px)).to_ndarray(format="rgb24")

perception/_pipeline/nodes/vision/face-track/loader.py

@@ -0,0 +1,18 @@
+"""How many processes decode frames beside the detector (py-feat's DataLoader workers).
+
+On an 8-vCPU L4 (OOO clip 2, 2026-10-24), 4 workers took detection from 99 s to 57 s; batch size barely mattered, and
+the boxes were byte-identical with and without workers. Workers must be forked: spawned ones (macOS, Windows)
+re-import node.py and would run the node again, so there the default is none.
+"""
+
+AUTO_WORKERS = 4
+
+
+def decode_workers(requested: int | None, start_method: str, cpus: int) -> int:
+    """`requested` from the config; None means automatic."""
+    if requested is None:
+        return min(AUTO_WORKERS, max(0, cpus - 1)) if start_method == "fork" else 0
+    if requested and start_method != "fork":
+        raise ValueError(f"num_workers needs forked workers; this platform starts them with {start_method!r}, "
+                         "which would run the node again in each")
+    return requested

perception/_pipeline/nodes/vision/face-track/manifest.yaml

@@ -0,0 +1,30 @@
+id: vision.face-track
+version: 1.0.0
+stage: faces
+title: Face tracks
+description: Face tracks within shots, identity clusters across the video (py-feat + ArcFace), crops
+inputs: {video: video.v1, media: media.v1, shots: bundle.shots@0.1}
+outputs: {faces: bundle.faces@0.1, crops: assets.v1}
+config:
+  fps: {type: number, exclusiveMinimum: 0, maximum: 30, default: 8, description: "frames sampled per second"}
+  cluster_threshold: {type: number, minimum: 0, maximum: 2, default: 0.65}
+  batch_size: {type: integer, minimum: 1, default: 32}
+  # Frame-decoding processes beside the detector (py-feat's DataLoader workers); null is automatic: 4 where workers
+  # can be forked (Linux), none where they would be spawned (macOS). See loader.py. Workers do not change the output.
+  num_workers: {type: [integer, "null"], minimum: 0, default: null}
+runners:
+  - {kind: native, env: envs/faces, entry: node.py, accelerator: mps, platforms: [darwin/arm64]}
+  # The faces lock resolves PyPI's CUDA 13 torch wheels on Linux (needs an NVIDIA driver >= 580).
+  - {kind: native, env: envs/faces, entry: node.py, accelerator: cuda, platforms: [linux/amd64], minimum_vram_gb: 8}
+  - {kind: native, env: envs/faces, entry: node.py, accelerator: cpu}
+  # TODO: [PERCEPTION-GRAPH] Phase 1 step 7 - CUDA image for linux/amd64 (pinned digest), validated against MPS output
+  # Plan: docs/plans/GRAPH_MVP_PLAN.md#phase-1--local-mvp
+resources: {cpu_cores: 4, memory_gb: 8, gpu: exclusive}
+# fetches the weights ahead of a run (perception setup)
+prepare: prepare.py
+models:
+  # py-feat fetches its weights from hf://py-feat/* on first use, through its own fallback logic: `plan` cannot check
+  # them repository by repository, so no hf:// source is declared here for the cache check.
+  - {id: "py-feat Detectorv2 (RetinaFace + multitask)", source: "py-feat==2.1.5 (weights from hf://py-feat)",
+     licence: "py-feat code MIT; v2 weights non-commercial", licence_mode: academic-only}
+  - {id: "ArcFace (InsightFace, via py-feat)", source: "py-feat==2.1.5", licence: research-only weights, licence_mode: academic-only}

perception/_pipeline/nodes/vision/face-track/node.py

@@ -0,0 +1,117 @@
+"""vision.face-track: who is on screen, across the whole video, as the bundle's faces layer.
+
+py-feat Detectorv2 (RetinaFace + multitask model + ArcFace) on frames sampled at `fps`; faces linked into tracks
+within each shot and clustered into people (tracking.py). Writes one crop per person (thumbnails.py: shortlisted by size and
+frontality, chosen on exposure, colour cast and sharpness) and one per track (its largest face), referenced from the
+layer as assets/crops/...
+
+Licence: py-feat's code is MIT, but the Detectorv2 multitask weights are non-commercial and InsightFace-derived
+ArcFace weights are research-only, so this layer is academic-only.
+"""
+
+import multiprocessing
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+
+import numpy as np
+
+from perception_node import Node, NodeError
+from frames import crop_pixels, longest_side, save_crop, width
+from loader import decode_workers
+from thumbnails import QUALITY_PX, choose, shortlist
+from tracking import FRONTAL_YAW, build_tracks, cluster_tracks, detection_rows, name_people, person_pairs
+
+PERSON_CROP_PAD, PERSON_CROP_PX = 0.25, 256   # padding as a fraction of the box; longest side of the saved crop
+TRACK_CROP_PAD, TRACK_CROP_WIDTH = 0.4, 320
+
+
+def run_jobs(jobs) -> list:
+    with ThreadPoolExecutor(8) as pool:
+        return list(pool.map(lambda job: job(), jobs))
+
+
+with Node() as node:
+    from feat.detector_v2 import Detectorv2
+
+    c = node.config
+    video, media = node.input("video"), node.input_json("media")
+    shots = [{"index": s["index"], "start": s["start"], "end": s["end"]} for s in node.input_json("shots")["shots"]]
+    fps = media["fps"]
+    device = {"mps": "mps", "cuda": "cuda"}.get(node.accelerator, "cpu")
+
+    try:
+        workers = decode_workers(c["num_workers"], multiprocessing.get_start_method(), os.cpu_count() or 1)
+    except ValueError as e:
+        raise NodeError("input", str(e)) from None
+
+    t0 = time.time()
+    detector = Detectorv2(device=device)
+    fex = detector.detect(str(video), data_type="video", skip_frames=max(1, round(fps / c["fps"])), batch_size=c["batch_size"],
+                          num_workers=workers, pin_memory=device == "cuda")
+    detect_s = round(time.time() - t0, 1)
+    node.model("py-feat Detectorv2 (RetinaFace + multitask)", licence_mode="academic-only", source="hf://py-feat",
+               licence="py-feat code MIT; v2 weights non-commercial")
+    node.model("ArcFace (InsightFace, via py-feat)", licence_mode="academic-only", licence="research-only weights")
+
+    rows = detection_rows(fex, fps)
+    tracks = build_tracks(rows, shots)
+    labels = cluster_tracks(tracks, c["cluster_threshold"]) if tracks else []
+    order, first_seen, names = name_people(tracks, labels)
+    emotion_cols = list(fex.emotion_columns)
+
+    assets = node.path("crops", "assets")
+    crops = assets / "crops"
+    crops.mkdir(parents=True, exist_ok=True)
+    t0 = time.time()
+    members = {label: [t for t, l in zip(tracks, labels) if l == label] for label in order}
+    candidates = {label: shortlist(members[label]) for label in order}
+    flat = [(label, r) for label in order for r in candidates[label]]
+    raw = run_jobs([partial(crop_pixels, video, r["t"], r["box"], 0.0, QUALITY_PX) for _, r in flat])   # the face box only
+    images = {label: [] for label in order}
+    for (label, _), pixels in zip(flat, raw):
+        images[label].append(pixels)
+    thumbnails_s = round(time.time() - t0, 1)
+
+    jobs, people, thumbnail_measures = [], [], {}
+    for label in order:
+        index, measures = choose(candidates[label], images[label])
+        best = candidates[label][index]
+        thumbnail_measures[names[label]] = {"t": best["t"], **measures[index]}
+        name = f"person_{names[label].replace(' ', '_')}.jpg"
+        jobs.append(partial(save_crop, video, best["t"], best["box"], PERSON_CROP_PAD, crops / name, longest_side(PERSON_CROP_PX)))
+        people.append({"id": f"person:{names[label]}", "first_seen": round(first_seen[label], 3),
+                       "seconds_on_screen": round(sum(len(t["rows"]) for t in members[label]) / c["fps"], 1),
+                       "crop": f"assets/crops/{name}", "shots": [f"shot:{s}" for s in sorted({t["shot"] for t in members[label]})]})
+
+    out_tracks = []
+    for i, (track, label) in enumerate(zip(tracks, labels)):
+        rs = track["rows"]
+        biggest = max(rs, key=lambda r: r["box"][2] * r["box"][3])
+        jobs.append(partial(save_crop, video, biggest["t"], biggest["box"], TRACK_CROP_PAD, crops / f"face_{i}.jpg", width(TRACK_CROP_WIDTH)))
+        mean_emotions = {col: float(np.mean([r["emotions"][col] for r in rs])) for col in emotion_cols}
+        out_tracks.append({
+            "id": f"face:{i}", "person": f"person:{names[label]}" if label != -1 else None, "shot": f"shot:{track['shot']}",
+            "start": rs[0]["t"], "end": rs[-1]["t"], "linked_body": None, "crop": f"assets/crops/face_{i}.jpg",
+            "samples": [{"t": r["t"], "box": [round(v, 1) for v in r["box"]], "mouth": round(r["mouth"], 3),
+                         "gaze": [round(r["gaze_yaw"], 3), round(r["gaze_pitch"], 3)]} for r in rs],
+            "summary": {"head_yaw": round(float(np.mean([r["yaw"] for r in rs])), 3),
+                        "gaze_yaw": round(float(np.mean([r["gaze_yaw"] for r in rs])), 3),
+                        "gaze_pitch": round(float(np.mean([r["gaze_pitch"] for r in rs])), 3),
+                        "dominant_emotion": max(mean_emotions, key=mean_emotions.get),
+                        "emotions": {k: round(v, 3) for k, v in mean_emotions.items()},
+                        "mean_box": [round(float(np.mean([r["box"][k] for r in rs])), 1) for k in range(4)]},
+        })
+    t0 = time.time()
+    run_jobs(jobs)
+    crops_s = round(time.time() - t0, 1)
+
+    pairs = [{**p, "a": f"person:{p['a']}", "b": f"person:{p['b']}"} for p in person_pairs(tracks, labels, names)]
+    node.write_json("faces", "faces.json", {
+        "layer": "faces", "schema_version": "0.1.0",
+        "producer": node.producer(timings={"detect_s": detect_s, "thumbnails_s": thumbnails_s, "crops_s": crops_s}, device=device),
+        "sample_fps": c["fps"], "angles": "radians; yaw + = subject's right (image left)",
+        "people": people, "tracks": out_tracks, "person_pairs": pairs, "detail": None})
+    node.usage(detections=len(rows), tracks=len(tracks), people=len(people), frontal_yaw=FRONTAL_YAW, thumbnails=thumbnail_measures,
+               decode_workers=workers)

perception/_pipeline/nodes/vision/face-track/prepare.py

@@ -0,0 +1,8 @@
+"""Fetch the face node's weights ahead of a run (`perception setup`, and the driver before a remote run): py-feat
+downloads Detectorv2's models (RetinaFace, the multitask model, ArcFace) from hf://py-feat the first time a detector is
+built, into the Hugging Face cache the node then reads. Built on the CPU: only the files matter here."""
+
+from feat.detector_v2 import Detectorv2
+
+Detectorv2(device="cpu")
+print("py-feat Detectorv2 weights are in the cache")

perception/_pipeline/nodes/vision/face-track/test_frames.py

@@ -0,0 +1,46 @@
+"""Face crops (frames.py), with nothing on PATH: the app runs nodes from the Finder, where ffmpeg is not found.
+Run: uv run --project envs/faces --with pytest pytest nodes/vision/face-track"""
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from frames import crop_box, crop_pixels, longest_side, save_crop, square, width
+
+
+@pytest.fixture(autouse=True)
+def no_tools_on_path(monkeypatch):
+    monkeypatch.setenv("PATH", "")
+
+
+def test_crop_box_is_padded_clipped_and_even():
+    assert crop_box([10, 10, 20, 20], 0.25, 64, 48) == (4, 4, 30, 30)    # 5..35 -> even
+    assert crop_box([50, 30, 20, 20], 0.5, 64, 48) == (40, 20, 24, 28)   # clipped to the frame
+
+
+def test_sizes_are_ffmpegs():
+    assert longest_side(256)(30, 20) == (256, 170)   # scale='if(gt(iw,ih),256,-2)':'if(gt(iw,ih),-2,256)'
+    assert longest_side(256)(20, 30) == (170, 256)
+    assert width(320)(30, 21) == (320, 224)          # scale=320:-2
+    assert square(64)(30, 20) == (64, 64)
+
+
+def test_crop_pixels(clip):
+    pixels = crop_pixels(clip.path, 0.4, [8, 8, 16, 16], 0.0, 64)
+    assert pixels.shape == (64, 64, 3) and pixels.dtype == np.uint8
+    assert abs(pixels.mean() - clip.nth(4)[8:24, 8:24].mean()) < 1
+
+
+def test_save_crop(clip, tmp_path):
+    save_crop(clip.path, 1.2, [8, 8, 30, 20], 0.0, tmp_path / "c.jpg", longest_side(256))
+    img = Image.open(tmp_path / "c.jpg")
+    assert img.size == (256, 170)
+    assert abs(np.asarray(img).mean() - clip.nth(12)[8:28, 8:38].mean()) < 1
+
+
+def test_a_crop_is_cut_from_the_upright_frame_the_detector_saw(rotated):
+    # torchcodec, which reads the detector's frames, turns a rotated clip upright: 32x64 here, so its boxes are in that
+    # frame. The crop of the bottom half is red, of the top half blue, and a box past the stored 64x32 still fits.
+    bottom = crop_pixels(rotated, 0.5, [0, 40, 32, 24], 0.0, 16).astype(int).mean((0, 1))
+    top = crop_pixels(rotated, 0.5, [0, 0, 32, 24], 0.0, 16).astype(int).mean((0, 1))
+    assert bottom[0] > 200 > bottom[2] and top[2] > 200 > top[0], (bottom, top)

perception/_pipeline/nodes/vision/face-track/test_loader.py

@@ -0,0 +1,16 @@
+import pytest
+
+from loader import decode_workers
+
+
+def test_automatic_means_four_where_workers_fork_and_none_where_they_spawn():
+    assert decode_workers(None, "fork", cpus=8) == 4
+    assert decode_workers(None, "fork", cpus=2) == 1      # leave a core for the detector's own process
+    assert decode_workers(None, "spawn", cpus=8) == 0     # macOS: a spawned worker would run the node again
+
+
+def test_an_explicit_count_is_kept_but_refused_where_workers_spawn():
+    assert decode_workers(0, "spawn", cpus=8) == 0
+    assert decode_workers(7, "fork", cpus=8) == 7
+    with pytest.raises(ValueError, match="needs forked workers"):
+        decode_workers(2, "spawn", cpus=8)

perception/_pipeline/nodes/vision/face-track/test_thumbnails.py

@@ -0,0 +1,66 @@
+"""Thumbnail choice (thumbnails.py). Run: uv run --project envs/faces --with pytest pytest nodes/vision/face-track"""
+
+import numpy as np
+
+from thumbnails import choose, shortlist
+
+rng = np.random.default_rng(0)
+
+
+def face(colour, size=64):
+    """A textured face-like crop: a base colour with detail, so sharpness is comparable between crops."""
+    base = np.array(colour, dtype=np.float64)
+    texture = rng.normal(0, 18, (size, size, 1))
+    return np.clip(base + texture, 0, 255).astype(np.uint8)
+
+
+def with_graphic(image):
+    """White on-screen graphics over the left third of the face (clip 2's panelist circles and name cards)."""
+    out = image.copy()
+    out[:, :22] = 255
+    out[::8, :22] = 0   # hard edges: graphics are much "sharper" than a face
+    return out
+
+
+def row(t, side, yaw=0.0, score=0.99):
+    return {"t": t, "box": [0, 0, side, side], "yaw": yaw, "score": score}
+
+
+RED_LIT = (120, 8, 10)        # a face under red stage light: dark, almost one channel
+WELL_LIT = (200, 160, 140)    # a face under white light (warm skin, but no strong cast)
+DIM = (110, 90, 80)           # a face against clip 2's black backdrop, fairly dark but neutral
+
+
+def pick_time(tracks, image_at):
+    candidates = shortlist(tracks)
+    index, _ = choose(candidates, [image_at(c["t"]) for c in candidates])
+    return candidates[index]["t"]
+
+
+def test_a_smaller_well_lit_face_beats_a_larger_red_lit_one():
+    tracks = [{"rows": [row(1.0, 200)]}, {"rows": [row(5.0, 150)]}]
+    assert pick_time(tracks, {1.0: face(RED_LIT), 5.0: face(WELL_LIT)}.get) == 5.0
+
+
+def test_graphics_over_a_face_do_not_win_on_sharpness():
+    tracks = [{"rows": [row(1.0, 214)]}, {"rows": [row(5.0, 217)]}]
+    images = {1.0: with_graphic(face(DIM)), 5.0: face(DIM)}
+    assert pick_time(tracks, images.get) == 5.0
+
+
+def test_geometry_still_decides_between_faces_of_similar_quality():
+    tracks = [{"rows": [row(1.0, 200)]}, {"rows": [row(5.0, 120)]}, {"rows": [row(9.0, 200, yaw=1.2)]}]
+    assert pick_time(tracks, lambda t: face(WELL_LIT)) == 1.0   # largest and frontal
+
+
+def test_a_person_with_one_track_still_gets_a_choice():
+    track = {"rows": [row(t, 240 if t < 3 else 200) for t in np.arange(0, 6, 0.25).round(2)]}
+    times = [c["t"] for c in shortlist([track])]
+    assert len(times) == 6 and all(abs(a - b) >= 1.0 for i, a in enumerate(times) for b in times[i + 1:])
+    assert pick_time([track], lambda t: face(RED_LIT) if t < 3 else face(WELL_LIT)) >= 3
+
+
+def test_shortlist_goes_round_the_tracks():
+    many_red = {"rows": [row(t, 200) for t in (1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0)]}
+    small_clean = {"rows": [row(20.0, 100)]}
+    assert 20.0 in [c["t"] for c in shortlist([many_red, small_clean])]

perception/_pipeline/nodes/vision/face-track/thumbnails.py

@@ -0,0 +1,68 @@
+"""Choosing each person's thumbnail from their face detections.
+
+Size and frontality alone picked a large frontal face under red stage light on OOO clip 2 (Person A). So the
+geometry score only shortlists, and each shortlisted face box (no padding: the backdrop must not count) is scored on
+its pixels. Measured on clip 2:
+  - exposure and colour cast reject the red-lit and the near-black faces;
+  - sharpness only gates blur (min(1, sharpness / SHARP_ENOUGH)): as a relative score, the edges of on-screen
+    graphics over a face made it look 40x sharper than a clean, larger view of the same person;
+  - blown-out pixels (white graphics, logos) are penalised directly.
+The shortlist goes round the person's tracks, best geometry first, taking detections at least SPACING_S apart, so a
+person with one track still gets a choice and one badly lit shot cannot fill the whole list.
+"""
+
+import numpy as np
+
+SHORTLIST = 12
+SPACING_S = 1.0          # shortlisted detections of one track are at least this far apart
+QUALITY_PX = 64          # face boxes are scored at this size (square)
+SHARP_ENOUGH = 3e-4      # Laplacian variance of a 64 px face box above which blur no longer counts against it
+CLIPPED_LUMA = 0.92      # a pixel this bright is blown out (white graphics, logos, overexposure)
+LUMA = np.array([0.2126, 0.7152, 0.0722])
+
+
+def geometry(row) -> float:
+    """Large and frontal: box area times the cosine of the head yaw (capped near profile)."""
+    return row["box"][2] * row["box"][3] * float(np.cos(min(abs(row["yaw"]), 1.4)))
+
+
+def shortlist(tracks: list[dict], k: int = SHORTLIST) -> list[dict]:
+    per_track = []
+    for t in tracks:
+        picked = []
+        for r in sorted(t["rows"], key=geometry, reverse=True):
+            if all(abs(r["t"] - p["t"]) >= SPACING_S for p in picked):
+                picked.append(r)
+        per_track.append(picked)
+    per_track.sort(key=lambda rows: geometry(rows[0]), reverse=True)
+    out, depth = [], 0
+    while len(out) < k and any(depth < len(rows) for rows in per_track):
+        out += [rows[depth] for rows in per_track if depth < len(rows)][: k - len(out)]
+        depth += 1
+    return out
+
+
+def quality(rgb: np.ndarray) -> dict:
+    """Pixel measures of one face box (H x W x 3, uint8): exposure, neutral and unclipped in 0..1; sharpness is the
+    Laplacian variance."""
+    pixels = rgb.astype(np.float64) / 255
+    luma = pixels @ LUMA
+    exposure = 1 - min(1.0, abs(float(luma.mean()) - 0.5) / 0.4)     # 1 at mid-grey; 0 below 0.1 or above 0.9
+    means = pixels.reshape(-1, 3).mean(axis=0)
+    cast = float((means.max() - means.min()) / max(means.max(), 1e-6))  # 0 neutral; 1 a single channel
+    lap = luma[:-2, 1:-1] + luma[2:, 1:-1] + luma[1:-1, :-2] + luma[1:-1, 2:] - 4 * luma[1:-1, 1:-1]
+    return {"exposure": round(exposure, 3), "neutral": round(1 - cast, 3), "sharpness": round(float(lap.var()), 6),
+            "unclipped": round(1 - min(1.0, 4 * float((luma > CLIPPED_LUMA).mean())), 3)}
+
+
+def score(row, measures: dict, top_geometry: float) -> float:
+    return (geometry(row) / top_geometry * measures["exposure"] * measures["neutral"] * measures["unclipped"]
+            * min(1.0, measures["sharpness"] / SHARP_ENOUGH) * row["score"])
+
+
+def choose(candidates: list[dict], images: list[np.ndarray]) -> tuple[int, list[dict]]:
+    """The shortlisted detection to use as the thumbnail, given each one's face box; also returns the measures.
+    Geometry counts relative to the best of this shortlist, so the choice is made among one person's faces only."""
+    measures = [quality(image) for image in images]
+    top_geometry = max(geometry(r) for r in candidates)
+    return int(np.argmax([score(r, m, top_geometry) for r, m in zip(candidates, measures)])), measures

perception/_pipeline/nodes/vision/face-track/tracking.py

@@ -0,0 +1,150 @@
+"""Face tracks and identity clusters from per-sample detections. Pure functions (numpy, scikit-learn); the node feeds
+them py-feat's rows. Ported unchanged from pipeline/bakeoff/faces/faces.py, whose comments record why each threshold
+is what it is.
+"""
+
+from itertools import combinations
+
+import numpy as np
+
+SAME_TRACK_IOU = 0.3          # box overlap between consecutive samples to continue a track
+SAME_PERSON_COSINE = 0.65     # anchor clustering; on OOO 0.75 merged panelists more fully but also merged junk (King Kong + blur + logo). Splits are fixable in review, wrong merges mislead agents
+FRONTAL_YAW = 0.5             # radians (~30 deg); py-feat head pose and gaze are in radians
+ANCHOR_FACE_PX = 60           # a track defines a person only if it has a near-frontal face at least this big
+ATTACH_COSINE = 0.6           # profile/small/blurry tracks join a person only within this distance; else Unknown
+MIN_FACE_PX = 40              # ignore faces smaller than this (too small to identify or read)
+MIN_FACE_SCORE = 0.9          # RetinaFace confidence. On NES - Biking this removed the title lettering and one of two skull shirt prints (30 -> 14 people)
+MOUTH_AUS = ("AU25", "AU26")  # lips part, jaw drop: mouth movement for linking voices to faces
+
+
+def iou(a, b):
+    ax2, ay2, bx2, by2 = a[0] + a[2], a[1] + a[3], b[0] + b[2], b[1] + b[3]
+    w, h = max(0, min(ax2, bx2) - max(a[0], b[0])), max(0, min(ay2, by2) - max(a[1], b[1]))
+    inter = w * h
+    return inter / (a[2] * a[3] + b[2] * b[3] - inter) if inter else 0.0
+
+
+def shot_of(t, shots):
+    for shot in shots:
+        if shot["start"] <= t < shot["end"]:
+            return shot["index"]
+    return shots[-1]["index"]
+
+
+def build_tracks(rows, shots):
+    """Greedy IoU linking of detections in consecutive samples, never across a cut."""
+    tracks = []
+    open_tracks = {}  # shot index -> tracks still extendable
+    for t in sorted({r["t"] for r in rows}):
+        frame_rows = [r for r in rows if r["t"] == t]
+        shot = shot_of(t, shots)
+        candidates = open_tracks.setdefault(shot, [])
+        used = set()
+        for r in sorted(frame_rows, key=lambda r: -r["box"][2] * r["box"][3]):
+            best = max((c for c in candidates if id(c) not in used),
+                       key=lambda c: iou(c["rows"][-1]["box"], r["box"]), default=None)
+            if best is not None and iou(best["rows"][-1]["box"], r["box"]) >= SAME_TRACK_IOU:
+                best["rows"].append(r)
+                used.add(id(best))
+            else:
+                track = {"shot": shot, "rows": [r]}
+                candidates.append(track)
+                tracks.append(track)
+                used.add(id(track))
+    return tracks
+
+
+def is_anchor(track):
+    """A track good enough to define a person: some near-frontal samples of a reasonably large face."""
+    return any(abs(r["yaw"]) <= FRONTAL_YAW and min(r["box"][2], r["box"][3]) >= ANCHOR_FACE_PX for r in track["rows"])
+
+
+def track_embedding(track):
+    frontal = [r["embedding"] for r in track["rows"] if abs(r["yaw"]) <= FRONTAL_YAW]
+    e = np.mean(frontal or [r["embedding"] for r in track["rows"]], axis=0)
+    return e / np.linalg.norm(e)
+
+
+def cluster_tracks(tracks, threshold=SAME_PERSON_COSINE):
+    """Two passes. People are defined only by anchor tracks (average-linkage clustering of frontal ArcFace embeddings,
+    never merging tracks on screen at the same time). Every other track (profile-only, small, blurry) joins its nearest
+    person if it is within ATTACH_COSINE and does not co-occur with that person; otherwise it is unknown (-1)."""
+    from sklearn.cluster import AgglomerativeClustering
+
+    emb = np.stack([track_embedding(t) for t in tracks])
+    times = [{r["t"] for r in t["rows"]} for t in tracks]
+    anchors = [i for i, t in enumerate(tracks) if is_anchor(t)]
+    labels = [-1] * len(tracks)
+    if not anchors:
+        return labels
+
+    dist = np.clip(1 - emb[anchors] @ emb[anchors].T, 0, 2)
+    for a in range(len(anchors)):
+        for b in range(a + 1, len(anchors)):
+            if times[anchors[a]] & times[anchors[b]]:
+                dist[a, b] = dist[b, a] = 2.0  # on screen together: different people
+    anchor_labels = ([0] if len(anchors) == 1 else
+                     AgglomerativeClustering(n_clusters=None, metric="precomputed", linkage="average",
+                                             distance_threshold=threshold).fit_predict(dist))
+    for i, label in zip(anchors, anchor_labels):
+        labels[i] = int(label)
+
+    people = sorted(set(anchor_labels))
+    centroid = {p: np.mean([emb[i] for i in anchors if labels[i] == p], axis=0) for p in people}
+    occupied = {p: set().union(*(times[i] for i in anchors if labels[i] == p)) for p in people}
+    for i, track in enumerate(tracks):
+        if labels[i] != -1:
+            continue
+        options = sorted((1 - float(emb[i] @ (centroid[p] / np.linalg.norm(centroid[p]))), p) for p in people
+                         if not (times[i] & occupied[p]))
+        if options and options[0][0] <= ATTACH_COSINE:
+            labels[i] = options[0][1]
+    return labels
+
+
+def name_people(tracks, labels):
+    """Person A, B, ... in order of first appearance; -1 stays unnamed (Unknown)."""
+    first_seen = {}
+    for track, label in zip(tracks, labels):
+        if label != -1:
+            first_seen[label] = min(first_seen.get(label, 1e9), track["rows"][0]["t"])
+    order = sorted(first_seen, key=first_seen.get)
+    return order, first_seen, {label: f"Person {chr(65 + i)}" if i < 26 else f"Person {i + 1}" for i, label in enumerate(order)}
+
+
+def person_pairs(tracks, labels, names):
+    """Cosine distance between people's ArcFace centroids, and whether they are ever on screen together (then they
+    cannot be one person): the annotator's merge suggestions (pipeline/bakeoff/faces/identity_similarity.py)."""
+    members = {}
+    for i, label in enumerate(labels):
+        if label != -1:
+            members.setdefault(names[label], []).append(i)
+    centroid = {}
+    for person, idx in members.items():
+        c = np.mean([track_embedding(tracks[i]) for i in idx], axis=0)
+        centroid[person] = c / np.linalg.norm(c)
+    times = {p: {round(r["t"], 3) for i in idx for r in tracks[i]["rows"]} for p, idx in members.items()}
+    pairs = [{"a": a, "b": b, "distance": round(float(1 - centroid[a] @ centroid[b]), 3), "co_visible": bool(times[a] & times[b])}
+             for a, b in combinations(sorted(members), 2)]
+    return sorted(pairs, key=lambda p: p["distance"])
+
+
+def detection_rows(fex, fps):
+    """py-feat rows -> detections kept for tracking (size and score filtered), with ArcFace embeddings."""
+    emotion_cols = list(fex.emotion_columns)
+    rows = []
+    for index, r in fex.iterrows():
+        if (np.isnan(r["FaceRectX"]) or min(r["FaceRectWidth"], r["FaceRectHeight"]) < MIN_FACE_PX
+                or r["FaceScore"] < MIN_FACE_SCORE):
+            continue
+        rows.append({
+            "index": index, "t": round(float(r["frame"]) / fps, 3), "frame": int(r["frame"]),
+            "box": [float(r["FaceRectX"]), float(r["FaceRectY"]), float(r["FaceRectWidth"]), float(r["FaceRectHeight"])],
+            "score": float(r["FaceScore"]),
+            "embedding": np.array([r[f"Identity_{i + 1}"] for i in range(512)], dtype=np.float32),
+            "yaw": float(r["Yaw"]), "pitch": float(r["Pitch"]),
+            "gaze_yaw": float(r["gaze_yaw"]), "gaze_pitch": float(r["gaze_pitch"]),
+            "emotions": {c: float(r[c]) for c in emotion_cols},
+            "mouth": float(np.mean([r[au] for au in MOUTH_AUS])),
+        })
+    return rows

perception/_pipeline/nodes/vlm/shot-captions/manifest.yaml

@@ -0,0 +1,29 @@
+id: vlm.shot-captions
+version: 1.0.0
+stage: extract+caption
+title: Shot captions
+service: Anthropic API
+description: Per-shot descriptions from keyframes (faces boxed and labelled), speech and the face roster; on-screen text
+inputs:
+  shots: bundle.shots@0.1
+  keyframes: assets.v1
+  transcript: bundle.transcript@0.1
+  faces: bundle.faces@0.1
+  media: media.v1
+outputs: {shots: bundle.shots@0.1, text: bundle.text@0.1}
+config:
+  model: {enum: [claude-haiku-5-5, claude-opus-5-5], default: claude-haiku-5-5, description: "bake-off choice: Haiku 5.5 with the face roster"}
+  effort: {enum: [low, medium, high], default: medium}
+runners:
+  - {kind: native, env: envs/api, entry: node.py, accelerator: cpu}
+resources: {cpu_cores: 1, memory_gb: 1, gpu: none}
+# Sent to Anthropic: keyframes with face boxes and labels, and the transcript lines of each shot.
+privacy: {external_upload: [frames, faces, transcript]}
+secrets: [ANTHROPIC_API_KEY]
+# Upper estimate per shot, per model (measured on OOO: $0.04 for 44 shots with Haiku 5.5, about $0.001 a shot; Opus 5.5
+# costs 40x more per token). A model added to the config enum needs a price here, or the manifest does not load.
+cost: {usd: {claude-haiku-5-5: 0.003, claude-opus-5-5: 0.12}, by: model, per: shot, input: shots}
+models:
+  - {id: claude-haiku-5-5, licence: Anthropic API terms, licence_mode: api-terms}
+  - {id: claude-opus-5-5, licence: Anthropic API terms, licence_mode: api-terms}
+  - {id: PySceneDetect AdaptiveDetector, licence: BSD-3-Clause, licence_mode: commercial-ok}