hyperstudy-perception 0.1.0__py3-none-any.whl

perception/_pipeline/nodes/vlm/shot-captions/node.py

@@ -0,0 +1,185 @@
+"""vlm.shot-captions: one description per shot from its keyframes and speech, with the face roster.
+
+Ported from pipeline/bakeoff/caption.py (Claude Haiku 5.5 with the face roster: 126 of 135 boxed people named on
+OOO, about $0.04 a clip). Each shot sees its three keyframes, with tracked faces boxed and labelled, the speech that
+starts inside it, the people seen so far and the previous shot's description; never anything from later shots.
+That chaining is why the calls run in order rather than as a Message Batch.
+
+Writes the shots layer with captions (it refines the decode node's shots layer) and the on-screen text layer.
+"""
+
+import base64
+import io
+import json
+import time
+
+from perception_node import Node, NodeError
+
+PRICE_PER_MTOK = {"claude-haiku-5-5": (0.10, 0.50), "claude-opus-5-5": (4.00, 20.00)}   # input, output (USD)
+
+SCHEMA = {
+    "type": "object",
+    "properties": {
+        "description": {"type": "string"},
+        "setting": {"type": "string"},
+        "people": {"type": "array", "items": {
+            "type": "object",
+            "properties": {"label": {"type": "string"}, "appearance": {"type": "string"}, "action": {"type": "string"},
+                           "speaking": {"type": "boolean"}},
+            "required": ["label", "appearance", "action", "speaking"], "additionalProperties": False}},
+        "onscreen_text": {"type": "array", "items": {"type": "string"}},
+        "mood": {"type": "string"},
+    },
+    "required": ["description", "setting", "people", "onscreen_text", "mood"],
+    "additionalProperties": False,
+}
+
+INSTRUCTIONS = """You are writing one entry of a time-coded description track for a short film. \
+An AI participant will read these entries while watching alongside people, so each entry must \
+say what someone watching would see and hear during THIS shot only.
+
+Rules:
+- Describe only what is visible in the keyframes or audible in the transcript lines for this shot. \
+Do not guess what happens later or why, and do not describe earlier shots again.
+- people: everyone visible. Reuse a label from the roster when it is clearly the same person; \
+otherwise use the next unused letter ("Person A", "Person B", ...). Never use a real name unless \
+the name is spoken in the transcript or written on screen.
+- onscreen_text: legible text in the frames, verbatim; [] if none.
+- description: 1-3 plain present-tense sentences. setting and mood: a short phrase each.
+Return only the JSON object."""
+
+FACES_INSTRUCTIONS = """Face tracking has already identified people. Labelled boxes are drawn on the \
+keyframes. Use exactly these labels for the boxed people and the roster below; do not invent new \
+labels for them. Someone visible but not boxed (back turned, too small) may be described without a \
+label, or as "Unlabelled person" in people. The boxes and labels were added for you; they are not \
+part of the film, so never describe or mention them."""
+
+BOX_COLOURS = ["#e6194b", "#3cb44b", "#4363d8", "#f58231", "#911eb4", "#42d4f4", "#f032e6", "#bfef45"]
+MAX_WORD_SECONDS = 2.0   # a longer "word" is an alignment failure that absorbed preceding silence or music
+
+
+def label(person_id: str) -> str:
+    return person_id.split(":", 1)[1]
+
+
+def shot_speech(shot, words) -> str:
+    lines = []
+    for w in words:
+        onset = w["end"] - 0.4 if w["end"] - w["start"] > MAX_WORD_SECONDS else w["start"]
+        if not shot["start"] <= onset < shot["end"]:
+            continue
+        speaker = label(w["voice"]) if w.get("voice") else "SPEAKER"
+        if lines and lines[-1]["speaker"] == speaker:
+            lines[-1]["words"].append(w["w"])
+        else:
+            lines.append({"t": onset, "speaker": speaker, "words": [w["w"]]})
+    return "\n".join(f"[{l['t']:.1f}s {l['speaker']}] {' '.join(l['words'])}" for l in lines) or "(no speech)"
+
+
+def labelled_keyframes(shot, keyframes_dir, faces, width) -> list[bytes]:
+    """Each keyframe with every tracked, assigned face within 0.2 s of it boxed and labelled."""
+    from PIL import Image, ImageDraw
+
+    people = [p["id"] for p in faces["people"]]
+    colour = {p: BOX_COLOURS[i % len(BOX_COLOURS)] for i, p in enumerate(people)}
+    out = []
+    root = keyframes_dir.resolve()
+    for kf in shot["keyframes"]:
+        path = (keyframes_dir / kf["file"].removeprefix("assets/")).resolve()   # follows links: one out is caught too
+        if not path.is_relative_to(root):
+            raise NodeError("input", f"keyframe {kf['file']} is outside the keyframes input; only those images are sent")
+        image = Image.open(path).convert("RGB")
+        scale = image.width / width
+        draw = ImageDraw.Draw(image)
+        for track in faces["tracks"]:
+            if track["person"] is None:   # unattributed faces are not labelled for the model
+                continue
+            near = [s for s in track["samples"] if abs(s["t"] - kf["t"]) <= 0.2]
+            if not near:
+                continue
+            x, y, w, h = (v * scale for v in min(near, key=lambda s: abs(s["t"] - kf["t"]))["box"])
+            name, c = label(track["person"]), colour[track["person"]]
+            draw.rectangle([x, y, x + w, y + h], outline=c, width=3)
+            top = max(0, y - 18)
+            draw.rectangle([x, top, x + 8 * len(name) + 6, top + 18], fill=c)
+            draw.text((x + 3, top + 2), name, fill="white")
+        buf = io.BytesIO()
+        image.save(buf, format="JPEG", quality=90)
+        out.append(buf.getvalue())
+    return out
+
+
+def caption(client, model, effort, prompt, images, spend):
+    import anthropic
+
+    content = [{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": base64.standard_b64encode(b).decode()}}
+               for b in images] + [{"type": "text", "text": prompt}]
+    try:
+        response = client.messages.create(model=model, max_tokens=4000, messages=[{"role": "user", "content": content}],
+                                          output_config={"effort": effort, "format": {"type": "json_schema", "schema": SCHEMA}})
+    except (anthropic.RateLimitError, anthropic.InternalServerError, anthropic.APIConnectionError) as e:
+        raise NodeError("transient", f"{type(e).__name__}: {e}") from None   # the SDK already retried twice
+    except anthropic.AuthenticationError:
+        raise NodeError("input", "the Anthropic API rejected the key (ANTHROPIC_API_KEY)") from None
+    spend(response.usage.input_tokens, response.usage.output_tokens)   # billed even if the reply is then rejected
+    if response.stop_reason == "refusal":
+        raise NodeError("input", f"refusal: {response.stop_details}")
+    text = next((b.text for b in response.content if b.type == "text"), None)
+    if text is None:
+        raise NodeError("internal", f"no text block in the response (stop_reason {response.stop_reason})")
+    return json.loads(text)
+
+
+def spent(model, tokens_in, tokens_out):
+    price = PRICE_PER_MTOK[model]
+    return round((tokens_in * price[0] + tokens_out * price[1]) / 1e6, 4)
+
+
+with Node() as node:
+    import anthropic
+
+    c = node.config
+    model, effort = c["model"], c["effort"]
+    client = anthropic.Anthropic(api_key=node.secret("ANTHROPIC_API_KEY"))
+    shots_doc = node.input_json("shots")
+    words = node.input_json("transcript")["words"]
+    faces = node.input_json("faces")
+    width = node.input_json("media")["width"]
+    keyframes_dir = node.input("keyframes")
+    first_seen = {p["id"]: p["first_seen"] for p in faces["people"]}
+
+    roster, previous, t0 = {}, None, time.time()
+    shots, texts, totals = [], [], {"in": 0, "out": 0, "calls": 0}
+
+    def spend(n_in, n_out):
+        """The running total after every billed call: if a later shot fails, the response still reports it."""
+        totals.update({"in": totals["in"] + n_in, "out": totals["out"] + n_out, "calls": totals["calls"] + 1})
+        node.usage(usd=spent(model, totals["in"], totals["out"]), tokens_in=totals["in"], tokens_out=totals["out"], calls=totals["calls"])
+
+    for shot in shots_doc["shots"]:
+        seen = [label(p) for p, t in first_seen.items() if t is not None and t < shot["end"]]   # never people who appear later
+        roster_text = "\n".join(f"- {k}: {v}" for k, v in roster.items()) or "(nobody yet)"
+        prompt = (f"{INSTRUCTIONS}\n\nShot {shot['index']} runs {shot['start']:.1f}s to {shot['end']:.1f}s. "
+                  f"Keyframes are at {', '.join(str(k['t']) + 's' for k in shot['keyframes'])}.\n\n"
+                  f"Speech during this shot:\n{shot_speech(shot, words)}\n\nPeople seen in earlier shots:\n{roster_text}\n\n"
+                  f"Previous shot: {previous or '(this is the first shot)'}\n\n{FACES_INSTRUCTIONS}\n\nFace roster so far:\n"
+                  + ("\n".join(f"- {s}" for s in seen) or "(nobody yet)"))
+        cap = caption(client, model, effort, prompt, labelled_keyframes(shot, keyframes_dir, faces, width), spend)
+        for person in cap["people"]:
+            roster.setdefault(person["label"], person["appearance"])
+        previous = cap["description"]
+        shots.append({**shot, "caption": {"description": cap["description"], "setting": cap["setting"], "mood": cap["mood"],
+                                          "people": cap["people"], "model": model}})
+        for string in cap["onscreen_text"]:   # the same string in consecutive shots is one item
+            same = next((t for t in reversed(texts) if t["text"] == string and t["end"] == shot["start"]), None)
+            if same:
+                same["end"] = shot["end"]
+            else:
+                texts.append({"id": f"text:{len(texts)}", "text": string, "start": shot["start"], "end": shot["end"], "box": None})
+
+    node.model(model, licence_mode="api-terms", licence="Anthropic API terms")
+    node.model("PySceneDetect AdaptiveDetector", licence_mode="commercial-ok", licence="BSD-3-Clause")   # the shots it refines
+    timings = {"captions_s": round(time.time() - t0, 1)}
+    node.write_json("shots", "shots.json", {**shots_doc, "producer": node.producer(timings=timings, device="Anthropic API"), "shots": shots})
+    node.write_json("text", "text.json", {"layer": "text", "schema_version": "0.1.0",
+                                          "producer": node.producer(timings=timings, device="Anthropic API"), "items": texts})

perception/_pipeline/schema/README.md

@@ -0,0 +1,54 @@
+# Track bundle format, v0.1 (draft)
+
+The contract between the pipeline (this repo), the annotator app and HyperStudy. A bundle is a directory:
+
+```
+manifest.json            what the bundle is: video identity, pipeline, layers with checksums and licence modes
+layers/<name>.json       one file per layer (shots, transcript, speakers, faces, bodies, hands, gestures, facing,
+                         gaze, objects, sounds, text, prosody); every layer is optional
+layers/faces_detail.json heavy per-sample arrays live in side files the layer points to
+assets/                  optional small images (keyframes, person and object crops) referenced by relative path
+```
+
+`bundle-0.1.schema.json` validates the manifest (the root schema) and each layer file
+(`#/$defs/layer/<name>`). `pipeline/bundle/validate.py` checks a bundle directory against both, plus
+cross-references the schema cannot express (every id a layer points at exists).
+
+## Conventions
+
+- **Video identity.** A bundle never contains the video. `manifest.video.sha256` identifies it; readers ask
+  for the file and check the hash. Stimuli are often copyrighted, and films are large.
+- **Time** is seconds from the start of the video (float, millisecond precision). Spans are `start`/`end`,
+  samples are `t`.
+- **Space** is source-video pixels. Boxes are `[x, y, w, h]`. Keypoints are `[x, y, score]`, where each
+  layer states its score scale and the threshold its producer used (`score_threshold`).
+- **IDs** are strings `"<kind>:<local>"`, unique within a bundle: `shot:3`, `utt:17`, `voice:SPEAKER_01`,
+  `face:126` (a face track), `person:Person A` (a machine identity cluster), `body:41`, `hand:125`, `obj:47`
+  (an object track), `objid:white box 1` (an object identity), `sound:9`, `text:2`, `gesture:4`,
+  `facing:7`, `gaze:31`. References between layers use these ids. Ids are stable for a given pipeline run,
+  and they change when a stage re-runs; corrections re-attach by time and geometry, not by id alone.
+- **Persons are machine clusters.** The pipeline over-splits on purpose (a wrong merge misleads agents, a split
+  is fixed in review); names and merges are corrections, never written into a bundle.
+- **Producer.** Every layer file carries `producer`: stage name and version, the models used (id, source,
+  revision, licence, licence mode), the configuration that affects outputs, run time, device and timings.
+- **Licence mode** per model and per layer: `academic-only`, `commercial-ok`, `needs-review`, or `api-terms`
+  (output of a commercial API, governed by its terms). A layer's mode is the most restrictive of its models'.
+  A commercial deployment drops layers that are not `commercial-ok` or `api-terms`.
+- **Extensibility.** Core fields are required and typed; objects allow extra, producer-specific fields
+  (`additionalProperties` is open), so stages can add detail without breaking readers. Readers ignore fields they
+  do not know.
+- **Versioning.** `schema_version` is semver. Minor versions only add optional fields or layers; a major version
+  may change required fields. Readers accept bundles whose major version they know.
+
+## Corrections and added data (not part of a bundle)
+
+The annotator stores corrections as an append-only op log over a bundle (`perception-corrections`), plus the
+user's own annotation types, notes and imported layers. They reference bundle ids and the video sha256, and are
+exported as their own files. The prototype's op log (v1) uses bare track numbers and person labels; mapping it to
+v0.1 ids is mechanical (`track 126` -> `face:126`, `Person O` -> `person:Person O`).
+
+## Open for v0.2
+
+- Arrow/Parquet side files for heavy per-frame arrays (a 2-hour film's face detail would be ~1 GB of JSON).
+- A `social` layer once the social-signal rules are settled (today: gaze events and facing).
+- Per-utterance speaker evidence from more than one detector (today: LR-ASD candidates + TalkNet agreement).

perception/_pipeline/schema/bundle-0.1.schema.json

@@ -0,0 +1,371 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hyperstudy.io/schemas/perception-bundle/0.1/bundle.schema.json",
+  "title": "HyperStudy perception track bundle, v0.1",
+  "description": "Root: manifest.json. Layer files validate against #/$defs/layer/<name>. Conventions in schema/README.md.",
+  "type": "object",
+  "required": ["format", "schema_version", "created_at", "video", "pipeline", "layers"],
+  "additionalProperties": false,
+  "properties": {
+    "format": { "const": "hyperstudy-perception-bundle" },
+    "schema_version": { "$ref": "#/$defs/semver" },
+    "created_at": { "type": "string", "format": "date-time" },
+    "video": {
+      "type": "object",
+      "required": ["sha256", "duration", "fps", "width", "height"],
+      "properties": {
+        "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+        "duration": { "type": "number", "exclusiveMinimum": 0 },
+        "fps": { "type": "number", "exclusiveMinimum": 0 },
+        "width": { "type": "integer", "minimum": 1 },
+        "height": { "type": "integer", "minimum": 1 },
+        "filename": { "type": "string", "description": "Name of the source file when the bundle was made; a hint only." }
+      }
+    },
+    "pipeline": {
+      "type": "object",
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "type": "string" },
+        "version": { "type": "string" },
+        "git_commit": { "type": ["string", "null"] },
+        "source": { "type": "string", "description": "How the bundle was made, e.g. 'pipeline run' or 'converted from bake-off run'." }
+      }
+    },
+    "layers": {
+      "type": "object",
+      "propertyNames": { "$ref": "#/$defs/layerName" },
+      "additionalProperties": {
+        "type": "object",
+        "required": ["file", "sha256", "bytes", "licence_mode", "stage"],
+        "properties": {
+          "file": { "type": "string", "pattern": "^layers/[a-z_]+\\.json$" },
+          "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+          "bytes": { "type": "integer", "minimum": 0 },
+          "licence_mode": { "$ref": "#/$defs/licenceMode" },
+          "stage": { "type": "string" },
+          "side_files": {
+            "type": "array",
+            "items": {
+              "type": "object", "required": ["file", "sha256", "bytes"],
+              "properties": { "file": { "type": "string" }, "sha256": { "type": "string" }, "bytes": { "type": "integer" } }
+            }
+          }
+        }
+      }
+    },
+    "assets": {
+      "type": "array",
+      "description": "Small images under assets/ (person, face-track and object crops), referenced by relative path from layers.",
+      "items": {
+        "type": "object", "required": ["file", "sha256", "bytes"],
+        "properties": {
+          "file": { "type": "string", "pattern": "^assets/[A-Za-z0-9_./-]+$" },
+          "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+          "bytes": { "type": "integer", "minimum": 0 }
+        }
+      }
+    }
+  },
+
+  "$defs": {
+    "semver": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" },
+    "layerName": { "enum": ["shots", "transcript", "speakers", "faces", "bodies", "hands", "gestures", "facing", "gaze", "objects", "sounds", "text", "prosody", "waveform"] },
+    "licenceMode": { "enum": ["academic-only", "commercial-ok", "needs-review", "api-terms"] },
+    "time": { "type": "number", "minimum": 0 },
+    "box": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "[x, y, w, h] in source pixels" },
+    "point": { "type": "array", "items": { "type": "number" }, "minItems": 2, "maxItems": 2 },
+    "keypoints": { "type": "array", "items": { "type": "array", "items": { "type": ["number", "null"] }, "minItems": 3, "maxItems": 3 }, "description": "[[x, y, score], ...]" },
+    "span": {
+      "type": "object", "required": ["start", "end"],
+      "properties": { "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" } }
+    },
+    "id": {
+      "shot": { "type": "string", "pattern": "^shot:\\d+$" },
+      "utt": { "type": "string", "pattern": "^utt:\\d+$" },
+      "voice": { "type": "string", "pattern": "^voice:.+$" },
+      "face": { "type": "string", "pattern": "^face:\\d+$" },
+      "person": { "type": "string", "pattern": "^person:.+$" },
+      "body": { "type": "string", "pattern": "^body:\\d+$" },
+      "hand": { "type": "string", "pattern": "^hand:\\d+$" },
+      "obj": { "type": "string", "pattern": "^obj:\\d+$" },
+      "objid": { "type": "string", "pattern": "^objid:.+$" },
+      "event": { "type": "string", "pattern": "^(sound|text|gesture|facing|gaze):\\d+$" }
+    },
+    "model": {
+      "type": "object", "required": ["id", "licence_mode"],
+      "properties": {
+        "id": { "type": "string", "description": "Model name, e.g. 'facebook/sam3' or 'LR-ASD TalkSet finetune'." },
+        "source": { "type": ["string", "null"], "description": "hf://repo@revision[/file] or a URL." },
+        "revision": { "type": ["string", "null"] },
+        "licence": { "type": ["string", "null"] },
+        "licence_mode": { "$ref": "#/$defs/licenceMode" }
+      }
+    },
+    "producer": {
+      "type": "object", "required": ["stage", "stage_version", "models"],
+      "properties": {
+        "stage": { "type": "string" },
+        "stage_version": { "type": "string" },
+        "models": { "type": "array", "items": { "$ref": "#/$defs/model" } },
+        "config": { "type": "object" },
+        "device": { "type": ["string", "null"] },
+        "run_at": { "type": ["string", "null"] },
+        "timings": { "type": "object" }
+      }
+    },
+    "layerBase": {
+      "type": "object", "required": ["layer", "schema_version", "producer"],
+      "properties": {
+        "layer": { "$ref": "#/$defs/layerName" },
+        "schema_version": { "$ref": "#/$defs/semver" },
+        "producer": { "$ref": "#/$defs/producer" }
+      }
+    },
+
+    "layer": {
+      "shots": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "shots" } } }],
+        "required": ["shots"],
+        "properties": {
+          "shots": { "type": "array", "items": {
+            "type": "object", "required": ["id", "index", "start", "end"],
+            "properties": {
+              "id": { "$ref": "#/$defs/id/shot" }, "index": { "type": "integer", "minimum": 0 },
+              "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "keyframes": { "type": "array", "items": { "type": "object", "required": ["t"], "properties": { "t": { "$ref": "#/$defs/time" }, "file": { "type": ["string", "null"] } } } },
+              "caption": { "type": ["object", "null"], "properties": {
+                "description": { "type": ["string", "null"] }, "setting": { "type": ["string", "null"] }, "mood": { "type": ["string", "null"] },
+                "people": { "type": "array" }, "model": { "type": "string" } } }
+            } } }
+        }
+      },
+
+      "transcript": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "transcript" } } }],
+        "required": ["words", "utterances", "voices"],
+        "properties": {
+          "voices": { "type": "array", "items": { "type": "object", "required": ["id"], "properties": { "id": { "$ref": "#/$defs/id/voice" } } } },
+          "words": { "type": "array", "items": {
+            "type": "object", "required": ["w", "start", "end"],
+            "properties": { "w": { "type": "string" }, "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "voice": { "oneOf": [{ "$ref": "#/$defs/id/voice" }, { "type": "null" }] } } } },
+          "utterances": { "type": "array", "items": {
+            "type": "object", "required": ["id", "start", "end", "text", "shot", "words"],
+            "properties": { "id": { "$ref": "#/$defs/id/utt" }, "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "text": { "type": "string" }, "shot": { "$ref": "#/$defs/id/shot" },
+              "words": { "type": "array", "items": { "type": "integer", "minimum": 0 }, "minItems": 2, "maxItems": 2, "description": "[first, last] word index, inclusive" },
+              "voice": { "oneOf": [{ "$ref": "#/$defs/id/voice" }, { "type": "null" }] } } } }
+        }
+      },
+
+      "speakers": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "speakers" } } }],
+        "required": ["utterances"],
+        "properties": {
+          "utterances": { "type": "array", "items": {
+            "type": "object", "required": ["utt", "predicted", "candidates"],
+            "properties": {
+              "utt": { "$ref": "#/$defs/id/utt" },
+              "predicted": { "type": "object", "required": ["kind"], "properties": {
+                "kind": { "enum": ["face", "person", "offscreen"], "description": "face: a visible face track; person: a known identity not visible (trusted voice link); offscreen: an unidentified voice" },
+                "face": { "$ref": "#/$defs/id/face" }, "person": { "$ref": "#/$defs/id/person" },
+                "voice": { "oneOf": [{ "$ref": "#/$defs/id/voice" }, { "type": "null" }] },
+                "via": { "enum": ["asd", "voice_link", "none"] } } },
+              "candidates": { "type": "array", "items": { "type": "object", "required": ["face", "score"],
+                "properties": { "face": { "$ref": "#/$defs/id/face" }, "score": { "type": "number" } } } },
+              "margin": { "type": ["number", "null"] },
+              "second_opinion_agrees": { "type": ["boolean", "null"] }
+            } } },
+          "voice_links": { "type": "array", "items": { "type": "object", "required": ["voice", "person", "word_share"],
+            "properties": { "voice": { "$ref": "#/$defs/id/voice" }, "person": { "$ref": "#/$defs/id/person" },
+              "word_share": { "type": "number" }, "trusted": { "type": "boolean" } } } }
+        }
+      },
+
+      "faces": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "faces" } } }],
+        "required": ["sample_fps", "people", "tracks"],
+        "properties": {
+          "sample_fps": { "type": "number" },
+          "angles": { "type": "string", "description": "Units and sign convention of head pose and gaze." },
+          "people": { "type": "array", "items": { "type": "object", "required": ["id"],
+            "properties": { "id": { "$ref": "#/$defs/id/person" }, "first_seen": { "type": ["number", "null"] },
+              "seconds_on_screen": { "type": ["number", "null"] }, "crop": { "type": ["string", "null"] } } } },
+          "tracks": { "type": "array", "items": {
+            "type": "object", "required": ["id", "person", "shot", "start", "end", "samples"],
+            "properties": {
+              "id": { "$ref": "#/$defs/id/face" }, "shot": { "$ref": "#/$defs/id/shot" },
+              "person": { "oneOf": [{ "$ref": "#/$defs/id/person" }, { "type": "null" }], "description": "null: not assigned to any identity cluster (the clusterer's Unknown)" },
+              "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "linked_body": { "type": ["boolean", "null"], "description": "Ever linked to a posed body; false is a strong false-face hint." },
+              "crop": { "type": ["string", "null"], "description": "assets/ path of a padded crop from the track's largest detection." },
+              "samples": { "type": "array", "items": { "type": "object", "required": ["t", "box"],
+                "properties": { "t": { "$ref": "#/$defs/time" }, "box": { "$ref": "#/$defs/box" },
+                  "mouth": { "type": ["number", "null"] }, "gaze": { "oneOf": [{ "$ref": "#/$defs/point" }, { "type": "null" }] } } } }
+            } } },
+          "person_pairs": { "type": "array", "items": { "type": "object", "required": ["a", "b", "distance"],
+            "properties": { "a": { "$ref": "#/$defs/id/person" }, "b": { "$ref": "#/$defs/id/person" },
+              "distance": { "type": "number" }, "co_visible": { "type": "boolean" } } } },
+          "detail": { "type": ["object", "null"], "required": ["file"], "properties": {
+            "file": { "type": "string" }, "fields": { "type": "array", "items": { "type": "string" } } } }
+        }
+      },
+
+      "bodies": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "bodies" } } }],
+        "required": ["keypoint_names", "score_threshold", "fps", "tracks"],
+        "properties": {
+          "keypoint_names": { "type": "array", "items": { "type": "string" } },
+          "skeleton": { "type": "array", "items": { "type": "array", "items": { "type": "integer" }, "minItems": 2, "maxItems": 2 } },
+          "score_threshold": { "type": "number" }, "fps": { "type": "number" },
+          "tracks": { "type": "array", "items": { "type": "object", "required": ["id", "shot", "start", "end", "samples"],
+            "properties": { "id": { "$ref": "#/$defs/id/body" }, "shot": { "$ref": "#/$defs/id/shot" },
+              "face": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "type": "null" }] },
+              "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "samples": { "type": "array", "items": { "type": "object", "required": ["t", "box", "kpts"],
+                "properties": { "t": { "$ref": "#/$defs/time" }, "box": { "$ref": "#/$defs/box" }, "kpts": { "$ref": "#/$defs/keypoints" } } } } } } }
+        }
+      },
+
+      "hands": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "hands" } } }],
+        "required": ["keypoint_names", "fps", "hands"],
+        "properties": {
+          "keypoint_names": { "type": "array", "items": { "type": "string" } },
+          "skeleton": { "type": "array" }, "score_threshold": { "type": ["number", "null"] }, "fps": { "type": "number" },
+          "hands": { "type": "array", "items": { "type": "object", "required": ["id", "shot", "side", "start", "end", "samples"],
+            "properties": { "id": { "$ref": "#/$defs/id/hand" }, "shot": { "$ref": "#/$defs/id/shot" },
+              "side": { "enum": ["left", "right", null] },
+              "body": { "oneOf": [{ "$ref": "#/$defs/id/body" }, { "type": "null" }] },
+              "face": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "type": "null" }] },
+              "box_source": { "type": "string" },
+              "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "samples": { "type": "array", "items": { "type": "object", "required": ["t", "box", "kpts"],
+                "properties": { "t": { "$ref": "#/$defs/time" }, "box": { "$ref": "#/$defs/box" }, "kpts": { "$ref": "#/$defs/keypoints" } } } } } } }
+        }
+      },
+
+      "gestures": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "gestures" } } }],
+        "required": ["events"],
+        "properties": { "events": { "type": "array", "items": { "type": "object", "required": ["id", "kind", "start", "end"],
+          "properties": { "id": { "$ref": "#/$defs/id/event" }, "kind": { "type": "string" },
+            "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+            "face": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "type": "null" }] },
+            "hand": { "oneOf": [{ "$ref": "#/$defs/id/hand" }, { "type": "null" }] },
+            "score": { "type": ["number", "null"] }, "detail": { "type": "object" } } } } }
+      },
+
+      "facing": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "facing" } } }],
+        "required": ["events"],
+        "properties": { "events": { "type": "array", "items": { "type": "object", "required": ["id", "from", "to", "start", "end"],
+          "properties": { "id": { "$ref": "#/$defs/id/event" }, "from": { "$ref": "#/$defs/id/face" }, "to": { "$ref": "#/$defs/id/face" },
+            "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+            "mutual": { "type": "boolean" }, "score": { "type": ["number", "null"] }, "detail": { "type": "object" } } } } }
+      },
+
+      "gaze": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "gaze" } } }],
+        "required": ["fps", "tracks", "events"],
+        "properties": {
+          "fps": { "type": "number" },
+          "tracks": { "type": "array", "items": { "type": "object", "required": ["face", "samples"],
+            "properties": { "face": { "$ref": "#/$defs/id/face" },
+              "samples": { "type": "array", "items": { "type": "object", "required": ["t"],
+                "properties": { "t": { "$ref": "#/$defs/time" },
+                  "peak": { "oneOf": [{ "$ref": "#/$defs/point" }, { "type": "null" }] },
+                  "in_frame": { "type": ["number", "null"] },
+                  "target": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "$ref": "#/$defs/id/obj" }, { "enum": ["out", "camera", null] }] },
+                  "target_score": { "type": ["number", "null"] } } } } } } },
+          "events": { "type": "array", "items": { "type": "object", "required": ["id", "kind", "from", "start", "end"],
+            "properties": { "id": { "$ref": "#/$defs/id/event" }, "kind": { "enum": ["looks_at", "mutual_gaze", "joint_attention"] },
+              "from": { "$ref": "#/$defs/id/face" },
+              "to": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "type": "null" }] },
+              "object": { "oneOf": [{ "$ref": "#/$defs/id/obj" }, { "type": "null" }] },
+              "also": { "oneOf": [{ "$ref": "#/$defs/id/face" }, { "type": "null" }] },
+              "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" }, "score": { "type": ["number", "null"] } } } }
+        }
+      },
+
+      "objects": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "objects" } } }],
+        "required": ["fps", "tracks", "identities"],
+        "properties": {
+          "fps": { "type": "number" },
+          "tracks": { "type": "array", "items": { "type": "object", "required": ["id", "label", "identity", "shot", "start", "end", "samples"],
+            "properties": { "id": { "$ref": "#/$defs/id/obj" }, "label": { "type": "string" }, "identity": { "$ref": "#/$defs/id/objid" },
+              "shot": { "$ref": "#/$defs/id/shot" }, "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+              "wearer": { "oneOf": [{ "$ref": "#/$defs/id/person" }, { "type": "null" }] },
+              "verdict": { "enum": ["object", "graphic", "absent", null] },
+              "samples": { "type": "array", "items": { "type": "object", "required": ["t", "box"],
+                "properties": { "t": { "$ref": "#/$defs/time" }, "box": { "$ref": "#/$defs/box" },
+                  "poly": { "type": "array", "items": { "$ref": "#/$defs/point" } }, "score": { "type": ["number", "null"] } } } } } } },
+          "identities": { "type": "array", "items": { "type": "object", "required": ["id", "label", "tracks"],
+            "properties": { "id": { "$ref": "#/$defs/id/objid" }, "label": { "type": "string" }, "family": { "type": ["string", "null"] },
+              "worn_by": { "type": "array", "items": { "$ref": "#/$defs/id/person" } },
+              "tracks": { "type": "array", "items": { "$ref": "#/$defs/id/obj" } }, "crop": { "type": ["string", "null"] } } } },
+          "rejected_tracks": { "type": "array", "items": { "type": "object", "required": ["id", "label", "verdict", "start", "end"],
+            "properties": { "id": { "$ref": "#/$defs/id/obj" }, "verdict": { "enum": ["graphic", "absent"] } } } }
+        }
+      },
+
+      "sounds": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "sounds" } } }],
+        "required": ["events"],
+        "properties": { "events": { "type": "array", "items": { "type": "object", "required": ["id", "label", "start", "end", "sources"],
+          "properties": { "id": { "$ref": "#/$defs/id/event" }, "label": { "type": "string" },
+            "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+            "sources": { "type": "array", "items": { "type": "string" } },
+            "agreement": { "type": ["boolean", "null"], "description": "For query-model events: did a tagger agree? null for tagger events." },
+            "score": { "type": ["number", "null"] } } } } }
+      },
+
+      "text": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "text" } } }],
+        "required": ["items"],
+        "properties": { "items": { "type": "array", "items": { "type": "object", "required": ["id", "text", "start", "end"],
+          "properties": { "id": { "$ref": "#/$defs/id/event" }, "text": { "type": "string" },
+            "start": { "$ref": "#/$defs/time" }, "end": { "$ref": "#/$defs/time" },
+            "box": { "oneOf": [{ "$ref": "#/$defs/box" }, { "type": "null" }] } } } } }
+      },
+
+      "prosody": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "prosody" } } }],
+        "required": ["contour", "utterances"],
+        "properties": {
+          "contour": { "type": "object", "required": ["step", "t0", "speakers", "f0_st", "speaker"],
+            "properties": { "step": { "type": "number" }, "t0": { "type": "number" },
+              "speakers": { "type": "array", "items": { "type": "string" } },
+              "f0_st": { "type": "array", "items": { "type": ["number", "null"] }, "description": "Semitones relative to each speaker's median" },
+              "speaker": { "type": "array", "items": { "type": ["integer", "null"] } } } },
+          "default_dimensional": { "type": ["string", "null"], "description": "The emotion_models key readers show by default." },
+          "emotion_models": { "type": "object", "description": "Per emotion key: the model that wrote it.",
+            "additionalProperties": { "type": "object", "required": ["model", "licence_mode", "kind"],
+              "properties": { "model": { "type": "string" }, "licence_mode": { "$ref": "#/$defs/licenceMode" },
+                "kind": { "enum": ["dimensional", "categorical"] } } } },
+          "caption_models": { "type": "object", "description": "Per caption key: the model that wrote it.",
+            "additionalProperties": { "type": "object", "required": ["model", "licence_mode"],
+              "properties": { "model": { "type": "string" }, "licence_mode": { "$ref": "#/$defs/licenceMode" } } } },
+          "utterances": { "type": "array", "items": { "type": "object", "required": ["utt"],
+            "properties": { "utt": { "$ref": "#/$defs/id/utt" }, "speaker": { "type": ["string", "null"] },
+              "measures": { "type": "object" },
+              "words": { "type": "array" },
+              "emotion": { "type": "object", "description": "Per emotion_models key: {arousal, valence, dominance} or categorical scores." },
+              "captions": { "type": "object", "description": "Per caption_models key: {description, tags, emphasized_words}." } } } }
+        }
+      },
+
+      "waveform": {
+        "allOf": [{ "$ref": "#/$defs/layerBase" }, { "properties": { "layer": { "const": "waveform" } } }],
+        "required": ["hz", "peaks"],
+        "properties": {
+          "hz": { "type": "number", "exclusiveMinimum": 0, "description": "Bins per second." },
+          "peaks": { "type": "array", "items": { "type": "integer", "minimum": 0, "maximum": 100 },
+            "description": "Peak absolute amplitude per bin, 0-100 relative to the loudest bin." }
+        }
+      }
+    }
+  }
+}