hyperstudy-perception 0.1.4__tar.gz → 0.1.5__tar.gz

{hyperstudy_perception-0.1.4 → hyperstudy_perception-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hyperstudy-perception
-Version: 0.1.4
+Version: 0.1.5
 Summary: Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers
 Project-URL: Homepage, https://github.com/hyperstudyio/hyperstudy-perception
 Project-URL: Changelog, https://github.com/hyperstudyio/hyperstudy-perception/releases

{hyperstudy_perception-0.1.4 → hyperstudy_perception-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hyperstudy-perception"
-version = "0.1.4"
+version = "0.1.5"
 description = "Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers"
 readme = "README.md"
 license = "MIT"

{hyperstudy_perception-0.1.4 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/RELEASE.json

@@ -1,5 +1,5 @@
 {
-"commit": "7a657d9319167d9fb6eb8374260e9d74d72365b2",
+"commit": "b174b9b1d8d56b59dd69d36db67f93bcf0537899",
 "files": [
 "cloud/runpod/guard.sh",
 "cloud/runpod/image/Dockerfile",
@@ -34,6 +34,7 @@
 "envs/vision/uv.lock",
 "graphs/decode.yaml",
 "graphs/faces.yaml",
+"graphs/full-objects-names.yaml",
 "graphs/full.yaml",
 "graphs/mvp-captions.yaml",
 "graphs/mvp.yaml",
@@ -245,8 +246,13 @@
 "nodes/vision/objects/testdata/node_1_1_0.py",
 "nodes/vision/objects/track.py",
 "nodes/vision/objects/vocabulary.py",
+"nodes/vlm/shot-captions-objects/manifest.yaml",
+"nodes/vlm/shot-captions-objects/node.py",
+"nodes/vlm/shot-captions-objects/test_shot_captions_objects.py",
+"nodes/vlm/shot-captions/captioning.py",
 "nodes/vlm/shot-captions/manifest.yaml",
 "nodes/vlm/shot-captions/node.py",
+"nodes/vlm/shot-captions/test_captioning.py",
 "schema/README.md",
 "schema/bundle-0.1.schema.json",
 "sdk/pyproject.toml",

hyperstudy_perception-0.1.4/src/perception/_pipeline/graphs/full.yaml → hyperstudy_perception-0.1.5/src/perception/_pipeline/graphs/full-objects-names.yaml

@@ -1,18 +1,11 @@
-# Everything the pipeline can do: the graph the annotator app runs. The app's feature list picks what runs (--only, with
-# what it needs), settings (--set) and places (--place). Nothing leaves the machine without --allow-uploads, and then
-# only the kinds allow_upload names: frames, faces and transcript for the captions API (frames and faces also for the
-# objects steps' naming and crop check), audio and transcript for the delivery captions (per-utterance snippets to
-# OpenAI), and video and audio for a step placed on another machine (a lab server, a cloud GPU), which receives its
-# inputs. Embeddings too: the faces layer holds ArcFace identity distances (person_pairs), and
-# every step that reads it (bodies, gaze, hands...) receives it when placed elsewhere; the run's consent still names it
-# per step. max_cost_usd caps the spend.
-#   perception plan graphs/full.yaml --input video=PATH/TO/VIDEO.mp4
-#   perception run graphs/full.yaml --input video=PATH --only asr,faces          # the local, free features
-#   perception run graphs/full.yaml --input video=PATH --only gestures           # gestures without objects (its optional
-#                                                    input): in a run of everything, a refused or failed objects cancels it
-#   perception run graphs/full.yaml --input video=PATH --allow-uploads           # everything, captions too
+# The full graph (full.yaml) with the objects named in a step of their own (run-anywhere 1b, A): objects_names names
+# them with Claude Haiku 5.5 from decode's keyframes, a second pass over the keyframes the captions step reads.
+# full.yaml has the captions call name them instead (B), chosen 2026-11-01: on held-out OOO clips 3 and 4 the
+# captions read the same blind, clothing was the same, and A's extra objects were mostly stage lights, which the
+# naming rules exclude (RUN_ANYWHERE_PLAN.md 1b). Kept to compare the two on more clips.
+#   perception remote run HOST graphs/full-objects-names.yaml --input video=PATH --allow-uploads
 schema_version: 1
-pipeline: perception-full
+pipeline: perception-full-objects-names
 inputs:
   video: {type: video.v1, uri: '../data/video.mp4'}
 policy:
@@ -63,8 +56,7 @@ nodes:
   # to the API: ANTHROPIC_API_KEY), as soon as decode is done; objects_track outlines and tracks them with SAM 3 and
   # embeds them with DINOv3 (both gated: HF_TOKEN), on the GPU with no API; objects checks each track's crops with Haiku
   # (frames and faces again), finds who wears what (faces, bodies) and links them across shots: the objects layer. The
-  # naming duplicates the captions' pass over the same keyframes; whether the captions call should name them instead
-  # is measured (RUN_ANYWHERE_PLAN.md 1b)
+  # naming duplicates the captions' pass over the same keyframes; full.yaml has the captions call name them
   - id: objects_names
     uses: vision.objects-names@1.0.0
     inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, media: $nodes.decode.media}

hyperstudy_perception-0.1.5/src/perception/_pipeline/graphs/full.yaml

@@ -0,0 +1,101 @@
+# Everything the pipeline can do: the graph the annotator app runs. The app's feature list picks what runs (--only, with
+# what it needs), settings (--set) and places (--place). Nothing leaves the machine without --allow-uploads, and then
+# only the kinds allow_upload names: frames, faces and transcript for the captions API (whose call also names the
+# objects; frames and faces also for the objects' crop check), audio and transcript for the delivery captions
+# (per-utterance snippets to OpenAI), and video and audio for a step placed on another machine (a lab server, a cloud
+# GPU), which receives its inputs. Embeddings too: the faces layer holds ArcFace identity distances (person_pairs), and
+# every step that reads it (bodies, gaze, hands...) receives it when placed elsewhere; the run's consent still names it
+# per step. max_cost_usd caps the spend.
+#   perception plan graphs/full.yaml --input video=PATH/TO/VIDEO.mp4
+#   perception run graphs/full.yaml --input video=PATH --only asr,faces          # the local, free features
+#   perception run graphs/full.yaml --input video=PATH --only gestures           # gestures without objects (its optional
+#                                                    input): in a run of everything, a refused or failed objects cancels it
+#   perception run graphs/full.yaml --input video=PATH --allow-uploads           # everything, captions too
+schema_version: 1
+pipeline: perception-full
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}
+policy:
+  placement: hybrid
+  privacy: local-preferred
+  allow_upload: [video, audio, frames, faces, transcript, embeddings]
+  # Measured on held-out OOO clips 3 and 4 (28 and 39 shots; liquidswords2, 2026-10-30): the captions with the names
+  # $0.029 and $0.037 (about $0.001 a shot, the class call included; A's captions alone $0.023 and $0.033), the crop
+  # check $0.022 and $0.021, whole runs $0.32 and $0.37 (A's $0.31 and $0.38)
+  max_cost_usd: 1.50
+  retries: 1
+nodes:
+  - id: decode
+    uses: core.decode@1.1.0
+    inputs: {video: $inputs.video}
+  - id: asr
+    uses: audio.asr@1.1.0
+    inputs: {audio: $nodes.decode.audio, shots: $nodes.decode.shots}
+  # refines asr's transcript: the same words and utterances, with their voices (pyannote; gated, HF_TOKEN)
+  - id: diarize
+    uses: audio.diarize@1.1.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.asr.transcript}
+  - id: sounds
+    uses: audio.sounds@1.1.0
+    inputs: {video: $inputs.video}
+  - id: waveform
+    uses: audio.waveform@1.0.0
+    inputs: {audio: $nodes.decode.audio}
+  - id: faces
+    uses: vision.face-track@1.4.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}
+  # who on screen speaks each utterance (LR-ASD over the face tracks and the audio), and which person each voice is
+  - id: speakers
+    uses: vision.active-speaker@1.1.0
+    inputs: {video: $inputs.video, audio: $nodes.decode.audio, shots: $nodes.decode.shots, faces: $nodes.faces.faces,
+             transcript: $nodes.diarize.transcript}
+  - id: bodies
+    uses: vision.body-pose@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots, faces: $nodes.faces.faces}
+  # who faces whom: MediaPipe 3D torso direction for the bodies' own person boxes (no second detector pass)
+  - id: facing
+    uses: vision.facing@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, bodies: $nodes.bodies.bodies}
+  # hand boxes from the bodies' hand keypoints and a hand detector; each hand's body and face come with the bodies
+  - id: hands
+    uses: vision.hands@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots, bodies: $nodes.bodies.bodies}
+  # objects in two steps: objects_track outlines and tracks the things the captions call names (its vocabulary; how
+  # this was chosen over a names step of its own is in graphs/full-objects-names.yaml)
+  # with SAM 3 and embeds them with DINOv3 (both gated: HF_TOKEN), on the GPU with no API; objects checks each track's
+  # crops with Haiku (frames and faces), finds who wears what (faces, bodies) and links them across shots
+  - id: objects_track
+    uses: vision.objects-track@1.0.0
+    inputs: {vocabulary: $nodes.captions.vocabulary, video: $inputs.video, media: $nodes.decode.media,
+             shots: $nodes.decode.shots, faces: $nodes.faces.faces, bodies: $nodes.bodies.bodies}
+  - id: objects
+    uses: vision.objects@2.0.0
+    inputs: {tracks: $nodes.objects_track.tracks, shots: $nodes.decode.shots, faces: $nodes.faces.faces,
+             bodies: $nodes.bodies.bodies, media: $nodes.decode.media}
+  # hand over mouth, self-touch, pointing and raised hands from the hands' keypoints, py-feat's landmarks and the bodies'
+  # shoulders. objects is optional: with it, an object can be what a hand points at; leaving objects out (--only)
+  # still runs gestures, pointing at faces only
+  - id: gestures
+    uses: vision.gestures@1.0.0
+    inputs: {video: $inputs.video, hands: $nodes.hands.hands, bodies: $nodes.bodies.bodies, faces: $nodes.faces.faces,
+             detail: $nodes.faces.detail, objects: $nodes.objects.objects}
+  # objects is optional for gaze: when the objects step runs, its tracks are gaze targets too (gaze waits for it); when
+  # --only leaves it out, gaze runs on the faces alone
+  - id: gaze
+    uses: vision.gaze@1.2.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, faces: $nodes.faces.faces, objects: $nodes.objects.objects}
+  # the speech it reads is diarized, as the bake-off's captions read it: lines by voice (SPEAKER_00, ...), not one SPEAKER
+  - id: captions
+    uses: vlm.shot-captions-objects@1.0.0
+    inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, transcript: $nodes.diarize.transcript,
+             faces: $nodes.faces.faces, media: $nodes.decode.media}
+  # how each utterance is said: Praat's measures against its speaker's pitch range (the speakers layer names the speaker)
+  # and the Odyssey baseline's arousal, dominance and valence, all on this machine
+  - id: prosody
+    uses: audio.prosody@1.0.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.diarize.transcript, speakers: $nodes.speakers.speakers}
+  # the delivery in words and tags, by gpt-audio-1.5 (each utterance's audio with 1 s of context, and its transcript
+  # lines, go to OpenAI: OPENAI_API_KEY), written into the prosody layer. Its own node, so prosody runs without uploads
+  - id: prosody_captions
+    uses: audio.prosody-captions@1.0.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.diarize.transcript, prosody: $nodes.prosody.prosody}

{hyperstudy_perception-0.1.4 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/vision/objects/haiku.py

@@ -44,12 +44,7 @@ VOCAB_SCHEMA = {
     "additionalProperties": False,
 }
 
-VOCAB_INSTRUCTIONS = f"""These are three frames from one shot of a video. List up to {MAX_OBJECTS} concrete \
-physical objects that are clearly visible in the frames and could be outlined and tracked with a \
-segmentation mask.
-
-Rules:
-- Generic nouns a detector would understand: 1-2 words, singular, lower case, the plain name of \
+VOCAB_RULES = """- Generic nouns a detector would understand: 1-2 words, singular, lower case, the plain name of \
 the kind of thing (e.g. "bicycle", "ramp", "mug", "car", "helmet", "chair"), not a description of \
 this one ("helmet", not "full-face helmet"; "cup", not "white paper cup").
 - Never people, faces, hands or other body parts, and never hair, wigs or beards.
@@ -61,7 +56,14 @@ separately. Other things worn or held are objects (e.g. "helmet", "glove", "earr
 "watch", "bag", "microphone").
 - On-screen text and logos are not objects.
 - If an object of the same kind was named in an earlier shot, reuse exactly that phrase.
-- Most prominent first. Return an empty list if there is nothing that fits."""
+- Most prominent first. Return an empty list if there is nothing that fits."""   # also given to the captions call that names the objects (vlm.shot-captions-objects)
+
+VOCAB_INSTRUCTIONS = f"""These are three frames from one shot of a video. List up to {MAX_OBJECTS} concrete \
+physical objects that are clearly visible in the frames and could be outlined and tracked with a \
+segmentation mask.
+
+Rules:
+{VOCAB_RULES}"""
 
 NAMES_SCHEMA = {
     "type": "object",
@@ -156,7 +158,7 @@ def reply_json(response):
 
 def fan_out(fn, items, workers: int | None = None) -> list:
     """[fn(x) for x in items] on `workers` threads (WORKERS when not given), in order, raising the first failure:
-    prosody-captions' style_captions.fan_out (a node's code is its own folder's, so it is not imported from there). At
+    prosody-captions' style_captions.fan_out (a copy: that node's runner does not share this folder, Runner.shares). At
     most `workers` calls are submitted at a time, the next only once one has finished, so after a failure no call
     starts: those running finish (each billed call reports its spend itself) and the failure is raised. The results are
     in the items' order whatever order the calls finish in."""

hyperstudy_perception-0.1.4/src/perception/_pipeline/nodes/vlm/shot-captions/node.py → hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions/captioning.py

@@ -1,11 +1,13 @@
-"""vlm.shot-captions: one description per shot from its keyframes and speech, with the face roster.
-
-Ported from pipeline/bakeoff/caption.py (Claude Haiku 5.5 with the face roster: 126 of 135 boxed people named on
-OOO, about $0.04 a clip). Each shot sees its three keyframes, with tracked faces boxed and labelled, the speech that
-starts inside it, the people seen so far and the previous shot's description; never anything from later shots.
-That chaining is why the calls run in order rather than as a Message Batch.
-
-Writes the shots layer with captions (it refines the decode node's shots layer) and the on-screen text layer.
+"""The shot captions' code (vlm.shot-captions, node.py), shared with the step that also names the things on screen in the
+same call (vlm.shot-captions-objects, which imports it: its runner shares this folder). Ported from
+pipeline/bakeoff/caption.py (Claude Haiku 5.5 with the face roster: 126 of 135 boxed people named on OOO, about $0.04 a
+clip). Each shot sees its three keyframes, with tracked faces boxed and labelled, the speech that starts inside it, the
+people seen so far and the previous shot's description; never anything from later shots. That chaining is why the calls
+run in order rather than as a Message Batch.
+
+run() writes the shots layer with captions (it refines the decode node's shots layer) and the on-screen text layer.
+Another step adds to each call with `more` (text after the prompt, and properties in the reply's schema) and reads each
+reply with `each`.
 """
 
 import base64
@@ -13,7 +15,7 @@ import io
 import json
 import time
 
-from perception_node import Node, NodeError
+from perception_node import NodeError
 
 PRICE_PER_MTOK = {"claude-haiku-5-5": (0.10, 0.50), "claude-opus-5-5": (4.00, 20.00)}   # input, output (USD)
 
@@ -109,14 +111,14 @@ def labelled_keyframes(shot, keyframes_dir, faces, width) -> list[bytes]:
     return out
 
 
-def caption(client, model, effort, prompt, images, spend):
+def caption(client, model, effort, prompt, images, spend, schema=None):
     import anthropic
 
     content = [{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": base64.standard_b64encode(b).decode()}}
                for b in images] + [{"type": "text", "text": prompt}]
     try:
         response = client.messages.create(model=model, max_tokens=4000, messages=[{"role": "user", "content": content}],
-                                          output_config={"effort": effort, "format": {"type": "json_schema", "schema": SCHEMA}})
+                                          output_config={"effort": effort, "format": {"type": "json_schema", "schema": schema or SCHEMA}})
     except (anthropic.RateLimitError, anthropic.InternalServerError, anthropic.APIConnectionError) as e:
         raise NodeError("transient", f"{type(e).__name__}: {e}") from None   # the SDK already retried twice
     except anthropic.AuthenticationError:
@@ -124,6 +126,8 @@ def caption(client, model, effort, prompt, images, spend):
     spend(response.usage.input_tokens, response.usage.output_tokens)   # billed even if the reply is then rejected
     if response.stop_reason == "refusal":
         raise NodeError("input", f"refusal: {response.stop_details}")
+    if response.stop_reason == "max_tokens":   # as haiku.reply_json: the JSON is cut off
+        raise NodeError("internal", f"{model}'s caption reached max_tokens: its JSON is cut off")
     text = next((b.text for b in response.content if b.type == "text"), None)
     if text is None:
         raise NodeError("internal", f"no text block in the response (stop_reason {response.stop_reason})")
@@ -135,7 +139,13 @@ def spent(model, tokens_in, tokens_out):
     return round((tokens_in * price[0] + tokens_out * price[1]) / 1e6, 4)
 
 
-with Node() as node:
+def run(node, more=None, each=None, schema_more=None):
+    """Caption every shot and write the shots and text layers. `more(shot)`: text added at the end of the shot's prompt;
+    `schema_more`: properties added to the reply's schema (each required); `each(shot, reply)`: called with every
+    reply, in order. Returns the captions' spend: USD and tokens (usd, tokens_in, tokens_out), which is what the step
+    reports; a step that makes other calls adds its own to them."""
+    schema = SCHEMA if not schema_more else {**SCHEMA, "properties": {**SCHEMA["properties"], **schema_more},
+                                            "required": SCHEMA["required"] + list(schema_more)}
     import anthropic
 
     c = node.config
@@ -163,8 +173,10 @@ with Node() as node:
                   f"Keyframes are at {', '.join(str(k['t']) + 's' for k in shot['keyframes'])}.\n\n"
                   f"Speech during this shot:\n{shot_speech(shot, words)}\n\nPeople seen in earlier shots:\n{roster_text}\n\n"
                   f"Previous shot: {previous or '(this is the first shot)'}\n\n{FACES_INSTRUCTIONS}\n\nFace roster so far:\n"
-                  + ("\n".join(f"- {s}" for s in seen) or "(nobody yet)"))
-        cap = caption(client, model, effort, prompt, labelled_keyframes(shot, keyframes_dir, faces, width), spend)
+                  + ("\n".join(f"- {s}" for s in seen) or "(nobody yet)")) + (more(shot) if more else "")
+        cap = caption(client, model, effort, prompt, labelled_keyframes(shot, keyframes_dir, faces, width), spend, schema)
+        if each:
+            each(shot, cap)
         for person in cap["people"]:
             roster.setdefault(person["label"], person["appearance"])
         previous = cap["description"]
@@ -183,3 +195,4 @@ with Node() as node:
     node.write_json("shots", "shots.json", {**shots_doc, "producer": node.producer(timings=timings, device="Anthropic API"), "shots": shots})
     node.write_json("text", "text.json", {"layer": "text", "schema_version": "0.1.0",
                                           "producer": node.producer(timings=timings, device="Anthropic API"), "items": texts})
+    return {"usd": spent(model, totals["in"], totals["out"]), "tokens_in": totals["in"], "tokens_out": totals["out"]}

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions/node.py

@@ -0,0 +1,15 @@
+"""vlm.shot-captions: one description per shot from its keyframes and speech, with the face roster.
+
+Ported from pipeline/bakeoff/caption.py (Claude Haiku 5.5 with the face roster: 126 of 135 boxed people named on
+OOO, about $0.04 a clip). Each shot sees its three keyframes, with tracked faces boxed and labelled, the speech that
+starts inside it, the people seen so far and the previous shot's description; never anything from later shots.
+That chaining is why the calls run in order rather than as a Message Batch.
+
+Writes the shots layer with captions (it refines the decode node's shots layer) and the on-screen text layer.
+"""
+
+from perception_node import Node
+from captioning import run
+
+with Node() as node:
+    run(node)

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions/test_captioning.py

@@ -0,0 +1,38 @@
+"""The captions call (captioning.caption) with Claude stood in for: a reply cut off at max_tokens is said so, not
+parsed (its JSON is incomplete), and is still billed.
+Run: uv run --no-project --with pytest --with-editable ./sdk pytest -q nodes/vlm/shot-captions"""
+
+import sys
+import types
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+
+@pytest.fixture
+def anthropic(monkeypatch):
+    m = types.ModuleType("anthropic")
+    for name in ("RateLimitError", "InternalServerError", "APIConnectionError", "AuthenticationError"):
+        setattr(m, name, type(name, (Exception,), {}))
+    monkeypatch.setitem(sys.modules, "anthropic", m)
+
+
+def client(stop_reason: str, text: str):
+    reply = SimpleNamespace(stop_reason=stop_reason, stop_details=None, content=[SimpleNamespace(type="text", text=text)],
+                            usage=SimpleNamespace(input_tokens=1000, output_tokens=4000))
+    return SimpleNamespace(messages=SimpleNamespace(create=lambda **request: reply))
+
+
+def test_a_reply_cut_off_at_max_tokens_is_said_so_and_billed(anthropic):
+    import captioning
+    from perception_node import NodeError
+    billed = []
+    with pytest.raises(NodeError, match="max_tokens") as e:
+        captioning.caption(client("max_tokens", '{"description": "Two people'), "claude-haiku-5-5", "low", "prompt", [],
+                           lambda n_in, n_out: billed.append((n_in, n_out)))
+    assert e.value.code == "internal" and billed == [(1000, 4000)]
+    assert captioning.caption(client("end_turn", '{"description": "x"}'), "claude-haiku-5-5", "low", "prompt", [],
+                              lambda *_: None) == {"description": "x"}

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions-objects/manifest.yaml

@@ -0,0 +1,35 @@
+# Run-anywhere 1b, B: the captions call names the things on screen too (one response, two parts), and writes the
+# vocabulary the objects' track step reads, instead of vision.objects-names (A). The full graph's since 2026-11-01,
+# measured against A on held-out clips (RUN_ANYWHERE_PLAN.md 1b). Its code is the captions' (nodes/vlm/shot-captions)
+# and the objects' naming rules (nodes/vision/objects), which its runner shares.
+id: vlm.shot-captions-objects
+version: 1.0.0
+stage: extract+caption
+title: Shot captions
+service: Anthropic API
+description: Per-shot descriptions from keyframes (faces boxed and labelled), speech and the face roster, with the things on screen named in the same call for the object tracks (up to 8 a shot, in generic detector nouns); on-screen text
+inputs:
+  shots: bundle.shots@0.1
+  keyframes: assets.v1
+  transcript: bundle.transcript@0.1
+  faces: bundle.faces@0.1
+  media: media.v1
+outputs: {shots: bundle.shots@0.1, text: bundle.text@0.1, vocabulary: vocabulary.v1}
+config:
+  model: {enum: [claude-haiku-5-5, claude-opus-5-5], default: claude-haiku-5-5, description: "bake-off choice: Haiku 5.5 with the face roster"}
+  effort: {enum: [low, medium, high], default: medium}
+runners:
+  # envs/gaze, the objects steps': the naming rules need numpy (linking.py), and it has anthropic and pillow
+  - {kind: native, env: envs/gaze, entry: node.py, accelerator: cpu, shares: [nodes/vlm/shot-captions, nodes/vision/objects]}
+resources: {cpu_cores: 1, memory_gb: 1, gpu: none}
+# Sent to Anthropic: keyframes with face boxes and labels, and the transcript lines of each shot (the captions'); then
+# the clip's object names, as text, for their classes.
+privacy: {external_upload: [frames, faces, transcript]}
+secrets: [ANTHROPIC_API_KEY]
+# Upper estimate per shot, per model: the captions' (vlm.shot-captions), with room for the names in each reply and the
+# one call (Haiku) that classes them.
+cost: {usd: {claude-haiku-5-5: 0.004, claude-opus-5-5: 0.13}, by: model, per: shot, input: shots}
+models:
+  - {id: claude-haiku-5-5, licence: Anthropic API terms, licence_mode: api-terms}
+  - {id: claude-opus-5-5, licence: Anthropic API terms, licence_mode: api-terms}
+  - {id: PySceneDetect AdaptiveDetector, licence: BSD-3-Clause, licence_mode: commercial-ok}

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions-objects/node.py

@@ -0,0 +1,57 @@
+"""vlm.shot-captions-objects: the shot captions (vlm.shot-captions), with the things on screen named in the same call,
+as the vocabulary the objects' track step reads (run-anywhere 1b, B; A is vision.objects-names, a call of its own).
+
+Each shot's captions call (captioning.run) also returns `objects`: up to 8 names, under the names step's rules (haiku.py's
+VOCAB_RULES, told the names used in earlier shots), cleaned and split as that step does (linking.clean_names,
+garments.session_names). The captions' keyframes have the faces boxed and labelled; the names are of the things in
+them, never the boxes. Then one call (Haiku 5.5, as the names step) classes the clip's names (haiku.name_classes), and
+the vocabulary is written as the names step writes it (vocabulary.document), with the captions' model as its namer.
+"""
+
+import sys
+from pathlib import Path
+
+here = Path(__file__).resolve().parent
+sys.path[:0] = [str(here.parent / "shot-captions"), str(here.parents[1] / "vision" / "objects")]   # its runner shares them
+
+from perception_node import Node, NodeError   # noqa: E402
+import captioning   # noqa: E402
+import vocabulary   # noqa: E402
+from garments import session_names   # noqa: E402
+from haiku import MAX_OBJECTS, MODEL as HAIKU, VOCAB_RULES as RULES, Haiku, spent as haiku_spent   # noqa: E402
+from linking import check_classes, clean_names   # noqa: E402
+
+with Node() as node:
+    prompts, body_named, earlier = {}, {}, set()
+
+    def more(shot):
+        return (f"\n\nobjects: up to {MAX_OBJECTS} concrete physical objects clearly visible in the keyframes that could be "
+                f"outlined and tracked with a segmentation mask. The face boxes and labels are not objects.\nRules for "
+                f"objects:\n{RULES}\nNames used in earlier shots: {', '.join(sorted(earlier)) or '(none yet)'}")
+
+    def each(shot, reply):
+        named = clean_names(reply["objects"], MAX_OBJECTS)
+        earlier.update(named)
+        prompts[shot["index"]], body_named[shot["index"]] = session_names(named)
+
+    captions = captioning.run(node, more=more, each=each,
+                                  schema_more={"objects": {"type": "array", "items": {"type": "string"}}})
+    shots = node.input_json("shots")["shots"]
+    totals = {"in": 0, "out": 0, "calls": 0}
+
+    def spend(n_in, n_out):
+        totals.update({"in": totals["in"] + n_in, "out": totals["out"] + n_out, "calls": totals["calls"] + 1})
+        node.usage(usd=round(captions["usd"] + haiku_spent(totals["in"], totals["out"]), 4), class_calls=totals["calls"],
+                   tokens_in=captions["tokens_in"] + totals["in"], tokens_out=captions["tokens_out"] + totals["out"])
+
+    names = sorted({n for shot_names in prompts.values() for n in shot_names})
+    if names:   # the class call is Haiku's, whichever model captions
+        node.model(HAIKU, licence_mode="api-terms", licence="Anthropic API terms")
+    try:
+        classes = check_classes(names, Haiku(node.secret("ANTHROPIC_API_KEY"), spend).name_classes(names)) if names else {}
+    except ValueError as e:
+        raise NodeError("internal", str(e)) from None
+    node.write_json("vocabulary", "vocabulary.json",
+                    vocabulary.document(shots, prompts, body_named, classes, node.config["model"]))
+    node.usage(vocabulary={str(k): v for k, v in prompts.items()}, classes=classes,
+               body_named={str(k): v for k, v in body_named.items() if v})

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/vlm/shot-captions-objects/test_shot_captions_objects.py

@@ -0,0 +1,154 @@
+"""vlm.shot-captions-objects (run-anywhere 1b, B) against the two steps it would replace, with Claude stood in for: for
+the same answers, its vocabulary is the names step's (vision.objects-names) byte for byte, and its shots and text
+layers are the captions step's (vlm.shot-captions). The stand-in answers by what it is asked (the shot, from the
+prompt or the keyframes' bytes), never by when.
+Run: uv run --no-project --with pytest --with numpy --with pillow --with-editable ./sdk pytest -q nodes/vlm/shot-captions-objects"""
+
+import base64
+import io
+import json
+import runpy
+import sys
+import types
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+HERE = Path(__file__).resolve().parent
+CAPTIONS = HERE.parent / "shot-captions"
+OBJECTS = HERE.parents[1] / "vision" / "objects"
+
+# what Claude names in each shot: repeats across shots, a garment and "person" (the body pass's own), a blank
+NAMED = {0: ["Chair", "mug", "jacket", "helmet", " "], 1: ["chair", "poster", "person", "helmet", "chair"]}
+CLASSES = {"chair": ("chair", "individual", False), "mug": ("cup", "individual", False),
+           "helmet": ("helmet", "individual", True), "poster": ("poster", "individual", False)}
+
+
+def caption_of(shot: int) -> dict:
+    return {"description": f"Two people talk in shot {shot}.", "setting": "a studio", "mood": "calm",
+            "people": [{"label": "0", "appearance": "a red top", "action": "talking", "speaking": True}],
+            "onscreen_text": ["OOO"] if shot == 0 else []}
+
+
+class Anthropic:
+    """The client: a caption (with the shot's names when its schema asks for objects), the names alone (the names
+    step: the shot found by its keyframes' bytes), or the names' classes. calls: every request, in order."""
+    calls: list = []
+    shot_of_image: dict = {}
+
+    def __init__(self, api_key):
+        assert api_key == "key"
+        self.messages = self
+
+    def create(self, **request):
+        self.calls.append(request)
+        content = request["messages"][0]["content"]
+        text = content[-1]["text"]
+        schema = request["output_config"]["format"]["schema"]["properties"]
+        if "description" in schema:
+            shot = int(text.split("Shot ")[1].split(" ")[0])
+            answer = {**caption_of(shot), **({"objects": NAMED[shot]} if "objects" in schema else {})}
+        elif "objects" in schema:
+            images = [base64.b64decode(b["source"]["data"]) for b in content if b["type"] == "image"]
+            answer = {"objects": NAMED[self.shot_of_image[images[0]]]}
+        else:
+            names = text.split("Names:\n")[1].split("\n")
+            answer = {"names": [dict(zip(("name", "family", "kind", "worn"), (n, *CLASSES[n]))) for n in names]}
+        return SimpleNamespace(stop_reason="end_turn", stop_details=None,
+                               content=[SimpleNamespace(type="text", text=json.dumps(answer))],
+                               usage=SimpleNamespace(input_tokens=1000, output_tokens=50))
+
+
+@pytest.fixture
+def stand_in(monkeypatch):
+    m = types.ModuleType("anthropic")
+    m.Anthropic = Anthropic
+    for name in ("RateLimitError", "InternalServerError", "APIConnectionError", "AuthenticationError"):
+        setattr(m, name, type(name, (Exception,), {}))
+    monkeypatch.setitem(sys.modules, "anthropic", m)
+    monkeypatch.setattr(Anthropic, "calls", [])
+    monkeypatch.setattr(Anthropic, "shot_of_image", {})
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "key")
+    for folder in (CAPTIONS, OBJECTS):
+        monkeypatch.syspath_prepend(str(folder))
+    return Anthropic
+
+
+def inputs(folder: Path) -> dict[str, Path]:
+    """Two shots, each with two real keyframes, one face track a shot (person 0), and a word of speech in each."""
+    from PIL import Image
+    (folder / "keyframes" / "keyframes").mkdir(parents=True)
+    shots = []
+    for k in range(2):
+        frames = []
+        for i, x in enumerate("ab"):
+            buf = io.BytesIO()
+            Image.new("RGB", (64, 48), (40 * k + 20 * i, 90, 160)).save(buf, format="JPEG")
+            (folder / "keyframes" / "keyframes" / f"{k}_{x}.jpg").write_bytes(buf.getvalue())
+            Anthropic.shot_of_image[buf.getvalue()] = k
+            frames.append({"file": f"assets/keyframes/{k}_{x}.jpg", "t": k + 0.25 + 0.5 * i})
+        shots.append({"id": f"shot:{k}", "index": k, "start": float(k), "end": float(k + 1), "keyframes": frames})
+    docs = {"media": {"sha256": "0" * 64, "duration": 2.0, "width": 64, "height": 48, "fps": 10},
+            "shots": {"layer": "shots", "schema_version": "0.1.0", "shots": shots},
+            "faces": {"layer": "faces", "people": [{"id": "person:0", "first_seen": 0.1}],
+                      "tracks": [{"id": f"face:{k}", "shot": f"shot:{k}", "person": "person:0",
+                                  "samples": [{"t": k + 0.25, "box": [10, 8, 12, 14]}]} for k in range(2)]},
+            "transcript": {"words": [{"w": "hello", "start": 0.3, "end": 0.6, "voice": "voice:0"},
+                                     {"w": "again", "start": 1.4, "end": 1.7, "voice": "voice:0"}]}}
+    for name, doc in docs.items():
+        (folder / f"{name}.json").write_text(json.dumps(doc))
+    return {"keyframes": folder / "keyframes", **{name: folder / f"{name}.json" for name in docs}}
+
+
+def run(script: Path, folder: Path, given: dict, config: dict, monkeypatch) -> dict:
+    """A node script's response, run here as the coordinator runs it (its request, its own folder)."""
+    folder.mkdir(parents=True)
+    request = folder / "request.json"
+    request.write_text(json.dumps({"protocol": 1, "out_dir": str(folder / "out"), "accelerator": "cpu", "config": config,
+                                   "secrets": ["ANTHROPIC_API_KEY"], "producer": {"stage": "captions", "stage_version": "1.0.0"},
+                                   "inputs": {name: {"path": str(path)} for name, path in given.items()}}))
+    monkeypatch.setattr(sys, "argv", [script.name, "--request", str(request)])
+    try:
+        runpy.run_path(str(script), run_name="__main__")
+    except SystemExit:   # how a node ends after writing a failed response
+        pass
+    response = json.loads((folder / "response.json").read_text())
+    assert response["ok"], response["error"]
+    return response
+
+
+CONFIG = {"model": "claude-haiku-5-5", "effort": "medium"}
+
+
+def test_its_vocabulary_is_the_names_steps_and_its_layers_are_the_captions_steps(tmp_path, monkeypatch, stand_in):
+    given = inputs(tmp_path / "in")
+    run(OBJECTS / "names.py", tmp_path / "names", {k: given[k] for k in ("shots", "keyframes", "media")}, {}, monkeypatch)
+    run(CAPTIONS / "node.py", tmp_path / "captions", given, CONFIG, monkeypatch)
+    plain = [c for c in stand_in.calls if "description" in c["output_config"]["format"]["schema"]["properties"]]
+    stand_in.calls.clear()
+    both = run(HERE / "node.py", tmp_path / "both", given, CONFIG, monkeypatch)
+    out = lambda step, name: (tmp_path / step / "out" / name).read_bytes()
+    assert out("both", "vocabulary.json") == out("names", "vocabulary.json")
+    vocabulary = json.loads(out("both", "vocabulary.json"))
+    assert vocabulary["shots"][1]["names"] == ["chair", "poster", "helmet"] and vocabulary["shots"][1]["body_named"] == ["person"]
+    assert out("both", "shots.json") == out("captions", "shots.json") and out("both", "text.json") == out("captions", "text.json")
+    # one call a shot, as the captions make, with the names part added; then the names' classes
+    calls = stand_in.calls
+    assert len(calls) == 3 and "names" in calls[2]["output_config"]["format"]["schema"]["properties"]
+    for mine, theirs in zip(calls[:2], plain):
+        assert mine["messages"][0]["content"][:-1] == theirs["messages"][0]["content"][:-1]   # the same keyframes
+        assert mine["messages"][0]["content"][-1]["text"].startswith(theirs["messages"][0]["content"][-1]["text"])
+    assert "Names used in earlier shots: (none yet)" in calls[0]["messages"][0]["content"][-1]["text"]
+    # every name given before (cleaned; a garment too, as the names step tells it), never a later shot's
+    assert "Names used in earlier shots: chair, helmet, jacket, mug" in calls[1]["messages"][0]["content"][-1]["text"]
+    assert both["usage"]["calls"] == 2 and both["usage"]["class_calls"] == 1 and both["usage"]["usd"] > 0
+
+
+def test_its_usage_counts_the_class_call_and_names_haiku_with_any_captions_model(tmp_path, monkeypatch, stand_in):
+    """The class call is Haiku's whatever model captions: it is among the models used, and its tokens are counted."""
+    given = inputs(tmp_path / "in")
+    both = run(HERE / "node.py", tmp_path / "both", given, {**CONFIG, "model": "claude-opus-5-5"}, monkeypatch)
+    assert {m["id"] for m in both["models"]} >= {"claude-opus-5-5", "claude-haiku-5-5"}
+    usage = both["usage"]   # the stand-in bills 1000 tokens in, 50 out a call: two shots' captions, one class call
+    assert (usage["tokens_in"], usage["tokens_out"], usage["calls"], usage["class_calls"]) == (3000, 150, 2, 1)