hyperstudy-perception 0.1.3__tar.gz → 0.1.5__tar.gz

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hyperstudy-perception
-Version: 0.1.3
+Version: 0.1.5
 Summary: Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers
 Project-URL: Homepage, https://github.com/hyperstudyio/hyperstudy-perception
 Project-URL: Changelog, https://github.com/hyperstudyio/hyperstudy-perception/releases

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hyperstudy-perception"
-version = "0.1.3"
+version = "0.1.5"
 description = "Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers"
 readme = "README.md"
 license = "MIT"

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/RELEASE.json

@@ -1,5 +1,5 @@
 {
-"commit": "ba5bb86926a9a7218d908ad5f7dc0371cf87a0c3",
+"commit": "b174b9b1d8d56b59dd69d36db67f93bcf0537899",
 "files": [
 "cloud/runpod/guard.sh",
 "cloud/runpod/image/Dockerfile",
@@ -34,6 +34,7 @@
 "envs/vision/uv.lock",
 "graphs/decode.yaml",
 "graphs/faces.yaml",
+"graphs/full-objects-names.yaml",
 "graphs/full.yaml",
 "graphs/mvp-captions.yaml",
 "graphs/mvp.yaml",
@@ -65,6 +66,7 @@
 "nodes/audio/prosody-captions/manifest.yaml",
 "nodes/audio/prosody-captions/node.py",
 "nodes/audio/prosody-captions/style_captions.py",
+"nodes/audio/prosody-captions/test_prosody_captions_node.py",
 "nodes/audio/prosody-captions/test_style_captions.py",
 "nodes/audio/prosody/manifest.yaml",
 "nodes/audio/prosody/node.py",
@@ -244,8 +246,13 @@
 "nodes/vision/objects/testdata/node_1_1_0.py",
 "nodes/vision/objects/track.py",
 "nodes/vision/objects/vocabulary.py",
+"nodes/vlm/shot-captions-objects/manifest.yaml",
+"nodes/vlm/shot-captions-objects/node.py",
+"nodes/vlm/shot-captions-objects/test_shot_captions_objects.py",
+"nodes/vlm/shot-captions/captioning.py",
 "nodes/vlm/shot-captions/manifest.yaml",
 "nodes/vlm/shot-captions/node.py",
+"nodes/vlm/shot-captions/test_captioning.py",
 "schema/README.md",
 "schema/bundle-0.1.schema.json",
 "sdk/pyproject.toml",

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/cloud/runpod/image/Dockerfile

@@ -8,10 +8,11 @@
 FROM runpod/pytorch:2.1.0-py3.10-cuda11.8.0-devel-ubuntu22.04@sha256:2594f25beeef751aeebcc207ba4c45a382ba174655f01e15cbbfad0e2bdb5ef8
 ENV DEBIAN_FRONTEND=noninteractive
 
-# what nodes and the driver use (setup then installs nothing); sshd takes keys only. Each pod makes its own host keys
-# at start (start.sh): the ones in the base image are public, and are never used
+# what nodes and the driver use (setup then installs nothing): ffmpeg, rsync, and the EGL and GLES libraries MediaPipe
+# loads even on the CPU (vision.facing failed for want of libEGL.so.1 on a pod, 2026-10-31). sshd takes keys only.
+# Each pod makes its own host keys at start (start.sh): the ones in the base image are public, and are never used
 RUN apt-get update \
- && apt-get install -y --no-install-recommends ffmpeg rsync \
+ && apt-get install -y --no-install-recommends ffmpeg rsync libegl1 libgles2 \
  && rm -rf /var/lib/apt/lists/* /etc/ssh/ssh_host_* \
  && mkdir -p /run/sshd /etc/ssh/sshd_config.d \
  && printf 'PermitRootLogin prohibit-password\nPasswordAuthentication no\nKbdInteractiveAuthentication no\n' \

hyperstudy_perception-0.1.3/src/perception/_pipeline/graphs/full.yaml → hyperstudy_perception-0.1.5/src/perception/_pipeline/graphs/full-objects-names.yaml

@@ -1,18 +1,11 @@
-# Everything the pipeline can do: the graph the annotator app runs. The app's feature list picks what runs (--only, with
-# what it needs), settings (--set) and places (--place). Nothing leaves the machine without --allow-uploads, and then
-# only the kinds allow_upload names: frames, faces and transcript for the captions API (frames and faces also for the
-# objects steps' naming and crop check), audio and transcript for the delivery captions (per-utterance snippets to
-# OpenAI), and video and audio for a step placed on another machine (a lab server, a cloud GPU), which receives its
-# inputs. Embeddings too: the faces layer holds ArcFace identity distances (person_pairs), and
-# every step that reads it (bodies, gaze, hands...) receives it when placed elsewhere; the run's consent still names it
-# per step. max_cost_usd caps the spend.
-#   perception plan graphs/full.yaml --input video=PATH/TO/VIDEO.mp4
-#   perception run graphs/full.yaml --input video=PATH --only asr,faces          # the local, free features
-#   perception run graphs/full.yaml --input video=PATH --only gestures           # gestures without objects (its optional
-#                                                    input): in a run of everything, a refused or failed objects cancels it
-#   perception run graphs/full.yaml --input video=PATH --allow-uploads           # everything, captions too
+# The full graph (full.yaml) with the objects named in a step of their own (run-anywhere 1b, A): objects_names names
+# them with Claude Haiku 5.5 from decode's keyframes, a second pass over the keyframes the captions step reads.
+# full.yaml has the captions call name them instead (B), chosen 2026-11-01: on held-out OOO clips 3 and 4 the
+# captions read the same blind, clothing was the same, and A's extra objects were mostly stage lights, which the
+# naming rules exclude (RUN_ANYWHERE_PLAN.md 1b). Kept to compare the two on more clips.
+#   perception remote run HOST graphs/full-objects-names.yaml --input video=PATH --allow-uploads
 schema_version: 1
-pipeline: perception-full
+pipeline: perception-full-objects-names
 inputs:
   video: {type: video.v1, uri: '../data/video.mp4'}
 policy:
@@ -63,8 +56,7 @@ nodes:
   # to the API: ANTHROPIC_API_KEY), as soon as decode is done; objects_track outlines and tracks them with SAM 3 and
   # embeds them with DINOv3 (both gated: HF_TOKEN), on the GPU with no API; objects checks each track's crops with Haiku
   # (frames and faces again), finds who wears what (faces, bodies) and links them across shots: the objects layer. The
-  # naming duplicates the captions' pass over the same keyframes; whether the captions call should name them instead
-  # is measured (RUN_ANYWHERE_PLAN.md 1b)
+  # naming duplicates the captions' pass over the same keyframes; full.yaml has the captions call name them
   - id: objects_names
     uses: vision.objects-names@1.0.0
     inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, media: $nodes.decode.media}

hyperstudy_perception-0.1.5/src/perception/_pipeline/graphs/full.yaml

@@ -0,0 +1,101 @@
+# Everything the pipeline can do: the graph the annotator app runs. The app's feature list picks what runs (--only, with
+# what it needs), settings (--set) and places (--place). Nothing leaves the machine without --allow-uploads, and then
+# only the kinds allow_upload names: frames, faces and transcript for the captions API (whose call also names the
+# objects; frames and faces also for the objects' crop check), audio and transcript for the delivery captions
+# (per-utterance snippets to OpenAI), and video and audio for a step placed on another machine (a lab server, a cloud
+# GPU), which receives its inputs. Embeddings too: the faces layer holds ArcFace identity distances (person_pairs), and
+# every step that reads it (bodies, gaze, hands...) receives it when placed elsewhere; the run's consent still names it
+# per step. max_cost_usd caps the spend.
+#   perception plan graphs/full.yaml --input video=PATH/TO/VIDEO.mp4
+#   perception run graphs/full.yaml --input video=PATH --only asr,faces          # the local, free features
+#   perception run graphs/full.yaml --input video=PATH --only gestures           # gestures without objects (its optional
+#                                                    input): in a run of everything, a refused or failed objects cancels it
+#   perception run graphs/full.yaml --input video=PATH --allow-uploads           # everything, captions too
+schema_version: 1
+pipeline: perception-full
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}
+policy:
+  placement: hybrid
+  privacy: local-preferred
+  allow_upload: [video, audio, frames, faces, transcript, embeddings]
+  # Measured on held-out OOO clips 3 and 4 (28 and 39 shots; liquidswords2, 2026-10-30): the captions with the names
+  # $0.029 and $0.037 (about $0.001 a shot, the class call included; A's captions alone $0.023 and $0.033), the crop
+  # check $0.022 and $0.021, whole runs $0.32 and $0.37 (A's $0.31 and $0.38)
+  max_cost_usd: 1.50
+  retries: 1
+nodes:
+  - id: decode
+    uses: core.decode@1.1.0
+    inputs: {video: $inputs.video}
+  - id: asr
+    uses: audio.asr@1.1.0
+    inputs: {audio: $nodes.decode.audio, shots: $nodes.decode.shots}
+  # refines asr's transcript: the same words and utterances, with their voices (pyannote; gated, HF_TOKEN)
+  - id: diarize
+    uses: audio.diarize@1.1.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.asr.transcript}
+  - id: sounds
+    uses: audio.sounds@1.1.0
+    inputs: {video: $inputs.video}
+  - id: waveform
+    uses: audio.waveform@1.0.0
+    inputs: {audio: $nodes.decode.audio}
+  - id: faces
+    uses: vision.face-track@1.4.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}
+  # who on screen speaks each utterance (LR-ASD over the face tracks and the audio), and which person each voice is
+  - id: speakers
+    uses: vision.active-speaker@1.1.0
+    inputs: {video: $inputs.video, audio: $nodes.decode.audio, shots: $nodes.decode.shots, faces: $nodes.faces.faces,
+             transcript: $nodes.diarize.transcript}
+  - id: bodies
+    uses: vision.body-pose@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots, faces: $nodes.faces.faces}
+  # who faces whom: MediaPipe 3D torso direction for the bodies' own person boxes (no second detector pass)
+  - id: facing
+    uses: vision.facing@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, bodies: $nodes.bodies.bodies}
+  # hand boxes from the bodies' hand keypoints and a hand detector; each hand's body and face come with the bodies
+  - id: hands
+    uses: vision.hands@1.1.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots, bodies: $nodes.bodies.bodies}
+  # objects in two steps: objects_track outlines and tracks the things the captions call names (its vocabulary; how
+  # this was chosen over a names step of its own is in graphs/full-objects-names.yaml)
+  # with SAM 3 and embeds them with DINOv3 (both gated: HF_TOKEN), on the GPU with no API; objects checks each track's
+  # crops with Haiku (frames and faces), finds who wears what (faces, bodies) and links them across shots
+  - id: objects_track
+    uses: vision.objects-track@1.0.0
+    inputs: {vocabulary: $nodes.captions.vocabulary, video: $inputs.video, media: $nodes.decode.media,
+             shots: $nodes.decode.shots, faces: $nodes.faces.faces, bodies: $nodes.bodies.bodies}
+  - id: objects
+    uses: vision.objects@2.0.0
+    inputs: {tracks: $nodes.objects_track.tracks, shots: $nodes.decode.shots, faces: $nodes.faces.faces,
+             bodies: $nodes.bodies.bodies, media: $nodes.decode.media}
+  # hand over mouth, self-touch, pointing and raised hands from the hands' keypoints, py-feat's landmarks and the bodies'
+  # shoulders. objects is optional: with it, an object can be what a hand points at; leaving objects out (--only)
+  # still runs gestures, pointing at faces only
+  - id: gestures
+    uses: vision.gestures@1.0.0
+    inputs: {video: $inputs.video, hands: $nodes.hands.hands, bodies: $nodes.bodies.bodies, faces: $nodes.faces.faces,
+             detail: $nodes.faces.detail, objects: $nodes.objects.objects}
+  # objects is optional for gaze: when the objects step runs, its tracks are gaze targets too (gaze waits for it); when
+  # --only leaves it out, gaze runs on the faces alone
+  - id: gaze
+    uses: vision.gaze@1.2.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, faces: $nodes.faces.faces, objects: $nodes.objects.objects}
+  # the speech it reads is diarized, as the bake-off's captions read it: lines by voice (SPEAKER_00, ...), not one SPEAKER
+  - id: captions
+    uses: vlm.shot-captions-objects@1.0.0
+    inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, transcript: $nodes.diarize.transcript,
+             faces: $nodes.faces.faces, media: $nodes.decode.media}
+  # how each utterance is said: Praat's measures against its speaker's pitch range (the speakers layer names the speaker)
+  # and the Odyssey baseline's arousal, dominance and valence, all on this machine
+  - id: prosody
+    uses: audio.prosody@1.0.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.diarize.transcript, speakers: $nodes.speakers.speakers}
+  # the delivery in words and tags, by gpt-audio-1.5 (each utterance's audio with 1 s of context, and its transcript
+  # lines, go to OpenAI: OPENAI_API_KEY), written into the prosody layer. Its own node, so prosody runs without uploads
+  - id: prosody_captions
+    uses: audio.prosody-captions@1.0.0
+    inputs: {audio: $nodes.decode.audio, transcript: $nodes.diarize.transcript, prosody: $nodes.prosody.prosody}

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/audio/prosody-captions/node.py

@@ -80,28 +80,25 @@ with Node() as node:
         u = units[k]
         i, j = u["samples"]
         text = sc.prompt(units, k)
-        r = call(lambda: client.chat.completions.create(
-            model=sc.GPT_MODEL, modalities=["text"],
-            tools=[{"type": "function", "function": {"name": "record_style", "description": "Record how the target is said",
-                                                     "parameters": sc.CAPTION_SCHEMA, "strict": True}}],
-            tool_choice={"type": "function", "function": {"name": "record_style"}},
-            messages=[{"role": "system", "content": sc.GPT_INSTRUCTIONS},
-                      {"role": "user", "content": [{"type": "text", "text": text},
-                                                   {"type": "input_audio", "input_audio": {"data": wav_b64(pcm[2 * i:2 * j]), "format": "wav"}}]}]))
-        d = r.usage.prompt_tokens_details
-        usage = sc.cost(sc.GPT_MODEL, r.usage.prompt_tokens, d.audio_tokens if d else 0, r.usage.completion_tokens)
-        spend(usage)
-        choice = r.choices[0]
-        if not choice.message.tool_calls:
-            raise NodeError("internal", f"{u['id']}: no function call in the reply (finish_reason {choice.finish_reason})")
-        arguments = choice.message.tool_calls[0].function.arguments
-        try:
-            parsed = sc.parse_caption(arguments)
-        except ValueError as e:
-            raise NodeError("internal", f"{u['id']}: {e}") from None
-        return {"parsed": parsed, "usage": usage, "prompt": text, "outside": sc.outside_tags(arguments),
-                "raw": {"model": r.model, "finish_reason": choice.finish_reason, "arguments": arguments,
-                        "usage": r.usage.model_dump(mode="json")}}
+
+        def ask():
+            r = call(lambda: client.chat.completions.create(
+                model=sc.GPT_MODEL, modalities=["text"],
+                tools=[{"type": "function", "function": {"name": "record_style", "description": "Record how the target is said",
+                                                         "parameters": sc.CAPTION_SCHEMA, "strict": True}}],
+                tool_choice={"type": "function", "function": {"name": "record_style"}},
+                messages=[{"role": "system", "content": sc.GPT_INSTRUCTIONS},
+                          {"role": "user", "content": [{"type": "text", "text": text},
+                                                       {"type": "input_audio", "input_audio": {"data": wav_b64(pcm[2 * i:2 * j]), "format": "wav"}}]}]))
+            d = r.usage.prompt_tokens_details
+            usage = sc.cost(sc.GPT_MODEL, r.usage.prompt_tokens, d.audio_tokens if d else 0, r.usage.completion_tokens)
+            spend(usage)
+            choice = r.choices[0]
+            arguments = choice.message.tool_calls[0].function.arguments if choice.message.tool_calls else None
+            return arguments, usage, {"model": r.model, "finish_reason": choice.finish_reason, "arguments": arguments,
+                                      "usage": r.usage.model_dump(mode="json")}
+        got = sc.caption_attempt(ask)
+        return {**got, "prompt": text, "ask": ask, "asked_again": []}
 
     def tag(description: str) -> dict:
         r = call(lambda: client.chat.completions.create(
@@ -125,8 +122,26 @@ with Node() as node:
     captions = sc.fan_out(caption, range(len(units)))   # on a failure, no further call starts
     caption_s = round(time.time() - t0, 1)
     t0 = time.time()
-    tagged = sc.fan_out(lambda c: tag(c["parsed"]["description"]), captions)
+    good = [k for k, c in enumerate(captions) if "rejected" not in c]
+    tagged = dict(zip(good, sc.fan_out(lambda k: tag(captions[k]["parsed"]["description"]), good)))
     tagger_s = round(time.time() - t0, 1)
+    # a reply that was not the schema's, asked again once every other call is made: then what the step spent is all but
+    # this second call and its tag, which must stay within its reservation (style_captions.may_ask_again)
+    for k, c in enumerate(captions):
+        if "rejected" not in c:
+            continue
+        tag_usd = max((t["usage"]["usd"] for t in tagged.values()), default=c["usage"]["usd"])
+        if not sc.may_ask_again(totals["usd"], c["usage"]["usd"], tag_usd, len(units)):
+            raise NodeError("internal", f"{units[k]['id']}: not asked again (the step's reservation would not cover it): "
+                                        f"{sc.rejected_text(c['rejected'])}")
+        try:
+            captions[k] = {**c, **sc.asked_again(c, sc.caption_attempt(c["ask"]))}
+        except ValueError as e:
+            raise NodeError("internal", f"{units[k]['id']}: {e}") from None
+        tagged[k] = tag(captions[k]["parsed"]["description"])
+    tagged = [tagged[k] for k in range(len(units))]
+    for c in captions:
+        c["outside"] = sc.outside_tags(c["arguments"])
 
     for m in prosody["producer"]["models"]:   # the measures' and emotion's models, as the prosody node recorded them
         node.model(m["id"], licence_mode=m["licence_mode"], source=m.get("source"), revision=m.get("revision"), licence=m.get("licence"))
@@ -143,6 +158,9 @@ with Node() as node:
         {"file": captions_raw.FILE, "format": "parquet", "description": captions_raw.DESCRIPTION}))
     # tags the audio model gave outside the list, dropped from their captions: utterance -> tags
     outside = {u["id"]: c["outside"] for u, c in zip(units, captions) if c["outside"]}
-    node.usage(utterances=len(units), workers=sc.WORKERS, tags_outside_list=outside,
+    # replies that were not the schema's JSON, asked again once (style_captions.caption_attempt, asked_again): utterance -> the first
+    # reply as it came (its problem, finish_reason and arguments), to find out why
+    asked_again = {u["id"]: c["asked_again"][0] for u, c in zip(units, captions) if c["asked_again"]}
+    node.usage(utterances=len(units), workers=sc.WORKERS, tags_outside_list=outside, asked_again=asked_again,
                usd_by_model={sc.GPT_MODEL: round(sum(c["usage"]["usd"] for c in captions), 4),
                              sc.TAGGER: round(sum(t["usage"]["usd"] for t in tagged), 4)})

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/audio/prosody-captions/style_captions.py

@@ -136,6 +136,48 @@ def cost(model: str, prompt_tokens: int, audio_tokens: int | None, completion_to
     return {"text_in": text_in, "audio_in": audio_in, "out": completion_tokens, "usd": usd}
 
 
+def caption_attempt(ask) -> dict:
+    """One call for one target's caption: `ask()` makes it and gives (its function call's arguments, or None when it
+    made none; its usage, as cost() gives it; its raw reply). {parsed, arguments, raw, usage}, or {rejected: {problem,
+    finish_reason, arguments}, usage} when the reply is not the schema's. Why one can fail the schema is known: the
+    audio models do not constrain their function call to it (strict or not; the same reason off-list tags are dropped,
+    parse_caption), so a sample can be malformed (one was cut short of a comma on a pod's run, 2026-10-31)."""
+    arguments, usage, raw = ask()
+    try:
+        if arguments is None:
+            raise ValueError(f"no function call in the reply (finish_reason {raw.get('finish_reason')})")
+        return {"parsed": parse_caption(arguments), "arguments": arguments, "raw": raw, "usage": usage}
+    except ValueError as e:
+        return {"rejected": {"problem": str(e), "finish_reason": raw.get("finish_reason"), "arguments": arguments},
+                "usage": usage}
+
+
+def rejected_text(rejected: dict) -> str:
+    return (f"{rejected['problem']} (finish_reason {rejected['finish_reason']}, "
+            f"arguments {str(rejected['arguments'])[:300]!r})")
+
+
+def may_ask_again(spent_usd: float, caption_usd: float, tag_usd: float, utterances: int) -> bool:
+    """Whether a rejected caption may be asked again: once every other call of the step has been made (spent_usd is
+    then all but this), only if this caption's second call and its tag stay within what the step reserved (the
+    manifest's cost, an utterance: review of #53)."""
+    return spent_usd + caption_usd + tag_usd <= utterances * COST_PER_UTTERANCE
+
+
+def asked_again(first: dict, second: dict) -> dict:
+    """A caption asked a second time: the second's, with both calls' usage and the first's rejected reply
+    (asked_again). ValueError when the second is not the schema's either."""
+    if "rejected" in second:
+        raise ValueError(f"asked twice, and neither reply was the schema's: {rejected_text(first['rejected'])}; "
+                         f"{rejected_text(second['rejected'])}")
+    usage = {k: first["usage"][k] + second["usage"][k] for k in first["usage"]}
+    return {**second, "usage": usage, "asked_again": [first["rejected"]]}
+
+
+# what the step reserves an utterance (manifest.yaml's cost; a test checks they agree): one caption call and its tags
+COST_PER_UTTERANCE = 0.008
+
+
 def parse_caption(arguments: str) -> dict:
     """The function call's arguments -> {description, tags, emphasized_words}; ValueError when they are not that. Its tags
     are those of the list only: gpt-audio-1.5 does not hold a function call to the schema's enum (strict or not), so a tag

hyperstudy_perception-0.1.5/src/perception/_pipeline/nodes/audio/prosody-captions/test_prosody_captions_node.py

@@ -0,0 +1,107 @@
+"""audio.prosody-captions run as the coordinator runs it, with OpenAI stood in for: every caption made once, then the
+tags, then a reply that was not the schema's asked again once, only within what the step reserved (review of #53).
+The stand-in answers by what it is asked (the target's line, from the prompt), never by when.
+Run: uv run --no-project --with pytest --with numpy --with-editable ./sdk pytest -q nodes/audio/prosody-captions"""
+
+import json
+import runpy
+import sys
+import threading
+import types
+import wave
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+import style_captions as sc
+from test_style_captions import N, PROSODY, TRANSCRIPT
+
+HERE = Path(__file__).resolve().parent
+GOOD = {"description": "Calm and even.", "tags": ["calm"], "emphasized_words": []}
+
+
+class OpenAI:
+    """The client: a caption (the first one for `bad`'s line cut short, as gpt-audio's was on a pod), or tags. calls:
+    every request, in order; usd: what each call is billed (prompt tokens, at the models' prices)."""
+    calls: list = []
+    bad: set = set()
+    prompt_tokens = 100
+    lock = threading.Lock()
+
+    def __init__(self, api_key):
+        assert api_key == "key"
+        self.chat = SimpleNamespace(completions=self)
+
+    def create(self, **request):
+        usage = SimpleNamespace(prompt_tokens=self.prompt_tokens, completion_tokens=20,
+                                prompt_tokens_details=SimpleNamespace(audio_tokens=0), model_dump=lambda mode: {})
+        if "tools" in request:
+            text = request["messages"][1]["content"][0]["text"]
+            target = text.split('Target transcript (automatic, may contain errors): "')[1].split('"')[0]
+            with self.lock:
+                self.calls.append(request)
+                asked = sum(1 for c in self.calls if "tools" in c and c["messages"][1]["content"][0]["text"] == text)
+            arguments = '{"description": "Calm" "tags": []}' if target in self.bad and asked == 1 else json.dumps(GOOD)
+            message = SimpleNamespace(tool_calls=[SimpleNamespace(function=SimpleNamespace(arguments=arguments))])
+            return SimpleNamespace(model=request["model"], usage=usage,
+                                   choices=[SimpleNamespace(message=message, finish_reason="tool_calls")])
+        with self.lock:
+            self.calls.append(request)
+        return SimpleNamespace(model=request["model"], usage=usage,
+                               choices=[SimpleNamespace(message=SimpleNamespace(content=json.dumps({"tags": ["calm"]})),
+                                                        finish_reason="stop")])
+
+
+@pytest.fixture
+def openai(monkeypatch):
+    module = types.ModuleType("openai")
+    module.OpenAI = OpenAI
+    for name in ("RateLimitError", "InternalServerError", "APIConnectionError", "AuthenticationError"):
+        setattr(module, name, type(name, (Exception,), {}))
+    monkeypatch.setitem(sys.modules, "openai", module)
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    OpenAI.calls, OpenAI.bad, OpenAI.prompt_tokens = [], set(), 100
+    return OpenAI
+
+
+def run(tmp_path, monkeypatch) -> dict:
+    audio = tmp_path / "audio.wav"
+    with wave.open(str(audio), "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(sc.RATE)
+        w.writeframes(b"\0\0" * N)
+    (tmp_path / "transcript.json").write_text(json.dumps(TRANSCRIPT))
+    (tmp_path / "prosody.json").write_text(json.dumps(PROSODY))
+    request = tmp_path / "request.json"
+    request.write_text(json.dumps({"protocol": 1, "out_dir": str(tmp_path / "out"), "accelerator": "cpu", "config": {},
+                                   "secrets": ["OPENAI_API_KEY"], "producer": {"stage": "prosody", "stage_version": "1.0.0"},
+                                   "inputs": {"audio": {"path": str(audio)}, "transcript": {"path": str(tmp_path / "transcript.json")},
+                                              "prosody": {"path": str(tmp_path / "prosody.json")}}}))
+    monkeypatch.setattr(sys, "argv", ["node.py", "--request", str(request)])
+    try:
+        runpy.run_path(str(HERE / "node.py"), run_name="__main__")
+    except SystemExit:
+        pass
+    return json.loads((tmp_path / "response.json").read_text())
+
+
+def test_a_reply_that_is_not_the_schemas_is_asked_again_after_every_other_call(tmp_path, monkeypatch, openai):
+    openai.bad = {"orange lamp."}   # utt:3's line
+    got = run(tmp_path, monkeypatch)
+    assert got["ok"], got.get("error")
+    kinds = ["caption" if "tools" in c else "tag" for c in openai.calls]
+    # three captions, two tags (the good ones), then utt:3 asked again and tagged
+    assert kinds == ["caption"] * 3 + ["tag"] * 2 + ["caption", "tag"]
+    first = got["usage"]["asked_again"]["utt:3"]
+    assert "delimiter" in first["problem"] and first["arguments"] == '{"description": "Calm" "tags": []}'
+    assert got["usage"]["calls"] == 7
+
+
+def test_a_reply_is_not_asked_again_past_the_steps_reservation(tmp_path, monkeypatch, openai):
+    openai.bad = {"orange lamp."}
+    openai.prompt_tokens = 4000   # each call near the utterance's reservation: a second caption would pass it
+    got = run(tmp_path, monkeypatch)
+    assert not got["ok"] and "not asked again" in got["error"]["message"] and "utt:3" in got["error"]["message"]
+    assert ["caption" if "tools" in c else "tag" for c in openai.calls] == ["caption"] * 3 + ["tag"] * 2

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/audio/prosody-captions/test_style_captions.py

@@ -162,3 +162,49 @@ def test_the_fan_out_keeps_the_items_order():
     client = FailingClient(-1)
     got = sc.fan_out(lambda i: client.chat.completions.create(utterance=i)["utterance"], range(50), workers=4)
     assert got == list(range(50)) and client.calls == 50
+
+
+GOOD = json.dumps({"description": "Calm.", "tags": ["calm"], "emphasized_words": []})
+
+
+def asker(*replies):
+    """One call per reply given, in order: (arguments or None, usage, raw), as node.py's ask makes them."""
+    calls = iter(replies)
+
+    def ask():
+        arguments = next(calls)
+        return arguments, {"text_in": 10, "audio_in": 5, "out": 3, "usd": 0.01}, {"finish_reason": "tool_calls", "arguments": arguments}
+    return ask
+
+
+def test_a_caption_reply_that_is_not_the_schemas_json_is_rejected_whole_and_a_second_one_taken():
+    """gpt-audio does not constrain its function call to the schema: a reply that is not its JSON (a pod's run,
+    2026-10-31: "Expecting ',' delimiter") is kept as it came, and a second reply, if asked, is the caption, both calls
+    billed."""
+    calls = asker('{"description": "Calm" "tags": []}', GOOD)
+    first = sc.caption_attempt(calls)
+    assert "delimiter" in first["rejected"]["problem"] and first["rejected"]["arguments"] == '{"description": "Calm" "tags": []}'
+    assert first["rejected"]["finish_reason"] == "tool_calls"
+    got = sc.asked_again(first, sc.caption_attempt(calls))
+    assert got["parsed"]["description"] == "Calm." and got["arguments"] == GOOD and got["asked_again"] == [first["rejected"]]
+    assert got["usage"] == {"text_in": 20, "audio_in": 10, "out": 6, "usd": 0.02}
+    assert "rejected" not in sc.caption_attempt(asker(GOOD))
+
+
+def test_a_second_reply_that_is_not_the_schemas_fails_the_caption():
+    calls = asker("{", None)
+    first = sc.caption_attempt(calls)
+    with pytest.raises(ValueError, match="asked twice"):
+        sc.asked_again(first, sc.caption_attempt(calls))
+
+
+def test_a_reply_is_asked_again_only_within_what_the_step_reserved():
+    """Review of #53: asked once every other call is made, so what the step has spent is all but this caption's second
+    call and its tag, which must stay within the step's reservation (COST_PER_UTTERANCE an utterance)."""
+    assert sc.may_ask_again(0.07, 0.005, 0.001, 10)          # 0.076 <= 0.08
+    assert not sc.may_ask_again(0.076, 0.005, 0.001, 10)     # 0.082 > 0.08
+
+
+def test_what_an_utterance_is_reserved_is_the_manifests():
+    lines = (Path(sc.__file__).parent / "manifest.yaml").read_text().splitlines()
+    assert f"cost: {{usd: {sc.COST_PER_UTTERANCE}, per: utterance, input: transcript}}" in lines

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/audio/sounds/manifest.yaml

@@ -16,9 +16,11 @@ config:
 runners:
   - {kind: native, env: envs/audio-events, entry: node.py, accelerator: mps, platforms: [darwin/arm64]}
   # PyPI's torch 2.8 wheels for Linux carry CUDA 12.8 (envs/audio-events); the machine needs an NVIDIA driver >= 570.
-  # PE-A-Frame large is 5.8 GB of float32 weights, run on up to 140 s of audio at once: 17 GB allocated on the Apple GPU
-  # (not yet measured on CUDA). 20, not 24: a "24 GB" card reports 23.99, an L4 22.5, and the cloud places 22 and 20.
-  - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cuda, platforms: [linux/amd64], minimum_vram_gb: 20}
+  # PE-A-Frame large is 5.8 GB of float32 weights, run on up to 140 s of audio at once: 17 GB allocated on the Apple GPU.
+  # On CUDA (an RTX 3090, 2026-10-31), with expandable segments (node.py), the whole step peaks at 17.1 GB for a 92 s
+  # soundtrack and 22.1 GB for 131.8 s, about 0.13 GB a second more: 23.1 GB for a 140 s piece. 23, not 24: a "24 GB"
+  # card reports 24 or 23.99; an L4 (22.5) and a 20 GB card are too small
+  - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cuda, platforms: [linux/amd64], minimum_vram_gb: 23}
   - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cpu}
 # on the CPU, 28 GB resident at most (PE-A-Frame on nes-biking's 137.6 s soundtrack, one piece)
 resources: {cpu_cores: 4, memory_gb: 28, gpu: exclusive}

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/audio/sounds/node.py

@@ -18,9 +18,15 @@ Licences: PANNs code MIT and weights CC-BY-4.0, EfficientAT code and weights MIT
 are CC-BY-4.0), PE-A-Frame Apache-2.0: commercial-ok, as PLAN.md §4 lists them.
 """
 
+import os
 import time
 
-import numpy as np
+# PE-A-Frame's largest allocation (7.38 GiB on a 92 s soundtrack) failed on a 24 GB card with 6.7 GiB free and 6.7 GiB
+# more reserved by PyTorch but unused: the caching allocator's blocks, split by the pass's own earlier tensors, could not
+# hold it (a RunPod pod, 2026-10-31). Expandable segments map that memory as one; set before torch is first imported
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import numpy as np  # noqa: E402
 
 from perception_node import Node, NodeError
 from models import PEAFRAME, PEAFRAME_SR, TAGGER_SR, WEIGHTS, device, load_mn40, load_panns, load_peaframe, run_mn40, run_panns, run_peaframe, sync

{hyperstudy_perception-0.1.3 → hyperstudy_perception-0.1.5}/src/perception/_pipeline/nodes/vision/objects/haiku.py

@@ -44,12 +44,7 @@ VOCAB_SCHEMA = {
     "additionalProperties": False,
 }
 
-VOCAB_INSTRUCTIONS = f"""These are three frames from one shot of a video. List up to {MAX_OBJECTS} concrete \
-physical objects that are clearly visible in the frames and could be outlined and tracked with a \
-segmentation mask.
-
-Rules:
-- Generic nouns a detector would understand: 1-2 words, singular, lower case, the plain name of \
+VOCAB_RULES = """- Generic nouns a detector would understand: 1-2 words, singular, lower case, the plain name of \
 the kind of thing (e.g. "bicycle", "ramp", "mug", "car", "helmet", "chair"), not a description of \
 this one ("helmet", not "full-face helmet"; "cup", not "white paper cup").
 - Never people, faces, hands or other body parts, and never hair, wigs or beards.
@@ -61,7 +56,14 @@ separately. Other things worn or held are objects (e.g. "helmet", "glove", "earr
 "watch", "bag", "microphone").
 - On-screen text and logos are not objects.
 - If an object of the same kind was named in an earlier shot, reuse exactly that phrase.
-- Most prominent first. Return an empty list if there is nothing that fits."""
+- Most prominent first. Return an empty list if there is nothing that fits."""   # also given to the captions call that names the objects (vlm.shot-captions-objects)
+
+VOCAB_INSTRUCTIONS = f"""These are three frames from one shot of a video. List up to {MAX_OBJECTS} concrete \
+physical objects that are clearly visible in the frames and could be outlined and tracked with a \
+segmentation mask.
+
+Rules:
+{VOCAB_RULES}"""
 
 NAMES_SCHEMA = {
     "type": "object",
@@ -156,7 +158,7 @@ def reply_json(response):
 
 def fan_out(fn, items, workers: int | None = None) -> list:
     """[fn(x) for x in items] on `workers` threads (WORKERS when not given), in order, raising the first failure:
-    prosody-captions' style_captions.fan_out (a node's code is its own folder's, so it is not imported from there). At
+    prosody-captions' style_captions.fan_out (a copy: that node's runner does not share this folder, Runner.shares). At
     most `workers` calls are submitted at a time, the next only once one has finished, so after a failure no call
     starts: those running finish (each billed call reports its spend itself) and the failure is raised. The results are
     in the items' order whatever order the calls finish in."""