hyperstudy-perception 0.1.2__tar.gz → 0.1.4__tar.gz

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hyperstudy-perception
-Version: 0.1.2
+Version: 0.1.4
 Summary: Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers
 Project-URL: Homepage, https://github.com/hyperstudyio/hyperstudy-perception
 Project-URL: Changelog, https://github.com/hyperstudyio/hyperstudy-perception/releases

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hyperstudy-perception"
-version = "0.1.2"
+version = "0.1.4"
 description = "Video perception pipeline: shots, speech, faces and captions from a video, as a versioned track bundle, run locally, on cloud GPUs or lab servers"
 readme = "README.md"
 license = "MIT"

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/RELEASE.json

@@ -1,5 +1,5 @@
 {
-"commit": "6301eca468f046c39fbf4a84f05e106ce6b12688",
+"commit": "7a657d9319167d9fb6eb8374260e9d74d72365b2",
 "files": [
 "cloud/runpod/guard.sh",
 "cloud/runpod/image/Dockerfile",
@@ -65,6 +65,7 @@
 "nodes/audio/prosody-captions/manifest.yaml",
 "nodes/audio/prosody-captions/node.py",
 "nodes/audio/prosody-captions/style_captions.py",
+"nodes/audio/prosody-captions/test_prosody_captions_node.py",
 "nodes/audio/prosody-captions/test_style_captions.py",
 "nodes/audio/prosody/manifest.yaml",
 "nodes/audio/prosody/node.py",

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/cloud/runpod/image/Dockerfile

@@ -8,10 +8,11 @@
 FROM runpod/pytorch:2.1.0-py3.10-cuda11.8.0-devel-ubuntu22.04@sha256:2594f25beeef751aeebcc207ba4c45a382ba174655f01e15cbbfad0e2bdb5ef8
 ENV DEBIAN_FRONTEND=noninteractive
 
-# what nodes and the driver use (setup then installs nothing); sshd takes keys only. Each pod makes its own host keys
-# at start (start.sh): the ones in the base image are public, and are never used
+# what nodes and the driver use (setup then installs nothing): ffmpeg, rsync, and the EGL and GLES libraries MediaPipe
+# loads even on the CPU (vision.facing failed for want of libEGL.so.1 on a pod, 2026-10-31). sshd takes keys only.
+# Each pod makes its own host keys at start (start.sh): the ones in the base image are public, and are never used
 RUN apt-get update \
- && apt-get install -y --no-install-recommends ffmpeg rsync \
+ && apt-get install -y --no-install-recommends ffmpeg rsync libegl1 libgles2 \
  && rm -rf /var/lib/apt/lists/* /etc/ssh/ssh_host_* \
  && mkdir -p /run/sshd /etc/ssh/sshd_config.d \
  && printf 'PermitRootLogin prohibit-password\nPasswordAuthentication no\nKbdInteractiveAuthentication no\n' \

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/nodes/audio/prosody-captions/node.py

@@ -80,28 +80,25 @@ with Node() as node:
         u = units[k]
         i, j = u["samples"]
         text = sc.prompt(units, k)
-        r = call(lambda: client.chat.completions.create(
-            model=sc.GPT_MODEL, modalities=["text"],
-            tools=[{"type": "function", "function": {"name": "record_style", "description": "Record how the target is said",
-                                                     "parameters": sc.CAPTION_SCHEMA, "strict": True}}],
-            tool_choice={"type": "function", "function": {"name": "record_style"}},
-            messages=[{"role": "system", "content": sc.GPT_INSTRUCTIONS},
-                      {"role": "user", "content": [{"type": "text", "text": text},
-                                                   {"type": "input_audio", "input_audio": {"data": wav_b64(pcm[2 * i:2 * j]), "format": "wav"}}]}]))
-        d = r.usage.prompt_tokens_details
-        usage = sc.cost(sc.GPT_MODEL, r.usage.prompt_tokens, d.audio_tokens if d else 0, r.usage.completion_tokens)
-        spend(usage)
-        choice = r.choices[0]
-        if not choice.message.tool_calls:
-            raise NodeError("internal", f"{u['id']}: no function call in the reply (finish_reason {choice.finish_reason})")
-        arguments = choice.message.tool_calls[0].function.arguments
-        try:
-            parsed = sc.parse_caption(arguments)
-        except ValueError as e:
-            raise NodeError("internal", f"{u['id']}: {e}") from None
-        return {"parsed": parsed, "usage": usage, "prompt": text,
-                "raw": {"model": r.model, "finish_reason": choice.finish_reason, "arguments": arguments,
-                        "usage": r.usage.model_dump(mode="json")}}
+
+        def ask():
+            r = call(lambda: client.chat.completions.create(
+                model=sc.GPT_MODEL, modalities=["text"],
+                tools=[{"type": "function", "function": {"name": "record_style", "description": "Record how the target is said",
+                                                         "parameters": sc.CAPTION_SCHEMA, "strict": True}}],
+                tool_choice={"type": "function", "function": {"name": "record_style"}},
+                messages=[{"role": "system", "content": sc.GPT_INSTRUCTIONS},
+                          {"role": "user", "content": [{"type": "text", "text": text},
+                                                       {"type": "input_audio", "input_audio": {"data": wav_b64(pcm[2 * i:2 * j]), "format": "wav"}}]}]))
+            d = r.usage.prompt_tokens_details
+            usage = sc.cost(sc.GPT_MODEL, r.usage.prompt_tokens, d.audio_tokens if d else 0, r.usage.completion_tokens)
+            spend(usage)
+            choice = r.choices[0]
+            arguments = choice.message.tool_calls[0].function.arguments if choice.message.tool_calls else None
+            return arguments, usage, {"model": r.model, "finish_reason": choice.finish_reason, "arguments": arguments,
+                                      "usage": r.usage.model_dump(mode="json")}
+        got = sc.caption_attempt(ask)
+        return {**got, "prompt": text, "ask": ask, "asked_again": []}
 
     def tag(description: str) -> dict:
         r = call(lambda: client.chat.completions.create(
@@ -125,8 +122,26 @@ with Node() as node:
     captions = sc.fan_out(caption, range(len(units)))   # on a failure, no further call starts
     caption_s = round(time.time() - t0, 1)
     t0 = time.time()
-    tagged = sc.fan_out(lambda c: tag(c["parsed"]["description"]), captions)
+    good = [k for k, c in enumerate(captions) if "rejected" not in c]
+    tagged = dict(zip(good, sc.fan_out(lambda k: tag(captions[k]["parsed"]["description"]), good)))
     tagger_s = round(time.time() - t0, 1)
+    # a reply that was not the schema's, asked again once every other call is made: then what the step spent is all but
+    # this second call and its tag, which must stay within its reservation (style_captions.may_ask_again)
+    for k, c in enumerate(captions):
+        if "rejected" not in c:
+            continue
+        tag_usd = max((t["usage"]["usd"] for t in tagged.values()), default=c["usage"]["usd"])
+        if not sc.may_ask_again(totals["usd"], c["usage"]["usd"], tag_usd, len(units)):
+            raise NodeError("internal", f"{units[k]['id']}: not asked again (the step's reservation would not cover it): "
+                                        f"{sc.rejected_text(c['rejected'])}")
+        try:
+            captions[k] = {**c, **sc.asked_again(c, sc.caption_attempt(c["ask"]))}
+        except ValueError as e:
+            raise NodeError("internal", f"{units[k]['id']}: {e}") from None
+        tagged[k] = tag(captions[k]["parsed"]["description"])
+    tagged = [tagged[k] for k in range(len(units))]
+    for c in captions:
+        c["outside"] = sc.outside_tags(c["arguments"])
 
     for m in prosody["producer"]["models"]:   # the measures' and emotion's models, as the prosody node recorded them
         node.model(m["id"], licence_mode=m["licence_mode"], source=m.get("source"), revision=m.get("revision"), licence=m.get("licence"))
@@ -141,6 +156,11 @@ with Node() as node:
     node.write_json("prosody", "prosody.json", sc.refine(
         prosody, entries, node.producer(timings={"caption_s": caption_s, "tagger_s": tagger_s}, device="OpenAI API"),
         {"file": captions_raw.FILE, "format": "parquet", "description": captions_raw.DESCRIPTION}))
-    node.usage(utterances=len(units), workers=sc.WORKERS,
+    # tags the audio model gave outside the list, dropped from their captions: utterance -> tags
+    outside = {u["id"]: c["outside"] for u, c in zip(units, captions) if c["outside"]}
+    # replies that were not the schema's JSON, asked again once (style_captions.caption_attempt, asked_again): utterance -> the first
+    # reply as it came (its problem, finish_reason and arguments), to find out why
+    asked_again = {u["id"]: c["asked_again"][0] for u, c in zip(units, captions) if c["asked_again"]}
+    node.usage(utterances=len(units), workers=sc.WORKERS, tags_outside_list=outside, asked_again=asked_again,
                usd_by_model={sc.GPT_MODEL: round(sum(c["usage"]["usd"] for c in captions), 4),
                              sc.TAGGER: round(sum(t["usage"]["usd"] for t in tagged), 4)})

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/nodes/audio/prosody-captions/style_captions.py

@@ -136,14 +136,68 @@ def cost(model: str, prompt_tokens: int, audio_tokens: int | None, completion_to
     return {"text_in": text_in, "audio_in": audio_in, "out": completion_tokens, "usd": usd}
 
 
+def caption_attempt(ask) -> dict:
+    """One call for one target's caption: `ask()` makes it and gives (its function call's arguments, or None when it
+    made none; its usage, as cost() gives it; its raw reply). {parsed, arguments, raw, usage}, or {rejected: {problem,
+    finish_reason, arguments}, usage} when the reply is not the schema's. Why one can fail the schema is known: the
+    audio models do not constrain their function call to it (strict or not; the same reason off-list tags are dropped,
+    parse_caption), so a sample can be malformed (one was cut short of a comma on a pod's run, 2026-10-31)."""
+    arguments, usage, raw = ask()
+    try:
+        if arguments is None:
+            raise ValueError(f"no function call in the reply (finish_reason {raw.get('finish_reason')})")
+        return {"parsed": parse_caption(arguments), "arguments": arguments, "raw": raw, "usage": usage}
+    except ValueError as e:
+        return {"rejected": {"problem": str(e), "finish_reason": raw.get("finish_reason"), "arguments": arguments},
+                "usage": usage}
+
+
+def rejected_text(rejected: dict) -> str:
+    return (f"{rejected['problem']} (finish_reason {rejected['finish_reason']}, "
+            f"arguments {str(rejected['arguments'])[:300]!r})")
+
+
+def may_ask_again(spent_usd: float, caption_usd: float, tag_usd: float, utterances: int) -> bool:
+    """Whether a rejected caption may be asked again: once every other call of the step has been made (spent_usd is
+    then all but this), only if this caption's second call and its tag stay within what the step reserved (the
+    manifest's cost, an utterance: review of #53)."""
+    return spent_usd + caption_usd + tag_usd <= utterances * COST_PER_UTTERANCE
+
+
+def asked_again(first: dict, second: dict) -> dict:
+    """A caption asked a second time: the second's, with both calls' usage and the first's rejected reply
+    (asked_again). ValueError when the second is not the schema's either."""
+    if "rejected" in second:
+        raise ValueError(f"asked twice, and neither reply was the schema's: {rejected_text(first['rejected'])}; "
+                         f"{rejected_text(second['rejected'])}")
+    usage = {k: first["usage"][k] + second["usage"][k] for k in first["usage"]}
+    return {**second, "usage": usage, "asked_again": [first["rejected"]]}
+
+
+# what the step reserves an utterance (manifest.yaml's cost; a test checks they agree): one caption call and its tags
+COST_PER_UTTERANCE = 0.008
+
+
 def parse_caption(arguments: str) -> dict:
-    """The function call's arguments -> {description, tags, emphasized_words}; ValueError when they are not that."""
+    """The function call's arguments -> {description, tags, emphasized_words}; ValueError when they are not that. Its tags
+    are those of the list only: gpt-audio-1.5 does not hold a function call to the schema's enum (strict or not), so a tag
+    outside it is dropped here (outside_tags says which; the side file keeps the reply as it came)."""
     import json
     out = json.loads(arguments)
     if not (isinstance(out, dict) and set(out) == {"description", "tags", "emphasized_words"}):
         raise ValueError(f"the caption is not {{description, tags, emphasized_words}}: {sorted(out) if isinstance(out, dict) else type(out).__name__}")
-    check_tags(out["tags"])
-    return out
+    if not isinstance(out["tags"], list):
+        check_tags(out["tags"])   # not a list: said as such
+    if not all(isinstance(t, str) for t in out["tags"]):
+        raise ValueError(f"the caption's tags are not strings: {[t for t in out['tags'] if not isinstance(t, str)][:5]}")
+    return {**out, "tags": [t for t in out["tags"] if t in ALL_TAGS]}
+
+
+def outside_tags(arguments: str) -> list[str]:
+    """The tags a caption's reply gave that are not in the list (dropped by parse_caption)."""
+    import json
+    tags = json.loads(arguments).get("tags")
+    return [t for t in tags if t not in ALL_TAGS] if isinstance(tags, list) else []
 
 
 def parse_tags(content: str) -> list[str]:

hyperstudy_perception-0.1.4/src/perception/_pipeline/nodes/audio/prosody-captions/test_prosody_captions_node.py

@@ -0,0 +1,107 @@
+"""audio.prosody-captions run as the coordinator runs it, with OpenAI stood in for: every caption made once, then the
+tags, then a reply that was not the schema's asked again once, only within what the step reserved (review of #53).
+The stand-in answers by what it is asked (the target's line, from the prompt), never by when.
+Run: uv run --no-project --with pytest --with numpy --with-editable ./sdk pytest -q nodes/audio/prosody-captions"""
+
+import json
+import runpy
+import sys
+import threading
+import types
+import wave
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+import style_captions as sc
+from test_style_captions import N, PROSODY, TRANSCRIPT
+
+HERE = Path(__file__).resolve().parent
+GOOD = {"description": "Calm and even.", "tags": ["calm"], "emphasized_words": []}
+
+
+class OpenAI:
+    """The client: a caption (the first one for `bad`'s line cut short, as gpt-audio's was on a pod), or tags. calls:
+    every request, in order; usd: what each call is billed (prompt tokens, at the models' prices)."""
+    calls: list = []
+    bad: set = set()
+    prompt_tokens = 100
+    lock = threading.Lock()
+
+    def __init__(self, api_key):
+        assert api_key == "key"
+        self.chat = SimpleNamespace(completions=self)
+
+    def create(self, **request):
+        usage = SimpleNamespace(prompt_tokens=self.prompt_tokens, completion_tokens=20,
+                                prompt_tokens_details=SimpleNamespace(audio_tokens=0), model_dump=lambda mode: {})
+        if "tools" in request:
+            text = request["messages"][1]["content"][0]["text"]
+            target = text.split('Target transcript (automatic, may contain errors): "')[1].split('"')[0]
+            with self.lock:
+                self.calls.append(request)
+                asked = sum(1 for c in self.calls if "tools" in c and c["messages"][1]["content"][0]["text"] == text)
+            arguments = '{"description": "Calm" "tags": []}' if target in self.bad and asked == 1 else json.dumps(GOOD)
+            message = SimpleNamespace(tool_calls=[SimpleNamespace(function=SimpleNamespace(arguments=arguments))])
+            return SimpleNamespace(model=request["model"], usage=usage,
+                                   choices=[SimpleNamespace(message=message, finish_reason="tool_calls")])
+        with self.lock:
+            self.calls.append(request)
+        return SimpleNamespace(model=request["model"], usage=usage,
+                               choices=[SimpleNamespace(message=SimpleNamespace(content=json.dumps({"tags": ["calm"]})),
+                                                        finish_reason="stop")])
+
+
+@pytest.fixture
+def openai(monkeypatch):
+    module = types.ModuleType("openai")
+    module.OpenAI = OpenAI
+    for name in ("RateLimitError", "InternalServerError", "APIConnectionError", "AuthenticationError"):
+        setattr(module, name, type(name, (Exception,), {}))
+    monkeypatch.setitem(sys.modules, "openai", module)
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    OpenAI.calls, OpenAI.bad, OpenAI.prompt_tokens = [], set(), 100
+    return OpenAI
+
+
+def run(tmp_path, monkeypatch) -> dict:
+    audio = tmp_path / "audio.wav"
+    with wave.open(str(audio), "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(sc.RATE)
+        w.writeframes(b"\0\0" * N)
+    (tmp_path / "transcript.json").write_text(json.dumps(TRANSCRIPT))
+    (tmp_path / "prosody.json").write_text(json.dumps(PROSODY))
+    request = tmp_path / "request.json"
+    request.write_text(json.dumps({"protocol": 1, "out_dir": str(tmp_path / "out"), "accelerator": "cpu", "config": {},
+                                   "secrets": ["OPENAI_API_KEY"], "producer": {"stage": "prosody", "stage_version": "1.0.0"},
+                                   "inputs": {"audio": {"path": str(audio)}, "transcript": {"path": str(tmp_path / "transcript.json")},
+                                              "prosody": {"path": str(tmp_path / "prosody.json")}}}))
+    monkeypatch.setattr(sys, "argv", ["node.py", "--request", str(request)])
+    try:
+        runpy.run_path(str(HERE / "node.py"), run_name="__main__")
+    except SystemExit:
+        pass
+    return json.loads((tmp_path / "response.json").read_text())
+
+
+def test_a_reply_that_is_not_the_schemas_is_asked_again_after_every_other_call(tmp_path, monkeypatch, openai):
+    openai.bad = {"orange lamp."}   # utt:3's line
+    got = run(tmp_path, monkeypatch)
+    assert got["ok"], got.get("error")
+    kinds = ["caption" if "tools" in c else "tag" for c in openai.calls]
+    # three captions, two tags (the good ones), then utt:3 asked again and tagged
+    assert kinds == ["caption"] * 3 + ["tag"] * 2 + ["caption", "tag"]
+    first = got["usage"]["asked_again"]["utt:3"]
+    assert "delimiter" in first["problem"] and first["arguments"] == '{"description": "Calm" "tags": []}'
+    assert got["usage"]["calls"] == 7
+
+
+def test_a_reply_is_not_asked_again_past_the_steps_reservation(tmp_path, monkeypatch, openai):
+    openai.bad = {"orange lamp."}
+    openai.prompt_tokens = 4000   # each call near the utterance's reservation: a second caption would pass it
+    got = run(tmp_path, monkeypatch)
+    assert not got["ok"] and "not asked again" in got["error"]["message"] and "utt:3" in got["error"]["message"]
+    assert ["caption" if "tools" in c else "tag" for c in openai.calls] == ["caption"] * 3 + ["tag"] * 2

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/nodes/audio/prosody-captions/test_style_captions.py

@@ -68,8 +68,15 @@ def test_cost_is_the_bakeoffs_at_its_prices():
 def test_replies_must_be_the_schemas_with_tags_from_the_list():
     ok = {"description": "Calm.", "tags": ["calm", "falling-end"], "emphasized_words": []}
     assert sc.parse_caption(json.dumps(ok)) == ok
-    with pytest.raises(ValueError, match="outside the vocabulary"):
-        sc.parse_caption(json.dumps({**ok, "tags": ["calm", "grumpy"]}))
+    # gpt-audio-1.5 does not hold its function call to the schema's tag list (strict or not: "friendly" on an OOO
+    # held-out clip, 2026-10-30): a tag outside it is dropped from that caption and said, not the step's failure
+    assert sc.parse_caption(json.dumps({**ok, "tags": ["calm", "grumpy", "falling-end"]})) == ok
+    assert sc.outside_tags(json.dumps({**ok, "tags": ["calm", "grumpy", "falling-end"]})) == ["grumpy"]
+    assert sc.outside_tags(json.dumps(ok)) == []
+    with pytest.raises(ValueError, match="not strings"):   # not a tag at all: the reply is not the schema's
+        sc.parse_caption(json.dumps({**ok, "tags": ["calm", 3, None]}))
+    with pytest.raises(ValueError, match="outside the vocabulary"):   # the tagger's json_schema holds it to the list
+        sc.parse_tags('{"tags": ["neutral", "grumpy"]}')
     with pytest.raises(ValueError, match="not {description, tags, emphasized_words}"):
         sc.parse_caption(json.dumps({"description": "x"}))
     assert sc.parse_tags('{"tags": ["neutral"]}') == ["neutral"]
@@ -155,3 +162,49 @@ def test_the_fan_out_keeps_the_items_order():
     client = FailingClient(-1)
     got = sc.fan_out(lambda i: client.chat.completions.create(utterance=i)["utterance"], range(50), workers=4)
     assert got == list(range(50)) and client.calls == 50
+
+
+GOOD = json.dumps({"description": "Calm.", "tags": ["calm"], "emphasized_words": []})
+
+
+def asker(*replies):
+    """One call per reply given, in order: (arguments or None, usage, raw), as node.py's ask makes them."""
+    calls = iter(replies)
+
+    def ask():
+        arguments = next(calls)
+        return arguments, {"text_in": 10, "audio_in": 5, "out": 3, "usd": 0.01}, {"finish_reason": "tool_calls", "arguments": arguments}
+    return ask
+
+
+def test_a_caption_reply_that_is_not_the_schemas_json_is_rejected_whole_and_a_second_one_taken():
+    """gpt-audio does not constrain its function call to the schema: a reply that is not its JSON (a pod's run,
+    2026-10-31: "Expecting ',' delimiter") is kept as it came, and a second reply, if asked, is the caption, both calls
+    billed."""
+    calls = asker('{"description": "Calm" "tags": []}', GOOD)
+    first = sc.caption_attempt(calls)
+    assert "delimiter" in first["rejected"]["problem"] and first["rejected"]["arguments"] == '{"description": "Calm" "tags": []}'
+    assert first["rejected"]["finish_reason"] == "tool_calls"
+    got = sc.asked_again(first, sc.caption_attempt(calls))
+    assert got["parsed"]["description"] == "Calm." and got["arguments"] == GOOD and got["asked_again"] == [first["rejected"]]
+    assert got["usage"] == {"text_in": 20, "audio_in": 10, "out": 6, "usd": 0.02}
+    assert "rejected" not in sc.caption_attempt(asker(GOOD))
+
+
+def test_a_second_reply_that_is_not_the_schemas_fails_the_caption():
+    calls = asker("{", None)
+    first = sc.caption_attempt(calls)
+    with pytest.raises(ValueError, match="asked twice"):
+        sc.asked_again(first, sc.caption_attempt(calls))
+
+
+def test_a_reply_is_asked_again_only_within_what_the_step_reserved():
+    """Review of #53: asked once every other call is made, so what the step has spent is all but this caption's second
+    call and its tag, which must stay within the step's reservation (COST_PER_UTTERANCE an utterance)."""
+    assert sc.may_ask_again(0.07, 0.005, 0.001, 10)          # 0.076 <= 0.08
+    assert not sc.may_ask_again(0.076, 0.005, 0.001, 10)     # 0.082 > 0.08
+
+
+def test_what_an_utterance_is_reserved_is_the_manifests():
+    lines = (Path(sc.__file__).parent / "manifest.yaml").read_text().splitlines()
+    assert f"cost: {{usd: {sc.COST_PER_UTTERANCE}, per: utterance, input: transcript}}" in lines

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/nodes/audio/sounds/manifest.yaml

@@ -16,9 +16,11 @@ config:
 runners:
   - {kind: native, env: envs/audio-events, entry: node.py, accelerator: mps, platforms: [darwin/arm64]}
   # PyPI's torch 2.8 wheels for Linux carry CUDA 12.8 (envs/audio-events); the machine needs an NVIDIA driver >= 570.
-  # PE-A-Frame large is 5.8 GB of float32 weights, run on up to 140 s of audio at once: 17 GB allocated on the Apple GPU
-  # (not yet measured on CUDA). 20, not 24: a "24 GB" card reports 23.99, an L4 22.5, and the cloud places 22 and 20.
-  - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cuda, platforms: [linux/amd64], minimum_vram_gb: 20}
+  # PE-A-Frame large is 5.8 GB of float32 weights, run on up to 140 s of audio at once: 17 GB allocated on the Apple GPU.
+  # On CUDA (an RTX 3090, 2026-10-31), with expandable segments (node.py), the whole step peaks at 17.1 GB for a 92 s
+  # soundtrack and 22.1 GB for 131.8 s, about 0.13 GB a second more: 23.1 GB for a 140 s piece. 23, not 24: a "24 GB"
+  # card reports 24 or 23.99; an L4 (22.5) and a 20 GB card are too small
+  - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cuda, platforms: [linux/amd64], minimum_vram_gb: 23}
   - {kind: native, env: envs/audio-events, entry: node.py, accelerator: cpu}
 # on the CPU, 28 GB resident at most (PE-A-Frame on nes-biking's 137.6 s soundtrack, one piece)
 resources: {cpu_cores: 4, memory_gb: 28, gpu: exclusive}

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/_pipeline/nodes/audio/sounds/node.py

@@ -18,9 +18,15 @@ Licences: PANNs code MIT and weights CC-BY-4.0, EfficientAT code and weights MIT
 are CC-BY-4.0), PE-A-Frame Apache-2.0: commercial-ok, as PLAN.md §4 lists them.
 """
 
+import os
 import time
 
-import numpy as np
+# PE-A-Frame's largest allocation (7.38 GiB on a 92 s soundtrack) failed on a 24 GB card with 6.7 GiB free and 6.7 GiB
+# more reserved by PyTorch but unused: the caching allocator's blocks, split by the pass's own earlier tensors, could not
+# hold it (a RunPod pod, 2026-10-31). Expandable segments map that memory as one; set before torch is first imported
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import numpy as np  # noqa: E402
 
 from perception_node import Node, NodeError
 from models import PEAFRAME, PEAFRAME_SR, TAGGER_SR, WEIGHTS, device, load_mn40, load_panns, load_peaframe, run_mn40, run_panns, run_peaframe, sync

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/backends/base.py

@@ -330,9 +330,15 @@ def child_env(requested: list[str], secrets: dict[str, str]) -> dict[str, str]:
 # once, not a download it has no token for. Nor does it load a token stored on the machine (`hf auth login`'s, or
 # HF_TOKEN_PATH's), which huggingface_hub otherwise does even offline
 OFFLINE = {"HF_HUB_OFFLINE": "1", "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1"}
-# a prepare on another machine: the token it was sent, or none, never the machine's own login (its HF_TOKEN in a
-# profile, its stored token): access to a gated model is a person's. A prepare that prepare_first runs has it from OFFLINE
-SENT_TOKEN_ONLY = {"HF_HUB_DISABLE_IMPLICIT_TOKEN": "1"}
+# A prepare is online, with the token it was sent as HF_TOKEN: access to a gated model is a person's. It runs without
+# HF_HUB_DISABLE_IMPLICIT_TOKEN (PREPARE_UNSET), and only when it was sent one: transformers 5.20 fetches a model's
+# config without the token it is given when implicit tokens are off (SAM 3's prepare got a 401 on liquidswords2,
+# 2026-10-30). The implicit token must then be the one it was sent: huggingface_hub (2.3) takes an OIDC identity first
+# (OIDC_LOGINS, unset), then HF_TOKEN (which replaces any other), then a stored token (its path NO_STORED_TOKEN)
+PREPARE_UNSET = ("HF_HUB_OFFLINE", "HF_HUB_DISABLE_IMPLICIT_TOKEN")
+OIDC_LOGINS = ("HF_OIDC_RESOURCE", "HF_OIDC_ID_TOKEN")
+# where a prepare looks for a stored token: a path that cannot exist (below /dev/null)
+NO_STORED_TOKEN = "/dev/null/no-stored-token"
 
 
 def hf_cache_env(home: str) -> dict[str, str]:

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/backends/cloud.py

@@ -143,9 +143,9 @@ class RunPodBackend(CloudBackend):
     guard = REPO / "cloud" / "runpod" / "guard.sh"
     # perception's own image (cloud/runpod/image): RunPod's cached PyTorch image with ffmpeg, rsync, uv and runpodctl 3
     # added; software only. Its tag is tag.py's, from what goes into it (a test checks they agree)
-    image_tag = "e92ef7a71b30"
+    image_tag = "efba77cf5632"
     # and the digest of the build that was measured: a rebuild under the same tag (apt's versions move) is never pulled
-    image_digest = "sha256:09e6eb03f0ab44353e13b198e9f2b114b07c29af5613b33da42e2e122b7a74c4"
+    image_digest = "sha256:472c7316c3bb200d88948b43acbb4afd054f9d56621f869d700a6680884e234a"
     # under the repository's owner, where .github/workflows/runpod-image.yml pushes it (a test checks)
     image = f"ghcr.io/hyperstudyio/perception-runpod:{image_tag}@{image_digest}"
     start = "/perception-start.sh"   # its own start (host keys, PUBLIC_KEY, sshd), run after the pod's removal is set
@@ -158,6 +158,14 @@ class RunPodBackend(CloudBackend):
             "NVIDIA RTX PRO 4500 Blackwell": 0.72, "NVIDIA L40": 0.82,
             "NVIDIA GeForce RTX 4090": 0.89, "NVIDIA RTX PRO 5000 Blackwell": 0.96, "NVIDIA RTX 6000 Ada Generation": 0.99,
             "NVIDIA L40S": 1.09, "NVIDIA GeForce RTX 5090": 1.19}
+    # each one's memory as nvidia-smi reports it (GB; the probe's memory_gb), against the steps' minimum_vram_gb: a "24 GB"
+    # card reports 24 or 23.99, an L4 22.5, a "48 GB" A40, L40 or L40S 45
+    gpu_vram_gb = {"NVIDIA RTX A4500": 20, "NVIDIA RTX A5000": 24, "NVIDIA RTX 4000 Ada Generation": 20,
+                   "NVIDIA GeForce RTX 3090": 24, "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition": 95.6,
+                   "NVIDIA RTX PRO 4000 Blackwell": 24, "NVIDIA A40": 45, "NVIDIA L4": 22.5, "NVIDIA RTX A6000": 48,
+                   "NVIDIA RTX PRO 4500 Blackwell": 32, "NVIDIA L40": 45, "NVIDIA GeForce RTX 4090": 24,
+                   "NVIDIA RTX PRO 5000 Blackwell": 48, "NVIDIA RTX 6000 Ada Generation": 48, "NVIDIA L40S": 45,
+                   "NVIDIA GeForce RTX 5090": 31.8}
     # datacenters in the US, Canada and Europe, whichever has a GPU free: those RunPod's REST API accepts (its schema's
     # enum, 2026-10-25; gpus above are in its enum too)
     datacenters = ["US-CA-2", "US-DE-1", "US-GA-1", "US-GA-2", "US-IL-1", "US-KS-2", "US-KS-3", "US-MD-1", "US-MO-1", "US-MO-2",
@@ -216,6 +224,18 @@ class Cloud:
     def runpod(self) -> bool:
         return self.read().get("runpod", False)
 
+    @property
+    def runpod_keys(self) -> bool:
+        """Whether a whole run on a RunPod pod (pod_run.py) may be given the keys its steps declare: the user's explicit
+        yes (`perception cloud keys runpod on`), off until then. Per-step pods are never given any."""
+        return self.read().get("runpod_keys", False) is True
+
+    def set_keys(self, allowed: bool):
+        doc = self.read()
+        doc["runpod_keys"] = allowed
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(doc))
+
     def set(self, enabled: bool, provider: str = "aws"):
         doc = self.read()
         doc["enabled" if provider == "aws" else provider] = enabled

{hyperstudy_perception-0.1.2 → hyperstudy_perception-0.1.4}/src/perception/backends/native.py

@@ -20,7 +20,7 @@ from pathlib import Path
 
 from ..manifests import Manifest, Runner
 from ..store import Artifact
-from .base import GPU_LEASES, LEASE_FOLDER, OFFLINE, Backend, Capabilities, LeaseRefused, gpu_env, holder, lease_command
+from .base import GPU_LEASES, LEASE_FOLDER, OFFLINE, Backend, Capabilities, LeaseRefused, SubmitRefused, gpu_env, holder, lease_command
 from .probe import probe
 
 from ..release import REPO, env_vars
@@ -88,6 +88,11 @@ class NativeBackend(Backend):
             command = [*lease_command(str(self.lease_folder()), gpu, manifest.resources.gpu,
                                       holder(json.loads(request.read_text()))), *command]
         if prepare_secrets is not None and manifest.download_secrets and self.downloads(manifest, runner):
+            if missing := [k for k in manifest.download_secrets if not (prepare_secrets.get(k) or "").strip()]:
+                # refused, not prepared with whatever login this machine has: the plan says so first (missing_secrets),
+                # but a resumed run is not planned again (as a host's submit, remote.py)
+                raise SubmitRefused("input", f"its weights are downloaded with {', '.join(missing)}, which is not set here "
+                                             f"(perception keys set {missing[0]})")
             # its prepare first, with them, then the node (with its lease: no card is held while weights download)
             # without them (prepare_first.py)
             from .. import prepare_first