hyperstudy-perception 0.1.0__py3-none-any.whl

perception/_pipeline/graphs/decode.yaml

@@ -0,0 +1,11 @@
+# Decode only: the smallest real graph, used for backend smoke tests (cloud/skypilot/smoke.yaml).
+#   perception run graphs/decode.yaml --input video=PATH/TO/VIDEO.mp4
+schema_version: 1
+pipeline: perception-decode
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}   # relative to this file; usually set with --input
+policy: {placement: local, privacy: local-only, max_cost_usd: 0, retries: 1}
+nodes:
+  - id: decode
+    uses: core.decode@1.0.0
+    inputs: {video: $inputs.video}

perception/_pipeline/graphs/faces.yaml

@@ -0,0 +1,14 @@
+# Decode and face tracks: the first graph timed on an NVIDIA GPU (cloud/skypilot/gpu-faces.yaml).
+#   perception run graphs/faces.yaml --input video=PATH/TO/VIDEO.mp4
+schema_version: 1
+pipeline: perception-faces
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}   # relative to this file; usually set with --input
+policy: {placement: local, privacy: local-only, max_cost_usd: 0, retries: 1}
+nodes:
+  - id: decode
+    uses: core.decode@1.0.0
+    inputs: {video: $inputs.video}
+  - id: faces
+    uses: vision.face-track@1.0.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}

perception/_pipeline/graphs/full.yaml

@@ -0,0 +1,31 @@
+# Everything the pipeline can do: the graph the annotator app runs. The app's feature list picks what runs (--only, with
+# what it needs), settings (--set) and places (--place). Nothing leaves the machine without --allow-uploads, and then
+# only the kinds allow_upload names: frames, faces and transcript for the captions API, and video and audio for a step
+# placed on another machine (a lab server, a cloud GPU), which receives its inputs. max_cost_usd caps the spend.
+#   perception plan graphs/full.yaml --input video=PATH/TO/VIDEO.mp4
+#   perception run graphs/full.yaml --input video=PATH --only asr,faces          # the local, free features
+#   perception run graphs/full.yaml --input video=PATH --allow-uploads           # everything, captions too
+schema_version: 1
+pipeline: perception-full
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}
+policy:
+  placement: hybrid
+  privacy: local-preferred
+  allow_upload: [video, audio, frames, faces, transcript]
+  max_cost_usd: 0.50
+  retries: 1
+nodes:
+  - id: decode
+    uses: core.decode@1.0.0
+    inputs: {video: $inputs.video}
+  - id: asr
+    uses: audio.asr@1.0.0
+    inputs: {audio: $nodes.decode.audio, shots: $nodes.decode.shots}
+  - id: faces
+    uses: vision.face-track@1.0.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}
+  - id: captions
+    uses: vlm.shot-captions@1.0.0
+    inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, transcript: $nodes.asr.transcript,
+             faces: $nodes.faces.faces, media: $nodes.decode.media}

perception/_pipeline/graphs/mvp-captions.yaml

@@ -0,0 +1,27 @@
+# The Phase 1 graph plus shot captions through the Anthropic API: the first node that sends data off the machine
+# and costs money. allow_upload names exactly what may leave; max_cost_usd caps the spend.
+#   perception run graphs/mvp-captions.yaml --input video=PATH/TO/VIDEO.mp4 --allow-uploads
+schema_version: 1
+pipeline: perception-mvp-captions
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}
+policy:
+  placement: hybrid
+  privacy: local-preferred
+  allow_upload: [frames, faces, transcript]
+  max_cost_usd: 0.50
+  retries: 1
+nodes:
+  - id: decode
+    uses: core.decode@1.0.0
+    inputs: {video: $inputs.video}
+  - id: asr
+    uses: audio.asr@1.0.0
+    inputs: {audio: $nodes.decode.audio, shots: $nodes.decode.shots}
+  - id: faces
+    uses: vision.face-track@1.0.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}
+  - id: captions
+    uses: vlm.shot-captions@1.0.0
+    inputs: {shots: $nodes.decode.shots, keyframes: $nodes.decode.keyframes, transcript: $nodes.asr.transcript,
+             faces: $nodes.faces.faces, media: $nodes.decode.media}

perception/_pipeline/graphs/mvp.yaml

@@ -0,0 +1,17 @@
+# Phase 1 graph: decode, face tracks and speech, published as a track bundle.
+#   perception run graphs/mvp.yaml --input video=PATH/TO/VIDEO.mp4
+schema_version: 1
+pipeline: perception-mvp
+inputs:
+  video: {type: video.v1, uri: '../data/video.mp4'}   # relative to this file; usually set with --input
+policy: {placement: local, privacy: local-only, max_cost_usd: 0, retries: 1}
+nodes:
+  - id: decode
+    uses: core.decode@1.0.0
+    inputs: {video: $inputs.video}
+  - id: asr
+    uses: audio.asr@1.0.0
+    inputs: {audio: $nodes.decode.audio, shots: $nodes.decode.shots}
+  - id: faces
+    uses: vision.face-track@1.0.0
+    inputs: {video: $inputs.video, media: $nodes.decode.media, shots: $nodes.decode.shots}

perception/_pipeline/nodes/audio/asr/manifest.yaml

@@ -0,0 +1,20 @@
+id: audio.asr
+version: 1.0.0
+stage: transcribe
+title: Speech transcript
+description: Word-timed transcript and utterances (Parakeet TDT v3); voices come from diarization
+inputs: {audio: audio.wav.v1, shots: bundle.shots@0.1}
+outputs: {transcript: bundle.transcript@0.1}
+config:
+  model: {enum: [mlx-community/parakeet-tdt-0.6b-v3], default: mlx-community/parakeet-tdt-0.6b-v3}
+runners:
+  - {kind: native, env: envs/audio, entry: node.py, accelerator: mps, platforms: [darwin/arm64]}
+  # TODO: [PERCEPTION-GRAPH] Phase 1 step 7 - a CUDA runner (NeMo nvidia/parakeet-tdt-0.6b-v3 in a pinned image)
+  # Spec: same output on linux/amd64 + CUDA; validate word times against the MLX conversion on the OOO clip.
+  # Plan: docs/plans/GRAPH_MVP_PLAN.md#phase-1--local-mvp
+resources: {cpu_cores: 2, memory_gb: 6, gpu: shared}
+# fetches the weights ahead of a run (perception setup)
+prepare: prepare.py
+models:
+  - {id: mlx-community/parakeet-tdt-0.6b-v3, source: hf://mlx-community/parakeet-tdt-0.6b-v3, licence: CC-BY-4.0,
+     licence_mode: commercial-ok, size_gb: 2.3}

perception/_pipeline/nodes/audio/asr/node.py

@@ -0,0 +1,74 @@
+"""audio.asr: words with times, grouped into utterances, as the bundle's transcript layer.
+
+Parakeet TDT 0.6B v3 (MLX, Apple silicon). Utterances split at a pause over 0.25 s, a sentence end, or a shot cut,
+as the annotator prototype did (pipeline/bakeoff/label_page.py). Voices come from the diarization node, not here:
+every word's voice is null.
+"""
+
+import time
+
+from perception_node import Node, NodeError
+from wav import read_wav
+
+PAUSE_SPLIT = 0.25            # seconds of silence between words that starts a new utterance
+SENTENCE_END = (".", "?", "!")
+
+
+def to_words(sentence) -> list[dict]:
+    """Merge Parakeet's subword tokens into words; a leading space starts a new word."""
+    words = []
+    for token in sentence.tokens:
+        if not words or token.text.startswith(" "):
+            words.append({"w": token.text.strip(), "start": token.start, "end": token.end})
+        else:
+            words[-1]["w"] += token.text
+            words[-1]["end"] = token.end
+    return [{**w, "start": round(w["start"], 3), "end": round(w["end"], 3)} for w in words if w["w"]]
+
+
+def utterances(words: list[dict], shots: list[dict]) -> list[dict]:
+    cuts = [s["start"] for s in shots[1:]]
+    units, current = [], []
+    for i, word in enumerate(words):
+        if current:
+            prev = words[current[-1]]
+            crosses_cut = any(prev["end"] <= c <= word["start"] or prev["start"] < c <= word["start"] for c in cuts)
+            if word["start"] - prev["end"] > PAUSE_SPLIT or prev["w"].endswith(SENTENCE_END) or crosses_cut:
+                units.append(current)
+                current = []
+        current.append(i)
+    if current:
+        units.append(current)
+
+    def shot_at(t):
+        return next(s["id"] for s in reversed(shots) if s["start"] <= t)
+
+    return [{"id": f"utt:{n}", "start": words[u[0]]["start"], "end": words[u[-1]]["end"],
+             "text": " ".join(words[i]["w"] for i in u), "shot": shot_at((words[u[0]]["start"] + words[u[-1]]["end"]) / 2),
+             "words": [u[0], u[-1]], "voice": None} for n, u in enumerate(units)]
+
+
+with Node() as node:
+    import mlx.core as mx
+    from parakeet_mlx import from_pretrained
+    from parakeet_mlx.audio import get_logmel
+
+    model_id = node.config["model"]
+    t0 = time.time()
+    model = from_pretrained(model_id)
+    # what model.transcribe(path) does with no chunking, from the samples read here (wav.py)
+    samples = read_wav(node.input("audio"), model.preprocessor_config.sample_rate)
+    if len(samples) < model.preprocessor_config.hop_length:
+        raise NodeError("input", f"the audio has {len(samples)} samples: too short to transcribe")
+    audio = mx.array(samples).astype(mx.float32) / 32768.0
+    result = model.generate(get_logmel(audio, model.preprocessor_config))[0]
+    seconds = round(time.time() - t0, 1)
+    node.model(model_id, licence_mode="commercial-ok", source=f"hf://{model_id}", licence="CC-BY-4.0")
+
+    words = [{**w, "voice": None} for sentence in result.sentences for w in to_words(sentence)]
+    shots = node.input_json("shots")["shots"]
+    node.write_json("transcript", "transcript.json", {
+        "layer": "transcript", "schema_version": "0.1.0",
+        "producer": node.producer(timings={"seconds": seconds}, device="mlx"),
+        "voices": [], "words": words, "utterances": utterances(words, shots)})
+    node.usage(audio_seconds=round(words[-1]["end"], 1) if words else 0, words=len(words))

perception/_pipeline/nodes/audio/asr/prepare.py

@@ -0,0 +1,12 @@
+"""Fetch the ASR node's weights ahead of a run (`perception setup`): parakeet-mlx downloads the model from Hugging Face
+the first time it is loaded, into the cache the node then reads.
+    python prepare.py CONFIG_JSON      (the node's settings, as its request would carry them)"""
+
+import json
+import sys
+
+from parakeet_mlx import from_pretrained
+
+model_id = json.loads(sys.argv[1])["model"]
+from_pretrained(model_id)
+print(f"{model_id} is in the cache")

perception/_pipeline/nodes/audio/asr/test_wav.py

@@ -0,0 +1,38 @@
+"""Decode's audio read as samples (wav.py), with nothing on PATH: parakeet-mlx's own loader runs ffmpeg, which an app
+started from the Finder does not find. Run: uv run --no-project --with pytest --with numpy --with-editable ./sdk
+pytest nodes/audio/asr"""
+
+import wave
+
+import numpy as np
+import pytest
+
+from perception_node import NodeError
+from wav import read_wav
+
+
+@pytest.fixture(autouse=True)
+def no_tools_on_path(monkeypatch):
+    monkeypatch.setenv("PATH", "")
+
+
+def write(path, samples, rate=16000, channels=1):
+    with wave.open(str(path), "wb") as w:
+        w.setnchannels(channels)
+        w.setsampwidth(2)
+        w.setframerate(rate)
+        w.writeframes(samples.astype(np.int16).tobytes())
+
+
+def test_reads_decodes_audio_as_it_was_written(tmp_path):
+    samples = (np.sin(np.arange(16000) / 5) * 12000).astype(np.int16)
+    write(tmp_path / "a.wav", samples)
+    out = read_wav(tmp_path / "a.wav", 16000)
+    assert out.dtype == np.int16 and np.array_equal(out, samples)
+
+
+@pytest.mark.parametrize("rate, channels", [(44100, 1), (16000, 2)])
+def test_refuses_audio_decode_does_not_write(tmp_path, rate, channels):
+    write(tmp_path / "a.wav", np.zeros(1000 * channels), rate, channels)
+    with pytest.raises(NodeError, match="decode writes 1 channel, 16-bit, 16000 Hz"):
+        read_wav(tmp_path / "a.wav", 16000)

perception/_pipeline/nodes/audio/asr/wav.py

@@ -0,0 +1,19 @@
+"""Decode's audio (16 kHz mono 16-bit WAV) as samples, read here rather than by parakeet-mlx: its load_audio runs the
+ffmpeg program, which an app started from the Finder does not find (its PATH is /usr/bin:/bin:/usr/sbin:/sbin)."""
+
+import wave
+from pathlib import Path
+
+import numpy as np
+
+from perception_node import NodeError
+
+
+def read_wav(path: Path, rate: int) -> np.ndarray:
+    """The samples, as int16: what `ffmpeg -i path -f s16le -ac 1 -ar rate -` printed for a WAV already in that form."""
+    with wave.open(str(path)) as w:
+        got = (w.getnchannels(), w.getsampwidth(), w.getframerate())
+        if got != (1, 2, rate):
+            raise NodeError("input", f"the audio is {got[0]} channel(s), {8 * got[1]}-bit, {got[2]} Hz; "
+                                     f"decode writes 1 channel, 16-bit, {rate} Hz")
+        return np.frombuffer(w.readframes(w.getnframes()), np.int16)

perception/_pipeline/nodes/conftest.py

@@ -0,0 +1,64 @@
+"""A small synthetic video for the node tests that read one (decode's media.py, face-track's frames.py)."""
+
+from fractions import Fraction
+from pathlib import Path
+from types import SimpleNamespace
+
+import av
+import numpy as np
+import pytest
+
+FPS, SECONDS, W, H = 10, 2, 64, 48
+LATE = 3   # frames the video starts after the audio (and the container)
+
+
+def level(i: int) -> int:
+    """Frame i's grey level: frames tell apart by brightness, through lossy encoding."""
+    return 10 + 10 * i
+
+
+def make(path):
+    """2 s of 64x48 H.264 at 10 fps, each frame one grey level, starting LATE frames after its 48 kHz stereo AAC (the
+    container's start): files cut from a longer one often start that way."""
+    with av.open(str(path), "w") as out:
+        v = out.add_stream("libx264", rate=FPS)
+        v.width, v.height, v.pix_fmt = W, H, "yuv420p"
+        a = out.add_stream("aac", rate=48000, layout="stereo")
+        for i in range(FPS * SECONDS):
+            frame = av.VideoFrame.from_ndarray(np.full((H, W, 3), level(i), np.uint8), format="rgb24")
+            frame.pts, frame.time_base = LATE + i, Fraction(1, FPS)
+            out.mux(v.encode(frame))
+        out.mux(v.encode())
+        t = np.arange(48000 * SECONDS) / 48000
+        tone = (0.3 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)
+        for start in range(0, len(tone), 1024):
+            chunk = np.stack([tone[start:start + 1024]] * 2)
+            frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(chunk), format="fltp", layout="stereo")
+            frame.sample_rate, frame.pts = 48000, start
+            out.mux(a.encode(frame))
+        out.mux(a.encode())
+
+
+def decoded(path) -> list[tuple[float, np.ndarray]]:
+    """Every frame in order: (seconds from the container's start, RGB)."""
+    with av.open(str(path)) as c:
+        return [(float(f.pts * f.time_base), f.to_ndarray(format="rgb24")) for f in c.decode(c.streams.video[0])]
+
+
+def nth(path, i: int) -> np.ndarray:
+    """Frame i of the video stream, counted from its first: the frame a detector sampling frame i saw."""
+    return [rgb for _, rgb in decoded(path)][i]
+
+
+@pytest.fixture
+def rotated():
+    """A clip stored sideways with a 90° rotation, as phones record (nodes/testdata/README.md): shown 32x64, blue on top
+    and red below."""
+    return Path(__file__).parent / "testdata" / "rotated-90.mp4"
+
+
+@pytest.fixture(scope="session")
+def clip(tmp_path_factory):
+    path = tmp_path_factory.mktemp("video") / "clip.mp4"
+    make(path)
+    return SimpleNamespace(path=path, fps=FPS, seconds=SECONDS, w=W, h=H, late=LATE, nth=lambda i: nth(path, i))

perception/_pipeline/nodes/core/decode/manifest.yaml

@@ -0,0 +1,19 @@
+id: core.decode
+version: 1.0.0
+stage: extract
+title: Shots and audio
+description: Video identity and timing, shots, keyframes and 16 kHz mono audio
+inputs: {video: video.v1}
+outputs:
+  media: media.v1
+  shots: bundle.shots@0.1
+  keyframes: assets.v1
+  audio: audio.wav.v1
+config:
+  shot_threshold: {type: number, default: 3.0, description: "AdaptiveDetector adaptive_threshold (PySceneDetect default)"}
+  keyframe_width: {type: integer, minimum: 64, default: 768}
+runners:
+  - {kind: native, env: envs/core, entry: node.py, accelerator: cpu}
+resources: {cpu_cores: 2, memory_gb: 2, gpu: none}
+models:
+  - {id: PySceneDetect AdaptiveDetector, licence: BSD-3-Clause, licence_mode: commercial-ok}

perception/_pipeline/nodes/core/decode/media.py

@@ -0,0 +1,104 @@
+"""The video's facts, keyframes and audio, read with PyAV: its wheels carry FFmpeg's libraries, so the node needs no
+ffmpeg or ffprobe on the machine (an app started from the Finder has only /usr/bin:/bin:/usr/sbin:/sbin on its PATH).
+Each function does what the ffprobe or ffmpeg command it replaced did; the comments say which."""
+
+import wave
+from fractions import Fraction
+from pathlib import Path
+
+import av
+import numpy as np
+
+from perception_node import NodeError
+
+KEYFRAME_POSITIONS = (0.15, 0.5, 0.85)
+
+
+def ratio(r: Fraction | None) -> str:
+    """A rate or time base as ffprobe printed it ("30000/1001", "30/1"; "0/0" when unknown)."""
+    return f"{r.numerator}/{r.denominator}" if r else "0/0"
+
+
+def probe(video: Path) -> dict:
+    """ffprobe's format=duration and the first video and audio streams' fields."""
+    try:
+        container = av.open(str(video))
+    except av.FFmpegError as e:
+        raise NodeError("input", f"cannot read the video: {str(e)[:300]}") from None
+    with container:
+        if not container.streams.video:
+            raise NodeError("input", "the file has no video stream")
+        v = container.streams.video[0]
+        a = container.streams.audio[0] if container.streams.audio else None
+        r, avg = v.base_rate, v.average_rate or v.base_rate
+        return {"duration": container.duration / av.time_base, "width": v.codec_context.width, "height": v.codec_context.height,
+                "fps": float(avg or r), "r_frame_rate": ratio(r), "avg_frame_rate": ratio(v.average_rate),
+                "time_base": ratio(v.time_base), "start_pts": v.start_time,
+                # containers report r_frame_rate as the base rate and avg_frame_rate from the frame count; they differ
+                # when frame durations vary
+                "variable_frame_rate": bool(avg) and abs(float(r) - float(avg)) > 1e-3,
+                "audio": {"sample_rate": a.codec_context.sample_rate, "channels": a.codec_context.channels} if a else None}
+
+
+def frame_at(container, t: float) -> av.VideoFrame:
+    """The frame nearest `t` seconds after the video stream's first frame. Callers' times are frame times counted from
+    that frame (face-track's frame index / fps, decode's shots from scenedetect's frame counts), rounded to the
+    millisecond: on either side of the frame's own time at 29.97 fps, so neither the first frame at or after t nor the
+    one shown at t is always it. (`ffmpeg -ss` counted from the container's start instead: when the video starts after
+    the audio, it cut crops and keyframes early.) The face-track node has the same function."""
+    v = container.streams.video[0]
+    target = Fraction(v.start_time or 0) * v.time_base + Fraction(round(t * 1000), 1000)
+    container.seek(int(target / v.time_base), stream=v, backward=True)
+    before = None
+    for frame in container.decode(v):
+        if frame.pts is None:
+            continue
+        at = frame.pts * v.time_base
+        if at >= target:
+            return before if before is not None and target - before.pts * v.time_base < at - target else frame
+        before = frame
+    if before is not None and target - before.pts * v.time_base < 1 / v.average_rate:   # within the last frame
+        return before
+    raise NodeError("input", f"the video has no frame at {t:.3f} s")
+
+
+def upright(frame: av.VideoFrame) -> np.ndarray:
+    """The frame as RGB, turned the way players show it. Phone recordings store the picture sideways with a rotation
+    to apply: ffmpeg applied it, and so does torchcodec, which reads the frames face-track's detector sees; PyAV only
+    reports it. The face-track node has the same function."""
+    return np.ascontiguousarray(np.rot90(frame.to_ndarray(format="rgb24"), round((frame.rotation or 0) / 90) % 4))
+
+
+def even_height(w: int, src_w: int, src_h: int) -> int:
+    """ffmpeg's `scale=w:-2`: the height that keeps the aspect ratio, to the nearest even number."""
+    return max(2, round(Fraction(w * src_h, src_w * 2)) * 2)
+
+
+def keyframes(video: Path, shots: list[dict], out_dir: Path, width: int) -> None:
+    """Three frames per shot, upright, `width` wide (ffmpeg's scale=width:-2, bicubic), as JPEG."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    with av.open(str(video)) as container:
+        for shot in shots:
+            shot["keyframes"] = []
+            for pos in KEYFRAME_POSITIONS:
+                t = shot["start"] + pos * (shot["end"] - shot["start"])
+                name = f"shot{shot['index']:04d}_{int(pos * 100):02d}.jpg"
+                rgb = upright(frame_at(container, t))
+                h, w = rgb.shape[:2]
+                av.VideoFrame.from_ndarray(rgb, format="rgb24").reformat(
+                    width=width, height=even_height(width, w, h), interpolation="BICUBIC").to_image().save(out_dir / name, quality=90)
+                shot["keyframes"].append({"t": round(t, 3), "file": f"assets/keyframes/{name}"})
+
+
+def audio(video: Path, wav: Path) -> None:
+    """The first audio stream as 16 kHz mono 16-bit WAV (ffmpeg -vn -ac 1 -ar 16000 -sample_fmt s16)."""
+    with av.open(str(video)) as container, wave.open(str(wav), "wb") as out:
+        out.setnchannels(1)
+        out.setsampwidth(2)
+        out.setframerate(16000)
+        resampler = av.AudioResampler(format="s16", layout="mono", rate=16000)
+        for frame in container.decode(container.streams.audio[0]):
+            for f in resampler.resample(frame):
+                out.writeframes(f.to_ndarray().tobytes())
+        for f in resampler.resample(None):
+            out.writeframes(f.to_ndarray().tobytes())

perception/_pipeline/nodes/core/decode/node.py

@@ -0,0 +1,56 @@
+"""core.decode: what every other node starts from.
+
+- media: the video's identity and timing (sha256, duration, size, frame rates, the stream's PTS time base, whether
+  the frame rate varies), so later nodes and the integer-microsecond mapping never need to re-probe the file;
+- shots: cuts by PySceneDetect's AdaptiveDetector, as the bundle's shots layer;
+- keyframes: three frames per shot (at 15%, 50%, 85% of it), referenced from the shots layer as assets/keyframes/...;
+- audio: 16 kHz mono 16-bit WAV for the audio nodes.
+
+From pipeline/bakeoff/extract.py (shots and keyframes unchanged), without its transcription.
+"""
+
+import time
+from pathlib import Path
+
+from perception_node import Node, NodeError
+from media import audio, keyframes, probe
+
+
+def detect_shots(video: Path, duration: float, threshold: float) -> list[dict]:
+    from scenedetect import AdaptiveDetector, detect
+
+    scenes = detect(str(video), AdaptiveDetector(adaptive_threshold=threshold))
+    if not scenes:
+        return [{"index": 0, "start": 0.0, "end": round(duration, 3)}]
+    return [{"index": i, "start": round(s.get_seconds(), 3), "end": round(e.get_seconds(), 3)} for i, (s, e) in enumerate(scenes)]
+
+
+with Node() as node:
+    video = node.input("video")
+    c = node.config
+    media = {"sha256": node.request["inputs"]["video"]["sha256"], "filename": video.name, **probe(video)}
+    node.write_json("media", "media.json", media)
+
+    timings = {}
+    t0 = time.time()
+    shots = detect_shots(video, media["duration"], c["shot_threshold"])
+    timings["shots_s"] = round(time.time() - t0, 1)
+    node.model("PySceneDetect AdaptiveDetector", licence_mode="commercial-ok", licence="BSD-3-Clause")
+
+    t0 = time.time()
+    assets = node.path("keyframes", "assets")
+    keyframes(video, shots, assets / "keyframes", c["keyframe_width"])
+    timings["keyframes_s"] = round(time.time() - t0, 1)
+
+    t0 = time.time()
+    wav = node.path("audio", "audio.wav")
+    if media["audio"] is None:
+        raise NodeError("input", "the video has no audio stream")
+    audio(video, wav)
+    timings["audio_s"] = round(time.time() - t0, 1)
+
+    layer = [{"id": f"shot:{s['index']}", "index": s["index"], "start": s["start"], "end": s["end"],
+              "keyframes": s["keyframes"], "caption": None} for s in shots]
+    node.write_json("shots", "shots.json", {"layer": "shots", "schema_version": "0.1.0",
+                                            "producer": node.producer(timings=timings, device="cpu"), "shots": layer})
+    node.usage(shots=len(shots))

perception/_pipeline/nodes/core/decode/test_media.py

@@ -0,0 +1,65 @@
+"""The decode node's media reading (media.py), with nothing on PATH: the app runs nodes from the Finder, where ffmpeg
+and ffprobe are not found. Run: uv run --project envs/core --with pytest pytest nodes/core/decode"""
+
+import wave
+
+import av
+import numpy as np
+import pytest
+from PIL import Image
+
+from media import audio, even_height, frame_at, keyframes, probe
+
+
+@pytest.fixture(autouse=True)
+def no_tools_on_path(monkeypatch):
+    monkeypatch.setenv("PATH", "")
+
+
+def test_probe(clip):
+    m = probe(clip.path)
+    assert (m["width"], m["height"], m["r_frame_rate"], m["avg_frame_rate"], m["fps"]) == (clip.w, clip.h, "10/1", "10/1", 10.0)
+    assert m["duration"] == pytest.approx(clip.seconds + clip.late / clip.fps, abs=0.05)   # the container's: from the audio's start
+    assert m["audio"] == {"sample_rate": 48000, "channels": 2}
+    assert not m["variable_frame_rate"]
+
+
+def test_frame_at_counts_from_the_video_streams_first_frame(clip):
+    with av.open(str(clip.path)) as c:
+        v = c.streams.video[0]
+        assert float(v.start_time * v.time_base) == clip.late / clip.fps and not c.start_time   # video after the audio
+        for i in (0, 1, 7, 19):   # frame i at i / fps, as face-track and scenedetect count (not ffmpeg -ss's i - late)
+            for off in (0, 0.0004, -0.0004):   # rounded to the millisecond, either way (29.97 fps frame times are)
+                assert np.array_equal(frame_at(c, i / clip.fps + off).to_ndarray(format="rgb24"), clip.nth(i)), (i, off)
+
+
+def test_frame_at_past_the_end(clip):
+    with av.open(str(clip.path)) as c, pytest.raises(Exception, match="no frame at 5.000 s"):
+        frame_at(c, 5.0)
+
+
+def test_keyframes(clip, tmp_path):
+    shots = [{"index": 0, "start": 0.0, "end": 1.0}, {"index": 1, "start": 1.0, "end": 2.0}]
+    keyframes(clip.path, shots, tmp_path, 33)
+    assert [k["t"] for k in shots[1]["keyframes"]] == [1.15, 1.5, 1.85]
+    img = Image.open(tmp_path / "shot0001_50.jpg")
+    assert img.size == (33, even_height(33, clip.w, clip.h)) == (33, 24)   # ffmpeg's scale=33:-2
+    assert abs(np.asarray(img).mean() - clip.nth(15).mean()) < 3   # frames are 10 grey levels apart
+
+
+def test_audio(clip, tmp_path):
+    audio(clip.path, tmp_path / "a.wav")
+    with wave.open(str(tmp_path / "a.wav")) as w:
+        assert (w.getnchannels(), w.getsampwidth(), w.getframerate()) == (1, 2, 16000)
+        assert w.getnframes() == pytest.approx(16000 * clip.seconds, abs=1600)
+        samples = np.frombuffer(w.readframes(w.getnframes()), np.int16)
+    assert np.abs(samples).max() > 5000   # the tone, downmixed, not silence
+
+
+def test_keyframes_are_upright(rotated, tmp_path):
+    shots = [{"index": 0, "start": 0.0, "end": 1.0}]
+    keyframes(rotated, shots, tmp_path, 16)
+    img = np.asarray(Image.open(tmp_path / "shot0000_50.jpg")).astype(int)
+    assert img.shape[:2] == (32, 16)   # 16 wide, as ffmpeg shows it: 32x64 upright
+    top, bottom = img[:12].mean((0, 1)), img[20:].mean((0, 1))
+    assert top[2] > 200 > top[0] and bottom[0] > 200 > bottom[2], (top, bottom)

perception/_pipeline/nodes/testdata/README.md

@@ -0,0 +1,10 @@
+Synthetic clips the node tests read (made with ffmpeg, which the tests themselves do not need):
+
+- `rotated-90.mp4`: 1 s of 64x32 H.264 at 10 fps, the left half red and the right half blue, stored with a 90° display
+  rotation, as phone recordings are. Players and ffmpeg show it 32x64, blue on top and red below.
+
+  ```sh
+  ffmpeg -f lavfi -i color=red:size=32x32:rate=10:duration=1 -f lavfi -i color=blue:size=32x32:rate=10:duration=1 \
+    -filter_complex hstack -c:v libx264 -pix_fmt yuv420p -map_metadata -1 -bitexact flat.mp4
+  ffmpeg -display_rotation 90 -i flat.mp4 -c copy -map_metadata -1 -bitexact rotated-90.mp4
+  ```