tryharness 0.7.25 → 0.7.28

package/lib/video.mjs

@@ -0,0 +1,332 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFile, spawn } from "node:child_process";
+
+// What an agent on this computer sees of a video (watch_video): where it cuts, a few frames from every shot as
+// pictures it can look at, and the words on screen in them. ffmpeg reads the video; judging it is the agent's own work.
+
+// Shots are found from each frame's colours, not its pixels: camera shake, flicker and fast motion move pixels but
+// keep the colours, while a cut changes them and they stay changed. ffmpeg's own scene score split one shaking shot
+// into ten in the content factory's cuts and missed two real cuts between pastel scenes.
+const SCAN_FPS = 30;
+const SCAN_SIDE = 64;
+// 8 levels each of red, green and blue.
+const BINS = 512;
+// Frames averaged on each side of a candidate cut.
+const BLOCK = 3;
+// The share of the picture's colours that must change for a new shot. In the content factory's videos real cuts
+// changed 0.26 or more and a card popping in 0.18 at most; a panel or large graphic appearing can pass it.
+const CUT_CHANGE = 0.25;
+// A flash shorter than this stays part of the shot it interrupts.
+const SHORTEST_SHOT = 0.2;
+const DEFAULT_FRAMES = 16;
+const MAX_FRAMES = 40;
+// Finding cuts decodes every frame, so a longer stretch is watched from its start only.
+const LONGEST_STRETCH = 600;
+// Pictures in the answer stay small enough for 16 to fit an agent's context; the saved copies are for a closer look.
+const LOOK_SIDE = 640;
+const SAVED_SIDE = 1280;
+const KEEP_FOR_MS = 24 * 60 * 60 * 1000;
+
+const round = (seconds) => Math.round(seconds * 100) / 100;
+const clamp = (value, low, high) => Math.min(high, Math.max(low, value));
+const lengthOf = (shot) => shot.end - shot.start;
+const lastLine = (text) => String(text).trim().split("\n").at(-1) || "";
+
+/** Runs a tool to its end: { stdout, stderr }. A missing ffmpeg says how to get it. */
+function run(command, args) {
+  return new Promise((resolve, reject) =>
+    execFile(
+      command,
+      args,
+      { maxBuffer: 64 * 1024 * 1024 },
+      (error, stdout, stderr) => {
+        if (error?.code === "ENOENT")
+          reject(
+            new Error(
+              "watching a video needs ffmpeg on this computer (brew install ffmpeg)"
+            )
+          );
+        else if (error)
+          reject(
+            new Error(`${command} could not read it: ${lastLine(stderr)}`)
+          );
+        else resolve({ stdout, stderr });
+      }
+    )
+  );
+}
+
+async function probe(file) {
+  const { stdout } = await run("ffprobe", [
+    "-v",
+    "error",
+    "-show_entries",
+    "format=duration:stream=codec_type,width,height,avg_frame_rate",
+    "-of",
+    "json",
+    file,
+  ]);
+  const { format, streams = [] } = JSON.parse(stdout);
+  const picture = streams.find((stream) => stream.codec_type === "video");
+  const seconds = Number(format?.duration);
+  if (!picture || !(seconds > 0))
+    throw new Error(`${path.basename(file)} is not a video`);
+  const [frames, per] = String(picture.avg_frame_rate).split("/").map(Number);
+  return {
+    seconds: round(seconds),
+    width: picture.width,
+    height: picture.height,
+    fps: per ? round(frames / per) : null,
+    sound: streams.some((stream) => stream.codec_type === "audio"),
+  };
+}
+
+/** Each frame's colour histogram, SCAN_FPS frames a second through [from, to), from a small copy of the picture. */
+async function colourHistograms(file, from, to) {
+  const child = spawn("ffmpeg", [
+    ...["-v", "error", "-ss", String(from), "-to", String(to), "-i", file],
+    ...["-an", "-sn", "-vf", `fps=${SCAN_FPS},scale=${SCAN_SIDE}:${SCAN_SIDE}`],
+    ...["-pix_fmt", "rgb24", "-f", "rawvideo", "pipe:1"],
+  ]);
+  let said = "";
+  child.stderr.on("data", (chunk) => (said += chunk));
+  const exited = new Promise((resolve, reject) => {
+    child.on("error", () =>
+      reject(new Error("watching a video needs ffmpeg on this computer"))
+    );
+    child.on("close", resolve);
+  });
+  const frameBytes = SCAN_SIDE * SCAN_SIDE * 3;
+  const histograms = [];
+  let pending = Buffer.alloc(0);
+  for await (const chunk of child.stdout) {
+    pending = Buffer.concat([pending, chunk]);
+    for (; pending.length >= frameBytes; pending = pending.subarray(frameBytes))
+      histograms.push(histogram(pending.subarray(0, frameBytes)));
+  }
+  if ((await exited) !== 0)
+    throw new Error(`ffmpeg could not read it: ${lastLine(said)}`);
+  return histograms;
+}
+
+function histogram(rgb) {
+  const counts = new Uint16Array(BINS);
+  for (let at = 0; at < rgb.length; at += 3)
+    counts[
+      ((rgb[at] >> 5) << 6) | ((rgb[at + 1] >> 5) << 3) | (rgb[at + 2] >> 5)
+    ]++;
+  return counts;
+}
+
+/** The share of colour that differs between two runs of frames, [a, b) and [b, c): 0 the same, 1 nothing shared. */
+function change(histograms, a, b, c) {
+  let differs = 0;
+  for (let bin = 0; bin < BINS; bin++) {
+    let before = 0;
+    let after = 0;
+    for (let i = a; i < b; i++) before += histograms[i][bin];
+    for (let i = b; i < c; i++) after += histograms[i][bin];
+    differs += Math.abs(before / (b - a) - after / (c - b));
+  }
+  return differs / 2 / (SCAN_SIDE * SCAN_SIDE);
+}
+
+/**
+ * The frames a new shot starts on: where the frames after differ from the frames before by CUT_CHANGE or more, the
+ * largest such change within a shortest shot either way, placed on the single frame that changes most.
+ */
+function cutFrames(histograms) {
+  const count = histograms.length;
+  const around = (i) =>
+    i === 0
+      ? 0
+      : change(
+          histograms,
+          Math.max(0, i - BLOCK),
+          i,
+          Math.min(count, i + BLOCK)
+        );
+  const changes = histograms.map((_, i) => around(i));
+  const reach = Math.round(SHORTEST_SHOT * SCAN_FPS);
+  const cuts = [];
+  for (let i = 1; i < count; i++) {
+    if (changes[i] < CUT_CHANGE) continue;
+    const start = Math.max(1, i - reach);
+    const nearby = changes.slice(start, i + reach + 1);
+    const firstLargest = start + nearby.indexOf(Math.max(...nearby));
+    if (firstLargest === i) cuts.push(sharpestFrame(histograms, i));
+  }
+  return cuts;
+}
+
+function sharpestFrame(histograms, i) {
+  let best = i;
+  let most = -1;
+  const last = Math.min(histograms.length - 1, i + BLOCK - 1);
+  for (let j = Math.max(1, i - BLOCK + 1); j <= last; j++) {
+    const step = change(histograms, j - 1, j, j + 1);
+    if (step > most) [best, most] = [j, step];
+  }
+  return best;
+}
+
+/** The times a new shot starts within [from, to). */
+async function cutTimes(file, from, to) {
+  const histograms = await colourHistograms(file, from, to);
+  // Seeking the input restarts its clock at zero, so each time is counted from `from`.
+  return cutFrames(histograms).map((frame) => from + frame / SCAN_FPS);
+}
+
+function shotsBetween(from, to, cuts) {
+  const starts = [from];
+  for (const cut of cuts)
+    if (cut - starts.at(-1) >= SHORTEST_SHOT && to - cut >= SHORTEST_SHOT)
+      starts.push(cut);
+  return starts.map((start, index) => ({
+    start: round(start),
+    end: round(starts[index + 1] ?? to),
+  }));
+}
+
+/** `count` items spread evenly through a list, first and last included. */
+const evenly = (list, count) =>
+  Array.from(
+    { length: count },
+    (_, i) => list[Math.round((i * (list.length - 1)) / Math.max(1, count - 1))]
+  );
+
+/**
+ * Where to take pictures: one inside every shot (an even spread of shots when there are more shots than pictures), and
+ * the pictures left over shared out by shot length, so a long take is seen more than once.
+ */
+function frameTimes(shots, count) {
+  const numbered = shots.map((shot, index) => ({ shot, number: index + 1 }));
+  const chosen = numbered.length > count ? evenly(numbered, count) : numbered;
+  const spare = count - chosen.length;
+  const total = chosen.reduce((sum, { shot }) => sum + lengthOf(shot), 0);
+  const shares = chosen.map(({ shot }) => (spare * lengthOf(shot)) / total);
+  const extra = shares.map(Math.floor);
+  const left = spare - extra.reduce((sum, n) => sum + n, 0);
+  shares
+    .map((share, i) => ({ part: share - extra[i], i }))
+    .sort((a, b) => b.part - a.part)
+    .slice(0, left)
+    .forEach(({ i }) => extra[i]++);
+  return chosen.flatMap(({ shot, number }, i) => {
+    const pictures = 1 + extra[i];
+    return Array.from({ length: pictures }, (_, k) => ({
+      at: round(shot.start + ((k + 0.5) / pictures) * lengthOf(shot)),
+      shot: number,
+    }));
+  });
+}
+
+// Scales the longer side down to `side` pixels, never up.
+const fit = (side) =>
+  `scale='if(gte(iw,ih),min(iw,${side}),-2)':'if(gte(iw,ih),-2,min(ih,${side}))'`;
+
+/** One frame at `at` seconds, saved twice: large at `saved` and small at `look`. */
+function takePicture(file, at, saved, look) {
+  return run("ffmpeg", [
+    "-v",
+    "error",
+    "-y",
+    "-ss",
+    String(at),
+    "-i",
+    file,
+    "-filter_complex",
+    `[0:v]split[a][b];[a]${fit(SAVED_SIDE)}[saved];[b]${fit(LOOK_SIDE)}[look]`,
+    ...["-map", "[saved]", "-frames:v", "1", "-update", "1", "-q:v", "3"],
+    saved,
+    ...["-map", "[look]", "-frames:v", "1", "-update", "1", "-q:v", "5"],
+    look,
+  ]);
+}
+
+async function readText(ocr, picture) {
+  const { lines } = await ocr.read(fs.readFileSync(picture), {
+    lane: "handoff",
+  });
+  return lines.map((line) => line.text).join("\n");
+}
+
+function pruneOld(folder) {
+  for (const entry of fs.readdirSync(folder, { withFileTypes: true })) {
+    const dir = path.join(folder, entry.name);
+    // Another watch may have removed it a moment ago.
+    const made = fs.statSync(dir, { throwIfNoEntry: false })?.mtimeMs;
+    if (entry.isDirectory() && Date.now() - made > KEEP_FOR_MS)
+      fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function summarise(shots, video, watched) {
+  const span = round(watched.to - watched.from);
+  const part =
+    span < video.seconds - 0.05
+      ? ` (${watched.from} s to ${watched.to} s of ${video.seconds} s)`
+      : "";
+  if (shots.length === 1) return `one shot of ${span} s${part}`;
+  const lengths = shots.map(lengthOf);
+  const shortest = round(Math.min(...lengths));
+  const longest = round(Math.max(...lengths));
+  const average = round(span / shots.length);
+  return `${shots.length} shots in ${span} s${part}: ${shortest} s to ${longest} s long, ${average} s on average`;
+}
+
+/**
+ * A video as an agent can watch it: { file, seconds, width, height, fps, sound, watched: { from, to }, shots: [{ start,
+ * end }], summary, frames: [{ at, shot, path, look, text? }] }, times in seconds from the start of the file. `look` is a
+ * small JPEG for the answer and `path` a larger one for a closer look, both kept a day under `out`; `text` is what the
+ * relay's OCR read in the frame, when `ocr` is given. Throws when ffmpeg is missing or the file is not a video.
+ */
+export async function watchVideo(
+  file,
+  { from = 0, to, frames = DEFAULT_FRAMES, ocr = null, out }
+) {
+  const video = await probe(file);
+  const start = clamp(Number(from) || 0, 0, video.seconds);
+  const end = Math.min(
+    Number(to) || video.seconds,
+    video.seconds,
+    start + LONGEST_STRETCH
+  );
+  if (end <= start)
+    throw new Error(`there is nothing between ${start} s and ${end} s`);
+  const shots = shotsBetween(start, end, await cutTimes(file, start, end));
+  const count = clamp(
+    Math.round(Number(frames)) || DEFAULT_FRAMES,
+    1,
+    MAX_FRAMES
+  );
+
+  fs.mkdirSync(out, { recursive: true });
+  pruneOld(out);
+  const name = path.basename(file, path.extname(file)).replace(/[^\w-]+/g, "-");
+  const dir = fs.mkdtempSync(
+    path.join(out, `${name.slice(0, 40) || "video"}-`)
+  );
+  const pictures = [];
+  for (const { at, shot } of frameTimes(shots, count)) {
+    const stem = path.join(dir, at.toFixed(2).replace(".", "s"));
+    const saved = `${stem}.jpg`;
+    const look = `${stem}-look.jpg`;
+    await takePicture(file, at, saved, look);
+    // A time in the last frame's gap can come back empty: that picture is skipped.
+    if (!fs.existsSync(look)) continue;
+    const text = ocr ? await readText(ocr, saved) : undefined;
+    pictures.push({ at, shot, path: saved, look, text });
+  }
+
+  const watched = { from: round(start), to: round(end) };
+  return {
+    file,
+    ...video,
+    watched,
+    shots,
+    summary: summarise(shots, video, watched),
+    frames: pictures,
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tryharness",
-  "version": "0.7.25",
+  "version": "0.7.28",
   "description": "Harness Relay: the daemon that turns the agents on this machine into your personal swarm. Memory, history, and work handed between any of them.",
   "bin": {
     "tryharness": "cli.mjs"