tryharness 0.7.26 → 0.7.29

package/lib/listen.mjs

@@ -0,0 +1,674 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { execFile, spawn } from "node:child_process";
+
+// What an agent on this computer can know of a recording without ears (listen): its tempo and beat grid, where bars
+// likely start, how loud it is by the broadcast measure (EBU R128), where it gets louder, quieter or silent, and the
+// words said in it. Measured here from the samples; nothing leaves the computer.
+
+// Loudness is defined on the full band, with K-weighting filters published for 48 kHz.
+const RATE = 48_000;
+// Rhythm and speech are read from a 16 kHz mono copy: every third sample, after a low-pass.
+const DOWN = 3;
+const LOW_RATE = RATE / DOWN;
+const LONGEST = 600;
+// Loudness is gathered in 100 ms steps: a momentary block is 4 of them, a short-term window 30.
+const STEP = RATE / 10;
+const STEPS_PER_SECOND = 10;
+const CLIP = 32767 / 32768;
+// A 100 ms step under this many LUFS is silence.
+const SILENT = -60;
+// A new section starts where the 3 s loudness moves this many LU from the section's level and stays moved.
+const LEVEL_SHIFT = 4;
+
+// The onset curve: 64 ms frames every 10 ms, so 100 points a second.
+const FRAME = 1024;
+const HOP = 160;
+const FPS = LOW_RATE / HOP;
+// Spectral flux peaks when a sound is three quarters into a frame, where the window rises fastest: a frame's onset is
+// that far after its start (48 ms on clicks over silence and over a steady pad alike).
+const ONSET_LAG = (3 * FRAME) / 4;
+// Under 150 Hz: the kick drum, which most often marks where a bar starts.
+const BASS_BINS = Math.round(150 / (LOW_RATE / FRAME));
+// How strictly beats keep to one period. At librosa's 100 the tracker held the off-beat for bars at a time on four of
+// the content factory's six tracks; at 400, five of the six stay on one grid and the tempos agree with their pages.
+const TIGHTNESS = 400;
+// A grid is steady when beats stray from it by less than this on average.
+const STEADY_STRAY = 0.03;
+
+const round = (value) => Math.round(value * 100) / 100;
+const round1 = (value) => Math.round(value * 10) / 10;
+const average = (values) =>
+  values.reduce((sum, value) => sum + value, 0) / values.length;
+const lufs = (energy) => -0.691 + 10 * Math.log10(energy);
+const lastLine = (text) => String(text).trim().split("\n").at(-1) || "";
+
+// ── Decoding ─────────────────────────────────────────────────────────────────
+
+const runs = (tool) =>
+  new Promise((resolve) =>
+    execFile(tool, ["-version"], (error) => resolve(!error))
+  );
+
+/**
+ * The recording from `from` for at most `seconds`, as a WAV byte stream of 16-bit samples at 48 kHz in its own
+ * channels. From ffmpeg; on a Mac without it, from afconvert through a scratch file, which cannot seek, so its first
+ * `skip` frames are for the reader to pass over.
+ */
+async function decode(file, from, seconds) {
+  if (await runs("ffmpeg"))
+    return { bytes: fromFfmpeg(file, from, seconds), skip: 0 };
+  if (process.platform === "darwin")
+    return { bytes: fromAfconvert(file), skip: Math.round(from * RATE) };
+  throw new Error("listening needs ffmpeg on this computer");
+}
+
+async function* fromFfmpeg(file, from, seconds) {
+  const child = spawn("ffmpeg", [
+    ...["-v", "error", "-ss", String(from), "-t", String(seconds)],
+    ...["-i", file, "-vn", "-sn", "-ar", String(RATE)],
+    ...["-c:a", "pcm_s16le", "-f", "wav", "pipe:1"],
+  ]);
+  let said = "";
+  child.stderr.on("data", (chunk) => (said += chunk));
+  const exited = new Promise((resolve) => child.on("close", resolve));
+  try {
+    yield* child.stdout;
+    if ((await exited) !== 0)
+      throw new Error(`ffmpeg found no sound to read: ${lastLine(said)}`);
+  } finally {
+    child.kill();
+  }
+}
+
+async function* fromAfconvert(file) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "harness-listen-"));
+  const wav = path.join(dir, "sound.wav");
+  try {
+    await new Promise((resolve, reject) =>
+      execFile(
+        "afconvert",
+        ["-f", "WAVE", "-d", `LEI16@${RATE}`, file, wav],
+        (error) =>
+          error
+            ? reject(new Error(`${path.basename(file)} needs ffmpeg to read`))
+            : resolve()
+      )
+    );
+    yield* fs.createReadStream(wav);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** Where a WAV stream's samples start and how many channels it has, once enough of it has arrived; null until then. */
+function wavStart(bytes) {
+  if (bytes.length < 12) return null;
+  const riff = bytes.toString("ascii", 0, 4);
+  const wave = bytes.toString("ascii", 8, 12);
+  if (riff !== "RIFF" || wave !== "WAVE")
+    throw new Error("the decoder did not give a WAV stream");
+  let channels = 0;
+  for (let at = 12; at + 8 <= bytes.length;) {
+    const tag = bytes.toString("ascii", at, at + 4);
+    const size = bytes.readUInt32LE(at + 4);
+    if (tag === "data") return channels ? { channels, at: at + 8 } : null;
+    if (tag === "fmt " && at + 12 <= bytes.length)
+      channels = bytes.readUInt16LE(at + 10);
+    at += 8 + size + (size % 2);
+  }
+  return null;
+}
+
+/** A WAV byte stream as chunks of interleaved samples from -1 to 1: { channels, samples }. */
+async function* sampleChunks(bytes) {
+  let channels = 0;
+  let pending = Buffer.alloc(0);
+  for await (const chunk of bytes) {
+    pending = pending.length ? Buffer.concat([pending, chunk]) : chunk;
+    if (!channels) {
+      const start = wavStart(pending);
+      if (!start) continue;
+      channels = start.channels;
+      pending = pending.subarray(start.at);
+    }
+    const whole = pending.length - (pending.length % (2 * channels));
+    const samples = new Float32Array(whole / 2);
+    for (let i = 0; i < samples.length; i++)
+      samples[i] = pending.readInt16LE(2 * i) / 32768;
+    pending = pending.subarray(whole);
+    yield { channels, samples };
+  }
+}
+
+// ── Measuring while decoding ─────────────────────────────────────────────────
+
+/** A second-order filter over one sample at a time, keeping its own state. */
+function biquad([b0, b1, b2], [, a1, a2]) {
+  let x1 = 0;
+  let x2 = 0;
+  let y1 = 0;
+  let y2 = 0;
+  return (x) => {
+    const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
+    x2 = x1;
+    x1 = x;
+    y2 = y1;
+    y1 = y;
+    return y;
+  };
+}
+
+// ITU-R BS.1770-4 K-weighting at 48 kHz: a high shelf for the head, then a high-pass.
+const SHELF = [
+  [1.53512485958697, -2.69169618940638, 1.19839281085285],
+  [1, -1.69065929318241, 0.73248077421585],
+];
+const HIGH_PASS = [
+  [1, -2, 1],
+  [1, -1.99004745483398, 0.99007225036621],
+];
+
+/** A low-pass biquad at `cutoff` Hz with quality `q` (RBJ's cookbook). */
+function lowPass(cutoff, q) {
+  const w = (2 * Math.PI * cutoff) / RATE;
+  const alpha = Math.sin(w) / (2 * q);
+  const a0 = 1 + alpha;
+  const b = (1 - Math.cos(w)) / 2 / a0;
+  return [
+    [b, 2 * b, b],
+    [1, (-2 * Math.cos(w)) / a0, (1 - alpha) / a0],
+  ];
+}
+// A fourth-order Butterworth at 7 kHz, so nothing above 8 kHz folds back into the 16 kHz copy.
+const ANTI_ALIAS = [lowPass(7000, 0.5412), lowPass(7000, 1.3066)];
+
+/**
+ * Reads the stream once: K-weighted energy per 100 ms step (summed over channels), the sample peak, how many samples
+ * clipped, and the 16 kHz mono copy.
+ */
+async function measure({ bytes, skip }, limit) {
+  const steps = [];
+  const pieces = [];
+  let filters = null;
+  const smooth = ANTI_ALIAS.map((stage) => biquad(...stage));
+  let frame = 0;
+  let energy = 0;
+  let filled = 0;
+  let peak = 0;
+  let clipped = 0;
+  const done = () => frame >= skip + limit;
+  for await (const { channels, samples } of sampleChunks(bytes)) {
+    filters ??= Array.from({ length: channels }, () => [
+      biquad(...SHELF),
+      biquad(...HIGH_PASS),
+    ]);
+    const piece = new Float32Array(Math.ceil(samples.length / channels / DOWN));
+    let kept = 0;
+    for (let at = 0; at < samples.length && !done(); at += channels, frame++) {
+      if (frame < skip) continue;
+      let sum = 0;
+      for (let c = 0; c < channels; c++) {
+        const sample = samples[at + c];
+        const [shelf, highPass] = filters[c];
+        const weighted = highPass(shelf(sample));
+        energy += weighted * weighted;
+        peak = Math.max(peak, Math.abs(sample));
+        if (Math.abs(sample) >= CLIP) clipped++;
+        sum += sample;
+      }
+      const low = smooth[1](smooth[0](sum / channels));
+      if ((frame - skip) % DOWN === 0) piece[kept++] = low;
+      if (++filled === STEP) {
+        steps.push(energy / STEP);
+        energy = 0;
+        filled = 0;
+      }
+    }
+    pieces.push(piece.subarray(0, kept));
+    if (done()) break;
+  }
+  return { steps, peak, clipped, mono: joined(pieces) };
+}
+
+function joined(pieces) {
+  const out = new Float32Array(
+    pieces.reduce((n, piece) => n + piece.length, 0)
+  );
+  let at = 0;
+  for (const piece of pieces) {
+    out.set(piece, at);
+    at += piece.length;
+  }
+  return out;
+}
+
+// ── Loudness ─────────────────────────────────────────────────────────────────
+
+/** Mean energy over every run of `size` steps, one per step. */
+function windows(steps, size) {
+  const out = [];
+  for (let i = 0; i + size <= steps.length; i++)
+    out.push(average(steps.slice(i, i + size)));
+  return out;
+}
+
+/** Integrated loudness (BS.1770-4): 400 ms blocks every 100 ms, gated at -70 LUFS and at 10 LU under their mean. */
+function integratedLoudness(steps) {
+  const blocks = windows(steps, 4).filter((block) => lufs(block) > -70);
+  if (!blocks.length) return null;
+  const gate = lufs(average(blocks)) - 10;
+  return round1(lufs(average(blocks.filter((block) => lufs(block) > gate))));
+}
+
+/** Loudness range (EBU Tech 3342): 3 s loudness from its 10th to 95th percentile, gated at -70 LUFS and 20 LU under. */
+function loudnessRange(steps) {
+  const shortTerm = windows(steps, 30).filter((block) => lufs(block) > -70);
+  if (!shortTerm.length) return null;
+  const gate = lufs(average(shortTerm)) - 20;
+  const levels = shortTerm
+    .map(lufs)
+    .filter((level) => level > gate)
+    .sort((a, b) => a - b);
+  const at = (share) => levels[Math.round(share * (levels.length - 1))];
+  return round1(at(0.95) - at(0.1));
+}
+
+/** Stretches of half a second or more in which every 100 ms step is under SILENT. */
+function silences(steps, from) {
+  const found = [];
+  let start = null;
+  for (let i = 0; i <= steps.length; i++) {
+    const quiet = i < steps.length && lufs(steps[i]) < SILENT;
+    if (quiet && start === null) start = i;
+    if (!quiet && start !== null) {
+      if (i - start >= STEPS_PER_SECOND / 2)
+        found.push({
+          start: round(from + start / STEPS_PER_SECOND),
+          end: round(from + i / STEPS_PER_SECOND),
+        });
+      start = null;
+    }
+  }
+  return found;
+}
+
+/**
+ * Where it gets louder or quieter: sections of the 3 s loudness, each starting where the level moves LEVEL_SHIFT LU
+ * from the section's own for two seconds running, with each section's mean level.
+ */
+function levelSections(steps, from) {
+  const seconds = Math.floor(steps.length / STEPS_PER_SECOND);
+  if (seconds < 2) return [];
+  const perSecond = Array.from({ length: seconds }, (_, s) => {
+    const around = steps.slice(Math.max(0, (s - 1) * 10), (s + 2) * 10);
+    return Math.max(SILENT, lufs(average(around)));
+  });
+  const starts = [0];
+  let level = perSecond[0];
+  for (let s = 1; s + 1 < seconds; s++) {
+    const moved = (t) => Math.abs(perSecond[t] - level) >= LEVEL_SHIFT;
+    if (!moved(s) || !moved(s + 1)) continue;
+    starts.push(s);
+    level = perSecond[s + 1];
+  }
+  return starts.map((start, i) => {
+    const end = starts[i + 1] ?? seconds;
+    return {
+      start: round(from + start),
+      end: round(from + end),
+      lufs: round1(average(perSecond.slice(start, end))),
+    };
+  });
+}
+
+// ── Rhythm ───────────────────────────────────────────────────────────────────
+
+const WINDOW = Float64Array.from(
+  { length: FRAME },
+  (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / FRAME)
+);
+const COS = Float64Array.from({ length: FRAME / 2 }, (_, k) =>
+  Math.cos((-2 * Math.PI * k) / FRAME)
+);
+const SIN = Float64Array.from({ length: FRAME / 2 }, (_, k) =>
+  Math.sin((-2 * Math.PI * k) / FRAME)
+);
+
+/** In-place radix-2 FFT of FRAME points. */
+function fft(re, im) {
+  for (let i = 1, j = 0; i < FRAME; i++) {
+    let bit = FRAME >> 1;
+    for (; j & bit; bit >>= 1) j ^= bit;
+    j ^= bit;
+    if (i < j) {
+      [re[i], re[j]] = [re[j], re[i]];
+      [im[i], im[j]] = [im[j], im[i]];
+    }
+  }
+  for (let size = 2; size <= FRAME; size *= 2) {
+    const half = size / 2;
+    const stride = FRAME / size;
+    for (let start = 0; start < FRAME; start += size)
+      for (let k = 0; k < half; k++) {
+        const wr = COS[k * stride];
+        const wi = SIN[k * stride];
+        const a = start + k;
+        const b = a + half;
+        const tr = wr * re[b] - wi * im[b];
+        const ti = wr * im[b] + wi * re[b];
+        re[b] = re[a] - tr;
+        im[b] = im[a] - ti;
+        re[a] += tr;
+        im[a] += ti;
+      }
+  }
+}
+
+/**
+ * How sharply sound starts at each 10 ms: the rise of every frequency's log level from one frame to the next (spectral
+ * flux), over all frequencies and under 150 Hz. Yields to the daemon's other work every few thousand frames.
+ */
+async function onsetCurves(mono) {
+  const frames = Math.max(0, Math.floor((mono.length - FRAME) / HOP) + 1);
+  const all = new Float32Array(frames);
+  const bass = new Float32Array(frames);
+  const re = new Float64Array(FRAME);
+  const im = new Float64Array(FRAME);
+  const previous = new Float64Array(FRAME / 2);
+  for (let t = 0; t < frames; t++) {
+    for (let i = 0; i < FRAME; i++) {
+      re[i] = mono[t * HOP + i] * WINDOW[i];
+      im[i] = 0;
+    }
+    fft(re, im);
+    for (let k = 1; k < FRAME / 2; k++) {
+      const level = Math.log1p(10 * Math.hypot(re[k], im[k]));
+      const rise = Math.max(0, level - previous[k]);
+      all[t] += rise;
+      if (k < BASS_BINS) bass[t] += rise;
+      previous[k] = level;
+    }
+    if (t % 2000 === 1999)
+      await new Promise((resolve) => setImmediate(resolve));
+  }
+  return { all, bass };
+}
+
+/** The onset curve less its local mean (over a second), kept above zero and scaled to unit size. */
+function normalised(curve) {
+  const sums = new Float64Array(curve.length + 1);
+  curve.forEach((value, i) => (sums[i + 1] = sums[i] + value));
+  const reach = FPS / 2;
+  const out = curve.map((value, i) => {
+    const a = Math.max(0, i - reach);
+    const b = Math.min(curve.length, i + reach + 1);
+    return Math.max(0, value - (sums[b] - sums[a]) / (b - a));
+  });
+  const size = Math.sqrt(average(out.map((value) => value * value))) || 1;
+  return out.map((value) => value / size);
+}
+
+/**
+ * The beat period in onset frames: the lag at which the curve best matches itself, between 40 and 240 BPM, leaning
+ * toward 120 as listeners do (one octave either side halves the weight twice).
+ */
+function beatPeriod(curve) {
+  const shortest = Math.floor((FPS * 60) / 240);
+  const longest = Math.ceil((FPS * 60) / 40);
+  const scores = new Map();
+  for (let lag = shortest - 1; lag <= longest + 1; lag++) {
+    let sum = 0;
+    for (let t = lag; t < curve.length; t++) sum += curve[t] * curve[t - lag];
+    const bpm = (FPS * 60) / lag;
+    const lean = Math.exp(-0.5 * Math.log2(bpm / 120) ** 2);
+    scores.set(lag, (sum / (curve.length - lag)) * lean);
+  }
+  let best = shortest;
+  for (let lag = shortest; lag <= longest; lag++)
+    if (scores.get(lag) > scores.get(best)) best = lag;
+  // A parabola through the peak and its neighbours places it between whole frames.
+  const [a, b, c] = [best - 1, best, best + 1].map((lag) => scores.get(lag));
+  const bend = a - 2 * b + c;
+  const shift = bend === 0 ? 0 : (0.5 * (a - c)) / bend;
+  return best + shift;
+}
+
+/** Beat frames by dynamic programming (Ellis 2007): onsets on a beat score, gaps away from the period cost. */
+function trackBeats(curve, period) {
+  const count = curve.length;
+  const score = new Float64Array(count);
+  const previous = new Int32Array(count).fill(-1);
+  const nearest = Math.max(1, Math.round(period / 2));
+  const farthest = Math.round(period * 2);
+  for (let t = 0; t < count; t++) {
+    let best = 0;
+    for (let p = Math.max(0, t - farthest); p <= t - nearest; p++) {
+      const drift = Math.log((t - p) / period);
+      const value = score[p] - TIGHTNESS * drift * drift;
+      if (value > best) {
+        best = value;
+        previous[t] = p;
+      }
+    }
+    score[t] = curve[t] + best;
+  }
+  // The chain ends on the best-scoring frame within the last period.
+  let end = Math.max(0, count - Math.round(period));
+  for (let t = end; t < count; t++) if (score[t] > score[end]) end = t;
+  const beats = [];
+  for (let t = end; t >= 0; t = previous[t]) beats.push(t);
+  return trimmed(curve, beats.reverse());
+}
+
+/**
+ * The chain runs on through an intro or outro with no onsets at the period's own pace; beats at either end weaker than
+ * half the beats' typical onset are dropped, as librosa does.
+ */
+function trimmed(curve, beats) {
+  const strengths = beats.map((frame) => peakNear(curve, frame));
+  const typical = Math.sqrt(average(strengths.map((s) => s * s)));
+  const strong = (i) => strengths[i] >= typical / 2;
+  let first = 0;
+  let last = beats.length - 1;
+  while (first < last && !strong(first)) first++;
+  while (last > first && !strong(last)) last--;
+  return beats.slice(first, last + 1);
+}
+
+/** The strongest onset within 20 ms of a frame; 0 off either end. */
+function peakNear(curve, frame) {
+  let most = 0;
+  const last = Math.min(curve.length - 1, frame + 2);
+  for (let t = Math.max(0, frame - 2); t <= last; t++)
+    most = Math.max(most, curve[t]);
+  return most;
+}
+const frameAt = (time) => Math.round((time * LOW_RATE - ONSET_LAG) / HOP);
+
+/**
+ * How much more onset the grid's beats have than any moment does, on average: about 1 for a grid the music ignores.
+ * A strict tracker keeps an even spacing through anything, so an even spacing alone does not show a beat.
+ */
+function beatStrength(curve, grid) {
+  const anywhere = average(curve.map((_, frame) => peakNear(curve, frame)));
+  const end = (curve.length * HOP) / LOW_RATE;
+  const onBeats = [];
+  for (let time = grid.first; time < end; time += grid.length)
+    onBeats.push(peakNear(curve, frameAt(time)));
+  return average(onBeats) / anywhere;
+}
+
+const beatNumber = (time, grid) =>
+  Math.round((time - grid.first) / grid.length);
+const missOf = (time, grid) =>
+  time - grid.first - beatNumber(time, grid) * grid.length;
+
+/** The straight line through beat times by their beat numbers on `grid`: { first, length }. */
+function line(times, grid) {
+  const numbers = times.map((time) => beatNumber(time, grid));
+  const meanNumber = average(numbers);
+  const meanTime = average(times);
+  let across = 0;
+  let spread = 0;
+  numbers.forEach((n, i) => {
+    across += (n - meanNumber) * (times[i] - meanTime);
+    spread += (n - meanNumber) ** 2;
+  });
+  const length = across / spread;
+  return { first: meanTime - length * meanNumber, length };
+}
+
+/**
+ * The steady grid through beat times: { first, length, stray, onGrid }. The tracker can hold the off-beat for a few
+ * bars (Candy, 18 to 28 s), so the line is fitted again on the beats near it, closer each time; `stray` is how far
+ * those beats are from it and `onGrid` the share of beats that are.
+ */
+function fitGrid(times, roughLength) {
+  let grid = { first: times[0], length: roughLength };
+  let kept = times;
+  for (const reach of [0.5, 0.25, 0.1]) {
+    const near = times.filter(
+      (time) => Math.abs(missOf(time, grid)) < reach * grid.length
+    );
+    if (near.length < 8) break;
+    kept = near;
+    grid = line(kept, grid);
+  }
+  const misses = kept.map((time) => missOf(time, grid));
+  const stray = Math.sqrt(average(misses.map((miss) => miss * miss)));
+  return { ...grid, stray, onGrid: kept.length / times.length };
+}
+
+/**
+ * Which beat of four starts each bar: the one with the most new bass on it (the kick), with `sure` when it stands
+ * clearly above the next. Music with a kick on every beat leaves it unsure.
+ */
+function barStart(bass, grid, times) {
+  const strength = [0, 0, 0, 0];
+  const counts = [0, 0, 0, 0];
+  for (const time of times) {
+    const n = Math.round((time - grid.first) / grid.length);
+    const beat = ((n % 4) + 4) % 4;
+    strength[beat] += peakNear(bass, frameAt(time));
+    counts[beat]++;
+  }
+  const mean = strength.map((total, beat) => total / (counts[beat] || 1));
+  const order = [0, 1, 2, 3].sort((a, b) => mean[b] - mean[a]);
+  return { beat: order[0], sure: mean[order[0]] >= 1.25 * mean[order[1]] };
+}
+
+/** The tempo and beat grid, or { steady: false } with the beats found when the sound has no steady beat. */
+async function rhythm(mono, from, quiet) {
+  const { all, bass } = await onsetCurves(mono);
+  if (all.length < FPS * 4) return { steady: false, beats: [] };
+  const curve = normalised(all);
+  const period = beatPeriod(curve);
+  const timeOf = (frame) => (frame * HOP + ONSET_LAG) / LOW_RATE;
+  // A beat right at the end of a silence is the sound starting, so it stays.
+  const inSilence = (time) =>
+    quiet.some(
+      (gap) => time >= gap.start - from && time < gap.end - from - 0.05
+    );
+  const times = trackBeats(curve, period)
+    .map(timeOf)
+    .filter((time) => !inSilence(time));
+  if (times.length < 8)
+    return { steady: false, beats: times.map((t) => round(from + t)) };
+  const grid = fitGrid(times, period / FPS);
+  const strength = beatStrength(curve, grid);
+  const steady =
+    grid.stray < STEADY_STRAY && grid.onGrid >= 0.75 && strength >= 1.5;
+  if (!steady)
+    return { steady: false, beats: times.map((time) => round(from + time)) };
+  const bars = barStart(bass, grid, times);
+  // The grid's first beat at or after the first beat heard, and the first bar start at or after that.
+  const firstNumber = Math.round((times[0] - grid.first) / grid.length);
+  const firstBeat = grid.first + firstNumber * grid.length;
+  const toBar = (((bars.beat - firstNumber) % 4) + 4) % 4;
+  return {
+    steady: true,
+    bpm: round(60 / grid.length),
+    beatSeconds: Math.round(grid.length * 10000) / 10000,
+    firstBeat: Math.round((from + firstBeat) * 1000) / 1000,
+    strayMs: Math.round(grid.stray * 1000),
+    beatStrength: round1(strength),
+    bars: {
+      beatsPerBar: 4,
+      firstDownbeat:
+        Math.round((from + firstBeat + toBar * grid.length) * 1000) / 1000,
+      sure: bars.sure,
+    },
+  };
+}
+
+// ── Words ────────────────────────────────────────────────────────────────────
+
+/** Where someone speaks and what they say, by the relay's speech model; its agent-name fixing is left out. */
+async function words(speech, mono, from) {
+  // An agent's question is handoff work: the person's own dictation goes first.
+  const spans = await speech.voiced(mono, { lane: "handoff" });
+  const voice = spans.map(([start, end]) => ({
+    start: round(from + start / LOW_RATE),
+    end: round(from + end / LOW_RATE),
+  }));
+  if (!voice.length) return { voice, words: "" };
+  const heard = await speech.transcribe(
+    { sampleRate: LOW_RATE, samples: mono },
+    { lane: "handoff" }
+  );
+  return { voice, words: heard.raw };
+}
+
+function summarise(listened, loudness, beat) {
+  const heard = `${round1(listened.to - listened.from)} s heard`;
+  const range = loudness.range === null ? "" : `range ${loudness.range} LU, `;
+  const level =
+    loudness.integrated === null
+      ? "silent"
+      : `${loudness.integrated} LUFS (${range}peak ${loudness.peak} dBFS)`;
+  if (!beat.steady) return `${heard}; ${level}; no steady beat`;
+  const bars = `bars ${beat.bars.sure ? "" : "perhaps "}from ${beat.bars.firstDownbeat} s`;
+  return `${heard}; ${level}; a steady ${beat.bpm} BPM from ${beat.firstBeat} s, ${bars}`;
+}
+
+/**
+ * A recording measured for an agent that cannot hear it: { file, listened: { from, to }, summary, loudness: {
+ * integrated, range, peak, clipped }, sections: [{ start, end, lufs }], silences: [{ start, end }], rhythm, voice?,
+ * words? }, times in seconds from the start of the file. `rhythm` is { steady: true, bpm, beatSeconds, firstBeat,
+ * strayMs, beatStrength, bars: { beatsPerBar, firstDownbeat, sure } } or { steady: false, beats }. At most LONGEST
+ * seconds from `from`. Words come only with `speech` (the relay's speech module, when its models are downloaded).
+ * Throws when nothing can decode the file.
+ */
+export async function listen(file, { from = 0, to, speech = null }) {
+  const start = Math.max(0, Number(from) || 0);
+  const end = Number(to) > start ? Number(to) : start + LONGEST;
+  const seconds = Math.min(LONGEST, end - start);
+  const measured = await measure(
+    await decode(file, start, seconds),
+    Math.round(seconds * RATE)
+  );
+  const heardSeconds = measured.mono.length / LOW_RATE;
+  if (heardSeconds < 0.5)
+    throw new Error(`${path.basename(file)} has no sound there`);
+  const listened = { from: round(start), to: round(start + heardSeconds) };
+  const loudness = {
+    integrated: integratedLoudness(measured.steps),
+    range: loudnessRange(measured.steps),
+    peak: measured.peak > 0 ? round1(20 * Math.log10(measured.peak)) : null,
+    clipped: measured.clipped,
+  };
+  const quiet = silences(measured.steps, start);
+  const beat = await rhythm(measured.mono, start, quiet);
+  return {
+    file,
+    listened,
+    summary: summarise(listened, loudness, beat),
+    loudness,
+    sections: levelSections(measured.steps, start),
+    silences: quiet,
+    rhythm: beat,
+    ...(speech ? await words(speech, measured.mono, start) : {}),
+  };
+}