tryharness 0.7.26 → 0.7.29

package/lib/library.mjs

@@ -0,0 +1,693 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { ATLAS_VERSION } from "./atlas.mjs";
+import { fileKind, readParts, readPart } from "./readers.mjs";
+import { clock } from "./media.mjs";
+import { encodeWav } from "./speech.mjs";
+
+// The library: what is in the person's files and in the files their threads hold, found by what it is about and by
+// its exact words. Folders the person added (`tryharness local add`) are read on the background lane: PDFs page by
+// page, documents in passages, pictures, recordings in 30-second windows, videos as frames and soundtrack. A thread's
+// files are read as they arrive. Everything stays in the atlas on this computer.
+//
+// Ranking follows DigUp (github.com/ARahim3/DigUp, MIT), which measured it on its own evals: every part is scored
+// against the question, and a part counts by meaning only when it stands out from all the others for that question
+// (its score in standard deviations, after each kind of part is centred on its own average). Words count beside
+// meaning, so a code or a name is found as written. Standing out is what lets a search say "nothing here".
+
+const MAX_FILES = 5_000;
+// A folder is walked this far to find its newest files.
+const MAX_WALK = 50_000;
+const MAX_DEPTH = 8;
+const MAX_BYTES = { image: 20e6, pdf: 200e6, doc: 20e6 };
+// A part counts by meaning from this many standard deviations above the rest, for a search an agent asked for.
+const Z_GATE = 2.0;
+// A brief nobody asked for takes a file only when it stands out much further, and close to the best one: among 268
+// files, right answers stood 3.9 to 8.2 above the rest and the best answer to a task about something never written 3.9
+// at most, while an unrelated README in second place trailed the right file by more than a point (eval file-search).
+const BRIEF_Z = 4.0;
+const BRIEF_NEAR_BEST = 1.0;
+// Below this many parts the spread says little, and a fixed similarity stands in (eval media-search: right answers
+// 0.68 and up, never-said questions 0.675 at most).
+const MIN_SCALE = 200;
+const FLOOR = { image: 0.7, audio: 0.7, text: 0.7 };
+const SMALL_GROUP = 20;
+const WORD_WEIGHT = 2.0;
+const VECTORS_IDLE_MS = 5 * 60_000;
+// A change in a searched folder is read once the folder has been quiet this long (a download writes for a while).
+const CHANGE_QUIET_MS = 5_000;
+
+const SKIP_DIR =
+  /^(?:\..*|node_modules|Library|\.Trash|__pycache__|venv|site-packages|.*\.(?:app|photoslibrary|noindex|bundle|framework))$/i;
+const CODE_MARKERS = [
+  ".git",
+  "package.json",
+  "Cargo.toml",
+  "go.mod",
+  "pyproject.toml",
+];
+const SECRET_NAME =
+  /^(?:id_(?:rsa|dsa|ecdsa|ed25519)\w*|\.env.*|wallet\.dat)$|\.(?:pem|key|p12|pfx|kdbx|keychain(?:-db)?|gpg|asc|ovpn|ppk|jks|keystore)$|(?:^|[-_. ])(?:credentials?|secrets?|passwords?|recovery[-_ ]codes?|seed[-_ ]?phrase)(?:[-_. ]|$)/i;
+const SCREENSHOT = /screen ?shot|cleanshot|capture|^scr[-_ ]?\d/i;
+const MACHINE_NAME =
+  /^(?:[a-z]+[-_]?)?\d{3,}$|^[0-9a-f]{8}-[0-9a-f]{4}-|^[0-9a-f]{16,}$/i;
+const DATASET_FILES = 200;
+
+const STOP = new Set(
+  "a an and are as at be by can did do does find for from had has have how i in is it its me my of on or our show that the their them they this to was we were what when where which who why with you your".split(
+    " "
+  )
+);
+// Words that say what kind of file to look for, not what is in it.
+const KIND_WORDS = {
+  image: [
+    "screenshot",
+    "screenshots",
+    "picture",
+    "pictures",
+    "photo",
+    "photos",
+    "image",
+    "images",
+  ],
+  audio: [
+    "recording",
+    "recordings",
+    "call",
+    "voice",
+    "audio",
+    "podcast",
+    "memo",
+  ],
+  video: ["video", "videos", "clip", "clips", "movie"],
+  pdf: ["pdf", "pdfs"],
+  doc: ["document", "documents", "doc", "docs", "notes", "note"],
+};
+
+/** Why a file is never read, or null. Keys and password files never; files that only live in iCloud are not downloaded. */
+export function skipReason(file, stat) {
+  const name = path.basename(file);
+  if (SECRET_NAME.test(name)) return "looks like a key or password file";
+  if (name.endsWith(".icloud")) return "only in iCloud";
+  // A file the cloud took back keeps its size but no blocks on the disk; reading it would download it.
+  if (stat.size > 0 && stat.blocks === 0) return "only in the cloud";
+  const kind = fileKind(name);
+  if (!kind) return "not a kind of file the relay reads";
+  if (stat.size > (MAX_BYTES[kind] ?? Infinity)) return "too large";
+  return null;
+}
+
+/** Search words: lowercased, no stopwords or file-kind words. */
+export function terms(text) {
+  const kindWords = new Set(Object.values(KIND_WORDS).flat());
+  const words =
+    String(text)
+      .toLowerCase()
+      .match(/[\p{L}\p{N}_-]+/gu) ?? [];
+  return [...new Set(words)]
+    .filter(
+      (word) => word.length > 1 && !STOP.has(word) && !kindWords.has(word)
+    )
+    .slice(0, 12);
+}
+
+/** Codes and IDs ("INV-20931", "K7Q2LM") mean nothing to an embedding: a query made only of them leans on words. */
+export const isLookup = (words) =>
+  words.length > 0 &&
+  words.every(
+    (word) => /\d/.test(word) && (/\p{L}/u.test(word) || word.length >= 4)
+  );
+
+const hintedKinds = (text) => {
+  const words = new Set(
+    String(text)
+      .toLowerCase()
+      .match(/\p{L}+/gu) ?? []
+  );
+  return new Set(
+    Object.entries(KIND_WORDS)
+      .filter(([, list]) => list.some((word) => words.has(word)))
+      .map(([kind]) => kind)
+  );
+};
+
+const phrase = (word) => `"${word.replaceAll('"', '""')}"`;
+
+/**
+ * Meaning scores on one scale for this query (DigUp's Searcher.meaning): each part's similarity minus the average of
+ * its modality, in standard deviations of all of them together.
+ */
+export function standOut(scores, modalities) {
+  const sums = new Map();
+  for (let i = 0; i < scores.length; i += 1) {
+    const entry = sums.get(modalities[i]) ?? { sum: 0, count: 0 };
+    entry.sum += scores[i];
+    entry.count += 1;
+    sums.set(modalities[i], entry);
+  }
+  let total = 0;
+  for (const entry of sums.values()) total += entry.sum;
+  const overall = total / Math.max(scores.length, 1);
+  const meanOf = (modality) => {
+    const entry = sums.get(modality);
+    return entry.count >= SMALL_GROUP ? entry.sum / entry.count : overall;
+  };
+  const centred = scores.map((score, i) => score - meanOf(modalities[i]));
+  const mean = centred.reduce((a, b) => a + b, 0) / Math.max(centred.length, 1);
+  const variance =
+    centred.reduce((sum, value) => sum + (value - mean) ** 2, 0) /
+    Math.max(centred.length, 1);
+  const deviation = Math.max(Math.sqrt(variance), 0.02);
+  return centred.map((value) => (value - mean) / deviation);
+}
+
+/** Where in its file a part is, in words: "page 3", "line 40", "1:20". */
+export function placeOf(part) {
+  if (part.kind === "pdf") return `page ${part.loc}`;
+  if (part.kind === "doc") return `line ${part.loc}`;
+  if (part.kind === "audio" || part.kind === "video") return clock(part.loc);
+  return null;
+}
+
+/**
+ * The newest files under a folder the library reads (up to max), and what was passed over and why. Asynchronous: the
+ * first read of a folder macOS protects waits for the person's answer to its prompt, and must not hold up the daemon.
+ */
+export async function crawl(
+  folder,
+  { max = MAX_FILES, walkAtMost = MAX_WALK } = {}
+) {
+  const files = [];
+  const skipped = [];
+  const walk = async (dir, depth, top) => {
+    if (depth > MAX_DEPTH || files.length >= walkAtMost) return;
+    let entries = [];
+    try {
+      entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      // macOS keeps Desktop, Documents and Downloads from a program the person has not allowed.
+      if (top && (error.code === "EPERM" || error.code === "EACCES"))
+        skipped.push({
+          path: dir,
+          reason: "Harness is not allowed to read it",
+        });
+      return;
+    }
+    // A code project is the code index's (atlas.mjs): here only its screenshots are read.
+    const project =
+      !top && entries.some((entry) => CODE_MARKERS.includes(entry.name));
+    const found = [];
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (!SKIP_DIR.test(entry.name)) await walk(full, depth + 1, false);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      if (project && !SCREENSHOT.test(entry.name)) continue;
+      let stat;
+      try {
+        stat = await fs.promises.stat(full);
+      } catch {
+        continue;
+      }
+      const reason = skipReason(full, stat);
+      if (reason) {
+        if (reason !== "not a kind of file the relay reads")
+          skipped.push({ path: full, reason });
+        continue;
+      }
+      found.push({
+        path: full,
+        kind: fileKind(entry.name),
+        mtime: stat.mtimeMs,
+        size: stat.size,
+      });
+    }
+    // Hundreds of machine-named files of one kind are training data, not the person's own: passed over and said.
+    const machineMade = found.filter((file) =>
+      MACHINE_NAME.test(path.parse(file.path).name)
+    );
+    if (!top && machineMade.length >= DATASET_FILES) {
+      skipped.push({ path: dir, reason: "looks like a dataset" });
+      return;
+    }
+    files.push(...found);
+  };
+  await walk(folder, 0, true);
+  const newest = files.sort((a, b) => b.mtime - a.mtime).slice(0, max);
+  return { files: newest, skipped };
+}
+
+const partKey = (contentKey, kind, part) =>
+  createHash("sha1")
+    .update(
+      // Recordings and pictures keep the keys the media index gave them, so their vectors carry over.
+      kind === "audio" || kind === "image"
+        ? `${contentKey}\0${part.loc}`
+        : `${contentKey}\0${part.modality}\0${part.loc}`
+    )
+    .digest("hex");
+
+export function createLibrary({
+  atlas,
+  inference,
+  folders = () => [],
+  threadFiles = () => [],
+  mediaReady = () => false,
+  wantMedia = () => {},
+  ocr = null,
+  mask = async (text) => text,
+  log = () => {},
+  version = ATLAS_VERSION,
+}) {
+  const store = atlas.library;
+  let cache = null;
+  let dropCache = null;
+  let refreshing = null;
+  let skipped = [];
+
+  const vectors = () => {
+    cache ??= store.vectors(version);
+    clearTimeout(dropCache);
+    dropCache = setTimeout(() => (cache = null), VECTORS_IDLE_MS);
+    dropCache.unref?.();
+    return cache;
+  };
+
+  /** The words a picture is found by: its text, read here (screenshots and other PNGs). */
+  const wordsIn = async (file, kind, part, lane) => {
+    if (part.modality === "text") return part.text;
+    const screenshotLike =
+      SCREENSHOT.test(path.basename(file)) || /\.png$/i.test(file);
+    if (!ocr || kind !== "image" || !screenshotLike) return "";
+    try {
+      const read = await ocr.read(part.data, { lane });
+      return read.lines.map((line) => line.text).join("\n");
+    } catch {
+      return "";
+    }
+  };
+
+  /** Reads one file into the library: its parts and their words now, their vectors as the model gets to them. */
+  const add = async (
+    { file, readFrom, source, name, kind, mtime, contentKey },
+    lane
+  ) => {
+    const parts = await readParts(readFrom, kind);
+    const rows = [];
+    for (const part of parts)
+      rows.push({
+        file,
+        vec: partKey(contentKey, kind, part),
+        source,
+        name,
+        kind,
+        modality: part.modality,
+        loc: part.loc,
+        locEnd: part.locEnd,
+        mtime,
+        embed: part.embed,
+        text: await wordsIn(readFrom, kind, part, lane),
+      });
+    atlas.batch(() => {
+      store.drop(file);
+      store.put(rows);
+    });
+    if (mediaReady()) await embedMedia(parts, rows, lane);
+    cache = null;
+    return parts;
+  };
+
+  /** Embeds a file's pictures and sound from the bytes just read. */
+  const embedMedia = async (parts, rows, lane) => {
+    for (const [i, part] of parts.entries()) {
+      if (part.modality === "text") continue;
+      const data = part.data ?? encodeWav(part.samples, part.sampleRate);
+      const [vector] = await inference.embedMedia(
+        [{ title: rows[i].name, data }],
+        { lane }
+      );
+      atlas.putVector(rows[i].vec, version, vector);
+    }
+  };
+
+  /** Embeds what is waiting: text parts always, pictures and sound once the image and audio part is here. */
+  const embedWaiting = async (lane) => {
+    let done = 0;
+    for (;;) {
+      const texts = store.toEmbed(version, ["text"], 16);
+      if (!texts.length) break;
+      const embedded = await inference.embed(
+        texts.map((part) => ({
+          kind: "document",
+          title: part.name,
+          text: part.text,
+        })),
+        { lane }
+      );
+      atlas.batch(() =>
+        texts.forEach((part, i) =>
+          atlas.putVector(part.vec, version, embedded[i])
+        )
+      );
+      done += texts.length;
+    }
+    // Pictures and sound read before the image and audio part came down: each file read again once.
+    const tried = new Set();
+    while (mediaReady()) {
+      const [waiting] = store.toEmbed(version, ["image", "audio"], 1);
+      if (!waiting || tried.has(waiting.file)) break;
+      tried.add(waiting.file);
+      const from = sourceOf(waiting);
+      try {
+        await add({ ...fileOf(waiting), readFrom: from }, lane);
+        done += 1;
+      } catch (error) {
+        log(`library: ${waiting.name}: ${error.message}`);
+        store.drop(waiting.file);
+      }
+    }
+    if (done) cache = null;
+    return done;
+  };
+
+  // Where a part's file is read from now: a folder file's own path, a thread file's stored copy.
+  const threadPaths = new Map();
+  const sourceOf = (part) =>
+    part.source === "folders" ? part.file : threadPaths.get(part.file);
+  const contentKeys = new Map();
+  const fileOf = (part) => ({
+    file: part.file,
+    source: part.source,
+    name: part.name,
+    kind: part.kind,
+    mtime: part.mtime,
+    contentKey:
+      part.source === "folders"
+        ? `${part.file}\0${part.mtime}`
+        : contentKeys.get(part.file),
+  });
+
+  // A file that could not be read is remembered by its name, so it is not tried again until it changes.
+  const unreadable = (file, source) =>
+    store.put([
+      {
+        file: file.path,
+        vec: "unreadable",
+        source,
+        name: file.name ?? path.basename(file.path),
+        kind: file.kind ?? fileKind(file.name ?? file.path),
+        modality: "none",
+        loc: 0,
+        mtime: file.mtime,
+        embed: false,
+      },
+    ]);
+
+  // Each searched folder is watched, so a new or changed file is read seconds after it lands.
+  const watchers = new Map();
+  let quiet = null;
+  const changed = (_, name) => {
+    const parts = String(name ?? "").split(path.sep);
+    if (parts.some((part) => SKIP_DIR.test(part))) return;
+    clearTimeout(quiet);
+    quiet = setTimeout(() => {
+      const running = refreshing ?? Promise.resolve();
+      void running.then(() => refresh()).catch(() => {});
+    }, CHANGE_QUIET_MS);
+    quiet.unref?.();
+  };
+  const watch = (wanted) => {
+    for (const [folder, watcher] of watchers)
+      if (!wanted.includes(folder)) {
+        watcher.close();
+        watchers.delete(folder);
+      }
+    for (const folder of wanted) {
+      if (watchers.has(folder)) continue;
+      try {
+        const watcher = fs.watch(
+          folder,
+          { recursive: true, persistent: false },
+          changed
+        );
+        watcher.on("error", () => {
+          watcher.close();
+          watchers.delete(folder);
+        });
+        watchers.set(folder, watcher);
+      } catch {}
+    }
+  };
+
+  /** Brings the library in step with the searched folders and the threads' files, then embeds what waits. */
+  const refresh = ({ lane = "background" } = {}) =>
+    (refreshing ??= (async () => {
+      let added = 0;
+      const seenSkips = [];
+      // The person's folders.
+      const searched = folders();
+      const listed = [];
+      for (const folder of searched) {
+        const found = await crawl(folder);
+        seenSkips.push(...found.skipped);
+        listed.push(...found.files);
+      }
+      // Watched once walked, so the first look into a folder macOS protects is the walk's.
+      watch(searched);
+      const known = store.files("folders");
+      const present = new Set(listed.map((file) => file.path));
+      for (const file of known.keys()) if (!present.has(file)) store.drop(file);
+      for (const file of listed) {
+        if (known.get(file.path) === file.mtime) continue;
+        try {
+          await add(
+            {
+              file: file.path,
+              readFrom: file.path,
+              source: "folders",
+              name: path.basename(file.path),
+              kind: file.kind,
+              mtime: file.mtime,
+              contentKey: `${file.path}\0${file.mtime}`,
+            },
+            lane
+          );
+          added += 1;
+        } catch (error) {
+          seenSkips.push({ path: file.path, reason: error.message });
+          log(`library: ${path.basename(file.path)}: ${error.message}`);
+          unreadable(file, "folders");
+        }
+      }
+      // The threads' files: stored once and never changed, so each is read once.
+      const threadList = threadFiles().filter((file) => fileKind(file.name));
+      threadPaths.clear();
+      for (const file of threadList) {
+        threadPaths.set(file.id, file.path);
+        contentKeys.set(file.id, file.sha256);
+      }
+      const keep = new Set(threadList.map((file) => file.id));
+      for (const source of store.sources())
+        if (source !== "folders")
+          for (const file of store.files(source).keys())
+            if (!keep.has(file)) store.drop(file);
+      for (const file of threadList) {
+        if (store.files(file.thread).has(file.id)) continue;
+        try {
+          await add(
+            {
+              file: file.id,
+              readFrom: file.path,
+              source: file.thread,
+              name: file.name,
+              kind: fileKind(file.name),
+              mtime: 0,
+              contentKey: file.sha256,
+            },
+            lane
+          );
+          added += 1;
+        } catch (error) {
+          log(`library: thread file ${file.name}: ${error.message}`);
+          unreadable({ ...file, path: file.id, mtime: 0 }, file.thread);
+        }
+      }
+      skipped = seenSkips;
+      const embedded = await embedWaiting(lane);
+      // Pictures and sound wait for the image and audio part of the search model.
+      if (!mediaReady() && store.toEmbed(version, ["image", "audio"], 1).length)
+        wantMedia();
+      return {
+        files: listed.length + threadList.length,
+        added,
+        embedded,
+        skipped: skipped.length,
+      };
+    })().finally(() => {
+      refreshing = null;
+    }));
+
+  /**
+   * The files closest to a question, best first, one hit per file at its best part. scope: folders (the person's added
+   * folders: agents on this computer only) and threads (thread ids whose files to search); allow(file) drops thread
+   * files a caller may not read. brief: only hits that stand out, for a brief nobody asked for.
+   */
+  const search = async (
+    query,
+    {
+      folders: inFolders = false,
+      threads = [],
+      allow = () => true,
+      limit = 8,
+      lane = "interactive",
+      brief = false,
+    } = {}
+  ) => {
+    const text = String(query).slice(0, 4_000);
+    const inScope = (part) =>
+      part.source === "folders"
+        ? inFolders
+        : threads.includes(part.source) && allow(part.file);
+    const words = terms(text);
+    const lookup = isLookup(words);
+
+    // Meaning: every part scored, so the spread of scores says what stands out for this question.
+    const rows = vectors();
+    const best = new Map();
+    if (rows.length) {
+      const [q] = await inference.embed([{ kind: "query", text }], { lane });
+      const scores = rows.map((row) => {
+        let dot = 0;
+        for (let i = 0; i < q.length; i += 1) dot += q[i] * row.vector[i];
+        return dot;
+      });
+      const z = standOut(
+        scores,
+        rows.map((row) => row.modality)
+      );
+      const scaled = rows.length >= MIN_SCALE;
+      const gate = brief ? BRIEF_Z : Z_GATE;
+      rows.forEach((row, i) => {
+        if (!inScope(row)) return;
+        const strong = scaled ? z[i] >= gate : scores[i] >= FLOOR[row.modality];
+        const held = best.get(row.file);
+        if (!held || z[i] > held.z)
+          best.set(row.file, { part: row, z: z[i], cosine: scores[i], strong });
+      });
+    }
+
+    // Words: a file holding at least half the question's words qualifies, a code or name matched as written. A brief's
+    // request is long, so there only the codes it names count, and one found is enough.
+    const counted = brief
+      ? words.filter((word) => word.length >= 5 && isLookup([word]))
+      : words;
+    const needed = brief ? 1 / Math.max(counted.length, 1) : 0.5;
+    const coverage = new Map();
+    const wordPart = new Map();
+    for (const word of counted) {
+      for (const [file, vecs] of store.holding(phrase(word))) {
+        coverage.set(file, (coverage.get(file) ?? 0) + 1 / counted.length);
+        if (vecs[0] && !wordPart.has(file)) wordPart.set(file, vecs[0]);
+      }
+    }
+
+    const [meaningWeight, wordsWeight] = lookup
+      ? [0.5, WORD_WEIGHT * 2]
+      : [1, WORD_WEIGHT];
+    const hinted = hintedKinds(text);
+    const hits = [];
+    for (const file of new Set([...best.keys(), ...coverage.keys()])) {
+      const meaning = best.get(file);
+      const share = coverage.get(file) ?? 0;
+      const byWords = share >= needed - 1e-9;
+      if (!meaning?.strong && !byWords) continue;
+      // Point at the part that stands out by meaning, else the one the words are in, else the file's first.
+      const vec = meaning?.strong
+        ? meaning.part.vec
+        : (wordPart.get(file) ?? meaning?.part.vec);
+      const part = vec ? store.part(file, vec) : store.first(file);
+      if (!part || !inScope(part)) continue;
+      let score =
+        meaningWeight * Math.max(meaning?.z ?? 0, 0) +
+        wordsWeight * share ** 1.5;
+      if (hinted.has(part.kind)) score += 1;
+      hits.push({ part, score, meaning, byWords });
+    }
+    // A brief's files stand close to the best one; a file found by a code it names always counts.
+    const bestZ = Math.max(...hits.map((hit) => hit.meaning?.z ?? -Infinity));
+    const kept = hits.filter(
+      (hit) =>
+        !brief ||
+        hit.byWords ||
+        !rows.length ||
+        rows.length < MIN_SCALE ||
+        hit.meaning.z >= bestZ - BRIEF_NEAR_BEST
+    );
+    const top = kept.sort((a, b) => b.score - a.score).slice(0, limit);
+    // An excerpt goes into an agent's context: a key in it is masked first.
+    const excerpts = await Promise.all(
+      top.map(({ part }) => {
+        const excerpt = excerptOf(part.text, words);
+        return excerpt ? mask(excerpt) : null;
+      })
+    );
+    return top.map(({ part, score, meaning, byWords }, i) => ({
+      file: part.file,
+      thread: part.source === "folders" ? null : part.source,
+      name: part.name,
+      kind: part.kind,
+      modality: part.modality,
+      loc: part.loc,
+      place: placeOf(part),
+      excerpt: excerpts[i],
+      score: Math.round(score * 100) / 100,
+      standsOut: meaning?.strong ? Math.round(meaning.z * 10) / 10 : null,
+      similarity: meaning ? Math.round(meaning.cosine * 1000) / 1000 : null,
+      byWords,
+      vec: part.vec,
+    }));
+  };
+
+  /** One part's content, read fresh from its file for sharing: { modality, text? | data? | samples } or null. */
+  const content = async (file, vec) => {
+    const part = store.part(file, vec);
+    const from = part && sourceOf(part);
+    return from ? { part, ...(await readPart(from, part)) } : null;
+  };
+
+  return {
+    refresh,
+    search,
+    content,
+    stop() {
+      clearTimeout(quiet);
+      clearTimeout(dropCache);
+      for (const watcher of watchers.values()) watcher.close();
+      watchers.clear();
+    },
+    status: () => ({
+      folders: folders(),
+      files: store.count(),
+      folderFiles: store.files("folders").size,
+      waiting: store.waiting(version),
+      skipped: skipped.slice(0, 50),
+    }),
+  };
+}
+
+const excerptOf = (text, words, size = 300) => {
+  const body = String(text || "")
+    .replace(/\s+/g, " ")
+    .trim();
+  if (!body) return null;
+  const lower = body.toLowerCase();
+  const at = words
+    .map((word) => lower.indexOf(word))
+    .filter((index) => index >= 0)
+    .sort((a, b) => a - b)[0];
+  const start = at === undefined ? 0 : Math.max(0, at - 80);
+  return `${start ? "…" : ""}${body.slice(start, start + size)}${start + size < body.length ? "…" : ""}`;
+};