tryharness 0.7.28 → 0.7.29

package/lib/mcp-stdio.mjs

@@ -454,15 +454,20 @@ const LOCAL_TOOLS = [
     },
   },
   {
-    name: "search_media",
+    name: "search_files",
     description:
-      "Find the minute of a recording or the screenshot that is about something, in folders of recordings and screenshots the person added to the relay: calls, voice notes, screen captures. Give what it was about in your own words; you get each file with the time to start at and how close it is. Runs on this computer; nothing leaves it.",
+      "Find what is in the person's files and a thread's files, by what it is about or its exact words: PDFs (the page), documents (the line), screenshots and pictures, recordings and videos (the moment). Searches the folders the person added to the relay and, given a thread, the files attached there. You get each file's path, the place in it, a short excerpt and how it matched. Runs on this computer; nothing leaves it.",
     inputSchema: {
       type: "object",
       properties: {
         query: {
           type: "string",
-          description: "what was said or shown, in words",
+          description:
+            "what it is about, or the exact words, code or name in it",
+        },
+        thread: {
+          type: "string",
+          description: "a thread (goal) id, to search its files too",
         },
       },
       required: ["query"],
@@ -654,9 +659,10 @@ const BOARD_CALLS = {
     from: args.from,
     to: args.to,
   }),
-  search_media: (args) => ({
-    command: "board.media.search",
+  search_files: (args) => ({
+    command: "board.files.search",
     query: args.query,
+    thread: args.thread,
   }),
   search_screen: (args) => ({
     command: "board.screen.search",
@@ -934,7 +940,7 @@ const SEARCHES = [
   "search_room",
   "search_swarm",
   "search_code",
-  "search_media",
+  "search_files",
   "search_screen",
   "compare_screens",
 ];

package/lib/readers.mjs

@@ -0,0 +1,459 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { execFile, spawnSync } from "node:child_process";
+import { documentText } from "./files.mjs";
+import { windows } from "./media.mjs";
+import { decodeWav, encodeWav, resample } from "./speech.mjs";
+
+// What is in a file, as the parts the search model reads: a PDF's pages, a document's passages, a picture, a
+// recording's 30-second windows, a video's frames and its soundtrack. Each part says where it is in its file (a page,
+// a line, a moment), so a match points there. Only tools every Mac has are needed (PDFKit through osascript, textutil,
+// sips, afconvert, Quick Look); elsewhere poppler and ffmpeg are used when installed, and a file needing a missing tool
+// is skipped with the reason. Every tool runs as its own process without holding up the daemon.
+
+const RATE = 16_000;
+const MAX_PAGES = 300;
+const MAX_PASSAGES = 300;
+const PASSAGE_CHARS = 1_500;
+// A page with less text than this is read as a picture (a scan, a slide).
+const SCANNED_BELOW = 40;
+const MAX_FRAMES = 120;
+const FRAME_EVERY_S = 5;
+
+const KINDS = {
+  image: /\.(?:png|jpe?g|heic|heif|webp|gif|tiff?|bmp)$/i,
+  audio: /\.(?:wav|mp3|m4a|aac|caf|aiff?|flac|ogg|opus)$/i,
+  video: /\.(?:mp4|mov|m4v|webm|mkv|avi)$/i,
+  pdf: /\.pdf$/i,
+  doc: /\.(?:txt|md|markdown|rtf|html?|docx|pptx|odt|doc)$/i,
+};
+const PLAIN = /\.(?:txt|md|markdown)$/i;
+const MODEL_READS = /\.(?:png|jpe?g)$/i;
+
+/** The kind of file a name says it is: "image", "audio", "video", "pdf", "doc", or null for one not read. */
+export function fileKind(name) {
+  for (const [kind, pattern] of Object.entries(KINDS))
+    if (pattern.test(name)) return kind;
+  return null;
+}
+
+const has = (tool) =>
+  spawnSync(process.platform === "win32" ? "where" : "which", [tool], {
+    encoding: "utf8",
+  }).status === 0;
+const tools = {};
+const available = (tool) => (tools[tool] ??= has(tool));
+
+const scratch = () => fs.mkdtempSync(path.join(os.tmpdir(), "harness-read-"));
+
+/** Runs a tool to its end: { ok, stdout }. A tool that is missing or fails is ok: false. */
+const tool = (command, args, { timeout = 300_000 } = {}) =>
+  new Promise((resolve) =>
+    execFile(
+      command,
+      args,
+      { encoding: "utf8", maxBuffer: 256 * 2 ** 20, timeout },
+      (error, stdout) => resolve({ ok: !error, stdout: stdout ?? "", error })
+    )
+  );
+
+// PDFKit through JavaScript for Automation: every page's text, and a page drawn as a picture.
+const PDF_PAGES = `
+ObjC.import("PDFKit");
+function run(argv) {
+  const url = $.NSURL.fileURLWithPath(argv[0]);
+  const doc = $.PDFDocument.alloc.initWithURL(url);
+  if (!doc) return "null";
+  const pages = [];
+  const count = Math.min(doc.pageCount, Number(argv[1]));
+  for (let i = 0; i < count; i++)
+    pages.push(ObjC.unwrap(doc.pageAtIndex(i).string) || "");
+  return JSON.stringify({ pages, total: doc.pageCount });
+}`;
+const PDF_DRAW = `
+ObjC.import("PDFKit");
+ObjC.import("AppKit");
+function run(argv) {
+  const url = $.NSURL.fileURLWithPath(argv[0]);
+  const page = $.PDFDocument.alloc.initWithURL(url).pageAtIndex(Number(argv[1]));
+  const size = $.NSMakeSize(1024, 1024);
+  const image = page.thumbnailOfSizeForBox(size, $.kPDFDisplayBoxMediaBox);
+  const bitmap = $.NSBitmapImageRep.imageRepWithData(image.TIFFRepresentation);
+  const png = bitmap.representationUsingTypeProperties($.NSBitmapImageFileTypePNG, $());
+  png.writeToFileAtomically(argv[2], true);
+  return "ok";
+}`;
+const jxa = (script, args) =>
+  tool("osascript", ["-l", "JavaScript", "-e", script, ...args], {
+    timeout: 120_000,
+  });
+
+async function pdfPages(file) {
+  if (process.platform === "darwin") {
+    const read = await jxa(PDF_PAGES, [file, String(MAX_PAGES)]);
+    const parsed = read.ok ? JSON.parse(read.stdout || "null") : null;
+    if (!parsed) throw new Error("PDFKit could not open it");
+    return parsed.pages;
+  }
+  if (!available("pdftotext"))
+    throw new Error("reading PDFs here needs pdftotext (poppler-utils)");
+  const read = await tool("pdftotext", [
+    "-l",
+    String(MAX_PAGES),
+    "-layout",
+    file,
+    "-",
+  ]);
+  if (!read.ok) throw new Error("pdftotext could not read it");
+  return read.stdout.split("\f").slice(0, -1);
+}
+
+/** A PDF page drawn as a PNG, for a page with no text of its own. */
+async function pdfPicture(file, page) {
+  const dir = scratch();
+  try {
+    if (process.platform === "darwin") {
+      const out = path.join(dir, "page.png");
+      await jxa(PDF_DRAW, [file, String(page - 1), out]);
+      return fs.existsSync(out) ? fs.readFileSync(out) : null;
+    }
+    if (!available("pdftoppm")) return null;
+    const prefix = path.join(dir, "page");
+    await tool("pdftoppm", [
+      "-png",
+      "-r",
+      "100",
+      "-f",
+      String(page),
+      "-l",
+      String(page),
+      "-singlefile",
+      file,
+      prefix,
+    ]);
+    return fs.existsSync(`${prefix}.png`)
+      ? fs.readFileSync(`${prefix}.png`)
+      : null;
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** The words of a document as plain text, or null when this computer cannot read its format. */
+async function docText(file) {
+  if (PLAIN.test(file)) return fs.readFileSync(file, "utf8");
+  if (/\.html?$/i.test(file))
+    return fs
+      .readFileSync(file, "utf8")
+      .replace(/<(script|style)[\s\S]*?<\/\1>/gi, " ")
+      .replace(/<br\s*\/?>|<\/(p|div|li|h\d|tr)>/gi, "\n")
+      .replace(/<[^>]+>/g, " ")
+      .replace(/&nbsp;/g, " ");
+  if (process.platform === "darwin") {
+    const read = await tool("textutil", ["-convert", "txt", "-stdout", file]);
+    if (read.ok) return read.stdout;
+  }
+  const office = {
+    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  }[path.extname(file).slice(1).toLowerCase()];
+  return office ? documentText(file, office) : null;
+}
+
+/** Text cut into passages of whole paragraphs, each with the line it starts on. */
+export function passages(text, size = PASSAGE_CHARS) {
+  const lines = String(text).replace(/\r\n?/g, "\n").split("\n");
+  const out = [];
+  let current = [];
+  let length = 0;
+  let start = 1;
+  const flush = (nextLine) => {
+    const body = current.join("\n").trim();
+    if (body) out.push({ line: start, text: body });
+    current = [];
+    length = 0;
+    start = nextLine;
+  };
+  lines.forEach((line, index) => {
+    const full = length + line.length > size;
+    const paragraphEnd = !line.trim() && length > size * 0.6;
+    if (current.length && (full || paragraphEnd)) flush(index + 1);
+    current.push(line.slice(0, size));
+    length += line.length + 1;
+  });
+  flush(lines.length + 1);
+  return out.slice(0, MAX_PASSAGES);
+}
+
+/** Text that is mostly numbers (an export, a table) is found by its exact words only. */
+const mostlyNumbers = (text) => {
+  const digits = (text.match(/\d/g) ?? []).length;
+  const letters = (text.match(/\p{L}/gu) ?? []).length;
+  return digits > letters;
+};
+
+const textPart = (text, at) => ({
+  modality: "text",
+  text,
+  embed: !mostlyNumbers(text),
+  ...at,
+});
+
+/** A picture the search model reads (PNG or JPEG), converted when it is another format. */
+export async function pictureBytes(file) {
+  if (MODEL_READS.test(file)) return fs.promises.readFile(file);
+  const dir = scratch();
+  const out = path.join(dir, "picture.jpg");
+  try {
+    if (process.platform === "darwin")
+      await tool("sips", [
+        "-s",
+        "format",
+        "jpeg",
+        "-Z",
+        "1536",
+        file,
+        "--out",
+        out,
+      ]);
+    else if (available("ffmpeg"))
+      await tool("ffmpeg", ["-v", "error", "-y", "-i", file, out]);
+    if (!fs.existsSync(out))
+      throw new Error("this computer cannot convert that picture");
+    return fs.readFileSync(out);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+async function duration(file) {
+  if (!available("ffprobe")) return null;
+  const read = await tool("ffprobe", [
+    "-v",
+    "error",
+    "-show_entries",
+    "format=duration",
+    "-of",
+    "csv=p=0",
+    file,
+  ]);
+  const seconds = Number(read.stdout.trim());
+  return Number.isFinite(seconds) ? seconds : null;
+}
+
+/** A video's frames, one every few seconds (more apart in a long one): [{ atMs, data }]. */
+async function videoFrames(file, seconds) {
+  const dir = scratch();
+  try {
+    if (!available("ffmpeg")) {
+      // Without ffmpeg a Mac still has Quick Look's poster frame.
+      if (process.platform !== "darwin") return [];
+      await tool("qlmanage", ["-t", "-s", "768", "-o", dir, file]);
+      const poster = fs.readdirSync(dir).find((name) => name.endsWith(".png"));
+      return poster
+        ? [{ atMs: 0, data: fs.readFileSync(path.join(dir, poster)) }]
+        : [];
+    }
+    const every = Math.max(FRAME_EVERY_S, (seconds ?? 0) / MAX_FRAMES);
+    await tool("ffmpeg", [
+      "-v",
+      "error",
+      "-i",
+      file,
+      "-vf",
+      `fps=1/${every},scale='min(768,iw)':-2`,
+      "-frames:v",
+      String(MAX_FRAMES),
+      "-q:v",
+      "4",
+      path.join(dir, "f%04d.jpg"),
+    ]);
+    return fs
+      .readdirSync(dir)
+      .filter((name) => name.endsWith(".jpg"))
+      .sort()
+      .map((name, index) => ({
+        atMs: Math.round(index * every * 1000),
+        data: fs.readFileSync(path.join(dir, name)),
+      }));
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** A recording as 16 kHz mono samples: WAV read here; anything else converted by the OS's own tool or ffmpeg. */
+async function recording(file) {
+  if (/\.wav$/i.test(file)) {
+    const { sampleRate, samples } = decodeWav(await fs.promises.readFile(file));
+    return resample(samples, sampleRate, RATE);
+  }
+  const dir = scratch();
+  const out = path.join(dir, "audio.wav");
+  try {
+    const converted =
+      process.platform === "darwin"
+        ? await tool("afconvert", [
+            "-f",
+            "WAVE",
+            "-d",
+            `LEI16@${RATE}`,
+            "-c",
+            "1",
+            file,
+            out,
+          ])
+        : await tool("ffmpeg", [
+            "-v",
+            "error",
+            "-y",
+            "-i",
+            file,
+            "-ac",
+            "1",
+            "-ar",
+            String(RATE),
+            out,
+          ]);
+    if (!converted.ok || !fs.existsSync(out))
+      throw new Error(
+        converted.error?.code === "ENOENT"
+          ? "no audio converter on this computer (ffmpeg)"
+          : `could not read ${path.basename(file)}`
+      );
+    const { sampleRate, samples } = decodeWav(await fs.promises.readFile(out));
+    return resample(samples, sampleRate, RATE);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const soundParts = async (file) =>
+  windows(await recording(file)).map((window) => ({
+    modality: "audio",
+    data: encodeWav(window.samples, RATE),
+    loc: window.startMs,
+    locEnd: window.endMs,
+    embed: true,
+  }));
+
+/**
+ * A file's parts: [{ modality: "text" | "image" | "audio", text?, data?, loc, locEnd?, embed }]. loc is a page (PDF),
+ * a line (document) or milliseconds (recording, video); embed is false for text found by its words alone. Throws
+ * with the reason when this computer cannot read the file.
+ */
+export async function readParts(file, kind = fileKind(file)) {
+  if (kind === "image")
+    return [
+      {
+        modality: "image",
+        data: await pictureBytes(file),
+        loc: 0,
+        embed: true,
+      },
+    ];
+  if (kind === "audio") return soundParts(file);
+  if (kind === "video") {
+    let sound = [];
+    try {
+      sound = await soundParts(file);
+    } catch {}
+    const seconds = sound.length
+      ? sound.at(-1).locEnd / 1000
+      : await duration(file);
+    const frames = (await videoFrames(file, seconds)).map((frame) => ({
+      modality: "image",
+      data: frame.data,
+      loc: frame.atMs,
+      embed: true,
+    }));
+    if (!sound.length && !frames.length)
+      throw new Error("this computer cannot read that video (needs ffmpeg)");
+    return [...frames, ...sound];
+  }
+  if (kind === "pdf") {
+    const parts = [];
+    for (const [index, text] of (await pdfPages(file)).entries()) {
+      const page = index + 1;
+      if (text.trim().length >= SCANNED_BELOW) {
+        parts.push(textPart(text.trim().slice(0, 6_000), { loc: page }));
+        continue;
+      }
+      const picture = await pdfPicture(file, page);
+      if (picture)
+        parts.push({
+          modality: "image",
+          data: picture,
+          loc: page,
+          embed: true,
+        });
+    }
+    return parts;
+  }
+  if (kind === "doc") {
+    const text = await docText(file);
+    if (text === null) throw new Error("this computer cannot read that format");
+    return passages(text).map((passage) =>
+      textPart(passage.text, { loc: passage.line })
+    );
+  }
+  throw new Error("not a kind of file the relay reads");
+}
+
+/** A video's frame at a moment, as a JPEG, or null. */
+async function videoFrameAt(file, atMs) {
+  if (!available("ffmpeg"))
+    return atMs === 0 ? ((await videoFrames(file, 0))[0]?.data ?? null) : null;
+  const dir = scratch();
+  const out = path.join(dir, "frame.jpg");
+  try {
+    await tool("ffmpeg", [
+      "-v",
+      "error",
+      "-ss",
+      String(atMs / 1000),
+      "-i",
+      file,
+      "-frames:v",
+      "1",
+      "-vf",
+      "scale='min(1280,iw)':-2",
+      out,
+    ]);
+    return fs.existsSync(out) ? fs.readFileSync(out) : null;
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** One part again, read fresh from the file for sharing: { modality, text? | data? | samples } or null. */
+export async function readPart(file, { kind, modality, loc }) {
+  if (kind === "image")
+    return { modality: "image", data: await pictureBytes(file) };
+  if (kind === "video" && modality === "image") {
+    const data = await videoFrameAt(file, loc);
+    return data ? { modality: "image", data } : null;
+  }
+  if (kind === "pdf") {
+    const pages = await pdfPages(file);
+    const text = pages[loc - 1]?.trim() ?? "";
+    if (text.length >= SCANNED_BELOW) return { modality: "text", text };
+    const picture = await pdfPicture(file, loc);
+    return picture ? { modality: "image", data: picture } : null;
+  }
+  if (kind === "doc") {
+    const text = (await docText(file)) ?? "";
+    const found = passages(text).find((p) => p.line === loc);
+    return found ? { modality: "text", text: found.text } : null;
+  }
+  if (kind === "audio" || kind === "video") {
+    const samples = await recording(file);
+    const start = Math.round((loc / 1000) * RATE);
+    return {
+      modality: "audio",
+      samples: samples.subarray(start, start + 30 * RATE),
+      sampleRate: RATE,
+    };
+  }
+  return null;
+}

package/lib/service.mjs

@@ -5,6 +5,7 @@ import { spawn } from "node:child_process";
 import { resolveCommand, run } from "./commands.mjs";
 import { ensureDir } from "./files.mjs";
 import { paths } from "./paths.mjs";
+import { appProgram } from "./desktop.mjs";
 
 const LABEL = "ai.tryharness.daemon";
 
@@ -183,6 +184,17 @@ function retireLegacyBridge() {
   }
 }
 
+/** Whether the Harness app's launcher is here on a Mac but the daemon does not start through it yet. */
+export function launcherMissing(platform = process.platform) {
+  if (platform !== "darwin" || !appProgram("HarnessDaemon", { platform }))
+    return false;
+  try {
+    return !fs.readFileSync(servicePath(), "utf8").includes("HarnessDaemon");
+  } catch {
+    return false;
+  }
+}
+
 export function installService({
   runtimeRoot,
   nodePath = process.execPath,
@@ -197,6 +209,14 @@ export function installService({
     return installWindowsTask({ runtimeRoot, nodePath, execute });
   if (platform === "darwin") {
     const plist = servicePath();
+    // Through the Harness app's launcher when the app is here, so macOS asks for the person's folders as Harness.
+    const launcher = appProgram("HarnessDaemon", { platform });
+    const program = [
+      ...(launcher ? [launcher] : []),
+      nodePath,
+      entry,
+      "daemon",
+    ];
     ensureDir(path.dirname(plist));
     // MallocLargeCache=0: macOS otherwise keeps the large blocks ONNX Runtime frees (a screenshot's OCR held 1.6 GB after
     // it finished; 255 MB with it off, image-redaction eval).
@@ -206,7 +226,7 @@ export function installService({
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0"><dict>
   <key>Label</key><string>${LABEL}</string>
-  <key>ProgramArguments</key><array><string>${xml(nodePath)}</string><string>${xml(entry)}</string><string>daemon</string></array>
+  <key>ProgramArguments</key><array>${program.map((arg) => `<string>${xml(arg)}</string>`).join("")}</array>
   <key>EnvironmentVariables</key><dict><key>PATH</key><string>${xml(envPath)}</string><key>MallocLargeCache</key><string>0</string></dict>
   <key>RunAtLoad</key><true/><key>KeepAlive</key><true/><key>ThrottleInterval</key><integer>5</integer>
   <key>StandardOutPath</key><string>${xml(paths.log)}</string><key>StandardErrorPath</key><string>${xml(paths.log)}</string>