tryharness 0.7.22 → 0.7.25

package/lib/media.mjs

@@ -0,0 +1,228 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { ATLAS_VERSION } from "./atlas.mjs";
+import { decodeWav, encodeWav, resample } from "./speech.mjs";
+
+// Recordings and screenshots in folders the person adds (`tryharness local add <folder>`; nothing is read until they
+// do), found by what they are about: the search model's image and audio part puts each 30-second window of a recording
+// and each image in the same space as a typed question, so "the threshold we agreed on the call" finds the minute it was
+// said. Windows overlap by 10 seconds so a sentence is never cut in two. Indexed on the background lane, first passes on
+// the charger; everything stays in the context engine's atlas on this computer.
+
+const WINDOW_MS = 30_000;
+const STEP_MS = 20_000;
+const RATE = 16_000;
+const AUDIO = /\.(?:wav|mp3|m4a|aac|caf|aiff?|flac)$/i;
+const IMAGE = /\.(?:png|jpe?g)$/i;
+const SKIP_DIR = /^(?:\..*|node_modules|Library|\.Trash)$/;
+const MAX_FILES = 5_000;
+const MAX_IMAGE_BYTES = 20 * 2 ** 20;
+const MAX_DEPTH = 6;
+
+/** The media files under a folder, newest first. */
+export function mediaFiles(folder, { max = MAX_FILES } = {}) {
+  const out = [];
+  const walk = (dir, depth) => {
+    if (depth > MAX_DEPTH || out.length >= max) return;
+    let entries = [];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (out.length >= max) return;
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (!SKIP_DIR.test(entry.name)) walk(full, depth + 1);
+        continue;
+      }
+      if (
+        !entry.isFile() ||
+        !(AUDIO.test(entry.name) || IMAGE.test(entry.name))
+      )
+        continue;
+      try {
+        const stat = fs.statSync(full);
+        out.push({
+          path: full,
+          kind: AUDIO.test(entry.name) ? "audio" : "image",
+          mtime: stat.mtimeMs,
+          size: stat.size,
+        });
+      } catch {}
+    }
+  };
+  walk(folder, 0);
+  return out.sort((a, b) => b.mtime - a.mtime);
+}
+
+/** A recording as 16 kHz mono samples: WAV read here; anything else converted by the OS's own tool (afconvert) or ffmpeg. */
+export function readRecording(file, { run = spawnSync } = {}) {
+  if (/\.wav$/i.test(file)) {
+    const { sampleRate, samples } = decodeWav(fs.readFileSync(file));
+    return resample(samples, sampleRate, RATE);
+  }
+  const out = path.join(
+    fs.mkdtempSync(path.join(os.tmpdir(), "harness-media-")),
+    "audio.wav"
+  );
+  try {
+    const converted =
+      process.platform === "darwin"
+        ? run(
+            "afconvert",
+            ["-f", "WAVE", "-d", `LEI16@${RATE}`, "-c", "1", file, out],
+            { encoding: "utf8" }
+          )
+        : run(
+            "ffmpeg",
+            [
+              "-v",
+              "error",
+              "-y",
+              "-i",
+              file,
+              "-ac",
+              "1",
+              "-ar",
+              String(RATE),
+              out,
+            ],
+            { encoding: "utf8" }
+          );
+    if (converted.status !== 0 || !fs.existsSync(out))
+      throw new Error(
+        converted.error?.code === "ENOENT"
+          ? "no audio converter on this computer (ffmpeg)"
+          : `could not read ${path.basename(file)}`
+      );
+    const { sampleRate, samples } = decodeWav(fs.readFileSync(out));
+    return resample(samples, sampleRate, RATE);
+  } finally {
+    fs.rmSync(path.dirname(out), { recursive: true, force: true });
+  }
+}
+
+/** A recording's windows: { startMs, endMs, samples }, 30 seconds each, a new one every 20. */
+export function windows(
+  samples,
+  { windowMs = WINDOW_MS, stepMs = STEP_MS, rate = RATE } = {}
+) {
+  const size = Math.round((windowMs / 1000) * rate),
+    step = Math.round((stepMs / 1000) * rate);
+  const out = [];
+  for (let at = 0; at < samples.length; at += step) {
+    out.push({
+      startMs: Math.round((at / rate) * 1000),
+      endMs: Math.round((Math.min(samples.length, at + size) / rate) * 1000),
+      samples: samples.subarray(at, at + size),
+    });
+    if (at + size >= samples.length) break;
+  }
+  return out;
+}
+
+export const clock = (ms) => {
+  const s = Math.floor(ms / 1000);
+  const h = Math.floor(s / 3600);
+  return `${h ? `${h}:` : ""}${String(Math.floor(s / 60) % 60).padStart(h ? 2 : 1, "0")}:${String(s % 60).padStart(2, "0")}`;
+};
+
+export function createMediaIndex({
+  atlas,
+  inference,
+  folders = () => [],
+  log = () => {},
+  version = ATLAS_VERSION,
+}) {
+  let cache = null;
+  const vectors = () => (cache ??= atlas.mediaVectors(version));
+
+  /** Brings the added folders' media into the atlas; files gone or changed are dropped and read again. */
+  const refresh = async ({ lane = "background" } = {}) => {
+    const files = folders().flatMap((folder) => mediaFiles(folder));
+    const present = new Set(files.map((file) => file.path));
+    for (const gone of atlas.mediaPaths())
+      if (!present.has(gone)) atlas.dropMedia(gone);
+    let added = 0;
+    for (const file of files) {
+      if (atlas.mediaSeen(file.path) === file.mtime) continue;
+      try {
+        const parts =
+          file.kind === "image"
+            ? file.size <= MAX_IMAGE_BYTES
+              ? [{ startMs: 0, endMs: 0, data: fs.readFileSync(file.path) }]
+              : []
+            : windows(readRecording(file.path)).map((window) => ({
+                startMs: window.startMs,
+                endMs: window.endMs,
+                data: encodeWav(window.samples, RATE),
+              }));
+        const embedded = await inference.embedMedia(
+          parts.map((part) => ({
+            title: path.basename(file.path),
+            data: part.data,
+          })),
+          { lane }
+        );
+        atlas.dropMedia(file.path);
+        atlas.batch(() =>
+          parts.forEach((part, i) => {
+            const hash = createHash("sha1")
+              .update(`${file.path}\0${file.mtime}\0${part.startMs}`)
+              .digest("hex");
+            atlas.putMedia({
+              hash,
+              path: file.path,
+              kind: file.kind,
+              startMs: part.startMs,
+              endMs: part.endMs,
+              mtime: file.mtime,
+            });
+            atlas.putVector(hash, version, embedded[i]);
+          })
+        );
+        added += 1;
+        cache = null;
+      } catch (error) {
+        log(`media: ${path.basename(file.path)}: ${error.message}`);
+      }
+    }
+    return { files: files.length, added };
+  };
+
+  /** The recordings' windows and images closest to a question: [{ path, kind, startMs, endMs, at, score }], best first, one per file. */
+  const search = async (query, { limit = 5, lane = "interactive" } = {}) => {
+    const rows = vectors();
+    if (!rows.length) return [];
+    const [q] = await inference.embed(
+      [{ kind: "query", text: String(query).slice(0, 2_000) }],
+      { lane }
+    );
+    const best = new Map();
+    for (const row of rows) {
+      let dot = 0;
+      for (let i = 0; i < q.length; i += 1) dot += q[i] * row.vector[i];
+      if ((best.get(row.path)?.score ?? -1) < dot)
+        best.set(row.path, {
+          path: row.path,
+          kind: row.kind,
+          startMs: row.startMs,
+          endMs: row.endMs,
+          at: row.kind === "audio" ? clock(row.startMs) : null,
+          score: Math.round(dot * 1000) / 1000,
+        });
+    }
+    return [...best.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+  };
+
+  return {
+    refresh,
+    search,
+    status: () => ({ folders: folders(), files: atlas.mediaPaths().length }),
+  };
+}

package/lib/members.json

@@ -20,7 +20,7 @@
       "about": "research on X and the web",
       "where": "in its own cloud", "limits": "Cannot see files on this computer unless one is attached with share_with_cloud, and cannot run anything here." },
     { "id": "bankr", "name": "Bankr", "aliases": [], "road": "server", "home": "cloud", "runs": "server", "target": true, "wake": "run",
-      "about": "the person's crypto wallet and trading, always under the person's own approval and limits",
+      "about": "holds the swarm's wallet, which is the person's own, and does anything onchain the work needs: buying, selling, swapping, sending or bridging tokens, or any other transaction, always under the person's own approval and limits",
       "where": "on Harness's server, through Bankr", "limits": "Cannot see files on this computer. Reading balances or a transaction needs no task: an agent on this computer reads them with the wallet tool in seconds." },
     { "id": "quotient", "name": "Quotient", "aliases": [], "road": "server", "home": "cloud", "runs": "server", "target": true, "wake": "run",
       "about": "prediction markets research: one search over the markets it tracks, with their odds and rules",

package/lib/members.mjs

@@ -5,29 +5,41 @@ import { readFileSync } from "node:fs";
 // waiting (wake). The server reads the same file (src/shared/members.ts). A caller on this machine names itself
 // (--agent) and is taken at its word: anything running as the person could read a token kept for it, so none is
 // kept (owner, October 10). A call through Harness is never taken for a member on this machine (hostedCaller).
-const ROSTER = JSON.parse(readFileSync(new URL("./members.json", import.meta.url), "utf8"));
+const ROSTER = JSON.parse(
+  readFileSync(new URL("./members.json", import.meta.url), "utf8")
+);
 
 export const ROSTER_VERSION = ROSTER.version;
 export const MEMBERS = ROSTER.members;
 const byId = new Map(MEMBERS.map((entry) => [entry.id, entry]));
 
 export const member = (id) => byId.get(String(id)) ?? null;
-export const ids = (keep = () => true) => MEMBERS.filter(keep).map((entry) => entry.id);
+export const ids = (keep = () => true) =>
+  MEMBERS.filter(keep).map((entry) => entry.id);
 /** Members others may hand work to, Harness included. */
 export const TARGETS = ids((entry) => entry.target);
 /** Coding agents the daemon runs on this machine. */
 export const MACHINE_RUNNERS = ids((entry) => entry.runs === "machine");
 /** Targets whose tasks live on Harness's server, not the machine (Harness itself excluded). */
-export const CLOUD_HOMED = ids((entry) => entry.target && entry.home === "cloud" && entry.id !== "harness");
+export const CLOUD_HOMED = ids(
+  (entry) => entry.target && entry.home === "cloud" && entry.id !== "harness"
+);
 /** Targets asked by a task rather than started here: they run themselves or on the server. */
-export const ASKED_BY_TASK = ids((entry) => entry.target && entry.runs !== "machine" && entry.id !== "harness");
+export const ASKED_BY_TASK = ids(
+  (entry) => entry.target && entry.runs !== "machine" && entry.id !== "harness"
+);
 export const nameOf = (id) => member(id)?.name ?? String(id);
 export const aboutOf = (id) => {
   const entry = member(id);
   return entry ? `${entry.name}: ${entry.about}` : String(id);
 };
 
-const plain = (value) => String(value || "").trim().toLowerCase().replace(/^@/, "").replace(/[\s_]+/g, " ");
+const plain = (value) =>
+  String(value || "")
+    .trim()
+    .toLowerCase()
+    .replace(/^@/, "")
+    .replace(/[\s_]+/g, " ");
 /**
  * A name as people and agents write it, resolved to one member: an id, a display name or an alias, else the one
  * member whose name it starts. A start shared by several ("claude": Claude Code and the Claude app) comes back
@@ -37,11 +49,25 @@ const plain = (value) => String(value || "").trim().toLowerCase().replace(/^@/,
 export function resolveMember(name, { extra = [] } = {}) {
   const wanted = plain(name);
   if (!wanted) return { ok: false, candidates: [] };
-  const exact = MEMBERS.find((entry) => entry.id === wanted || plain(entry.name) === wanted || entry.aliases.some((alias) => plain(alias) === wanted));
+  const exact = MEMBERS.find(
+    (entry) =>
+      entry.id === wanted ||
+      plain(entry.name) === wanted ||
+      entry.aliases.some((alias) => plain(alias) === wanted)
+  );
   if (exact) return { ok: true, id: exact.id };
   if (extra.map(String).includes(wanted)) return { ok: true, id: wanted };
   const words = wanted.split(" ");
-  const candidates = MEMBERS.filter((entry) => [entry.id.replace(/-/g, " "), plain(entry.name), ...entry.aliases.map(plain)]
-    .some((label) => words.every((word, index) => label.split(" ")[index]?.startsWith(word)))).map((entry) => entry.id);
-  return candidates.length === 1 ? { ok: true, id: candidates[0] } : { ok: false, candidates };
+  const candidates = MEMBERS.filter((entry) =>
+    [
+      entry.id.replace(/-/g, " "),
+      plain(entry.name),
+      ...entry.aliases.map(plain),
+    ].some((label) =>
+      words.every((word, index) => label.split(" ")[index]?.startsWith(word))
+    )
+  ).map((entry) => entry.id);
+  return candidates.length === 1
+    ? { ok: true, id: candidates[0] }
+    : { ok: false, candidates };
 }