tryharness 0.7.23 → 0.7.26

package/lib/models.mjs

@@ -19,104 +19,230 @@ const CLAUDE_NAMES = ["fable", "opus", "sonnet", "haiku"];
  * model that can do this work well; more effort only for hard work.", sent demanding work to cheap models (eval
  * model-routing v1, 20261014-010225-80250750-357a).
  */
-export const DEFAULT_ROUTING = "Match the model to the work: for easy work, the cheapest model that does it well; for demanding work, the most capable one. Pass over a model that keeps failing here. More effort only for hard work.";
+export const DEFAULT_ROUTING =
+  "Match the model to the work: for easy work, the cheapest model that does it well; for demanding work, the most capable one. Pass over a model that keeps failing here. More effort only for hard work.";
 
 const tokens = (text) => {
   const match = /^(\d+(?:\.\d+)?)([KM]?)$/i.exec(String(text || ""));
-  return match ? Math.round(Number(match[1]) * ({ k: 1e3, m: 1e6 }[match[2].toLowerCase()] ?? 1)) : null;
+  return match
+    ? Math.round(
+        Number(match[1]) * ({ k: 1e3, m: 1e6 }[match[2].toLowerCase()] ?? 1)
+      )
+    : null;
 };
 
 // One reader per agent's own list command, its output as the agent prints it today.
 const LISTS = {
-  codex: { command: "codex", args: ["debug", "models"], parse: (out) => JSON.parse(out).models.map((entry) => ({
-    id: entry.slug, about: entry.description || null, efforts: (entry.supported_reasoning_levels || []).map((level) => level.effort),
-    defaultEffort: entry.default_reasoning_level || null, context: entry.context_window || null })) },
+  codex: {
+    command: "codex",
+    args: ["debug", "models"],
+    parse: (out) =>
+      JSON.parse(out).models.map((entry) => ({
+        id: entry.slug,
+        about: entry.description || null,
+        efforts: (entry.supported_reasoning_levels || []).map(
+          (level) => level.effort
+        ),
+        defaultEffort: entry.default_reasoning_level || null,
+        context: entry.context_window || null,
+      })),
+  },
   // provider  model  context  max-out  thinking  images
-  pi: { command: "pi", args: ["--list-models"], parse: (out) => out.split("\n").map((line) => line.trim().split(/\s+/))
-    .filter((cells) => cells.length >= 5 && cells[0] !== "provider")
-    .map(([provider, model, context, , thinking]) => ({ id: `${provider}/${model}`, efforts: thinking === "yes" ? PI_THINKING : [], context: tokens(context) })) },
+  pi: {
+    command: "pi",
+    args: ["--list-models"],
+    parse: (out) =>
+      out
+        .split("\n")
+        .map((line) => line.trim().split(/\s+/))
+        .filter((cells) => cells.length >= 5 && cells[0] !== "provider")
+        .map(([provider, model, context, , thinking]) => ({
+          id: `${provider}/${model}`,
+          efforts: thinking === "yes" ? PI_THINKING : [],
+          context: tokens(context),
+        })),
+  },
   // Each model as "provider/id" on its own line, then its details as JSON.
-  opencode: { command: "opencode", args: ["models", "--verbose"], parse: (out) => {
-    const models = [];
-    let id = null, body = [];
-    const flush = () => {
-      if (!id) return;
-      const entry = JSON.parse(body.join("\n"));
-      models.push({ id, efforts: Object.keys(entry.variants || {}), context: entry.limit?.context ?? null,
-        ...(entry.cost && Number.isFinite(entry.cost.input) ? { price: { inputUsdPerMillion: entry.cost.input, outputUsdPerMillion: entry.cost.output } } : {}) });
-    };
-    for (const line of out.split("\n")) {
-      if (/^[^\s{}"]+\/\S+$/.test(line)) { flush(); id = line.trim(); body = []; } else if (id) body.push(line);
-    }
-    flush();
-    return models;
-  } },
+  opencode: {
+    command: "opencode",
+    args: ["models", "--verbose"],
+    parse: (out) => {
+      const models = [];
+      let id = null,
+        body = [];
+      const flush = () => {
+        if (!id) return;
+        const entry = JSON.parse(body.join("\n"));
+        models.push({
+          id,
+          efforts: Object.keys(entry.variants || {}),
+          context: entry.limit?.context ?? null,
+          ...(entry.cost && Number.isFinite(entry.cost.input)
+            ? {
+                price: {
+                  inputUsdPerMillion: entry.cost.input,
+                  outputUsdPerMillion: entry.cost.output,
+                },
+              }
+            : {}),
+        });
+      };
+      for (const line of out.split("\n")) {
+        if (/^[^\s{}"]+\/\S+$/.test(line)) {
+          flush();
+          id = line.trim();
+          body = [];
+        } else if (id) body.push(line);
+      }
+      flush();
+      return models;
+    },
+  },
 };
 
 function capture(command, args) {
   return new Promise((resolve, reject) => {
     const environment = agentEnvironment(command);
-    if (environment.error) { reject(new Error(environment.error)); return; }
+    if (environment.error) {
+      reject(new Error(environment.error));
+      return;
+    }
     let child;
-    try { child = spawn(command, args, { cwd: os.tmpdir(), env: environment.env }); } catch (error) { reject(error); return; }
+    try {
+      child = spawn(command, args, { cwd: os.tmpdir(), env: environment.env });
+    } catch (error) {
+      reject(error);
+      return;
+    }
     child.stdin.end();
     trackHelper(child);
-    let out = "", err = "";
-    child.stdout.on("data", (chunk) => { out += chunk; });
-    child.stderr.on("data", (chunk) => { err += chunk; });
+    let out = "",
+      err = "";
+    child.stdout.on("data", (chunk) => {
+      out += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      err += chunk;
+    });
     child.on("error", reject);
-    child.on("close", (code) => code === 0 ? resolve(out) : reject(new Error(`${command} ${args.join(" ")} exited with code ${code}: ${err.trim().slice(0, 200)}`)));
+    child.on("close", (code) =>
+      code === 0
+        ? resolve(out)
+        : reject(
+            new Error(
+              `${command} ${args.join(" ")} exited with code ${code}: ${err.trim().slice(0, 200)}`
+            )
+          )
+    );
   });
 }
 
 const catalogKey = (agent) => `models:${agent}`;
 /** An agent's last listing: { at, models, error? }, error when its list command last failed. */
-export const listingOf = (store, agent) => { try { return JSON.parse(store.meta(catalogKey(agent)) || "null"); } catch { return null; } };
+export const listingOf = (store, agent) => {
+  try {
+    return JSON.parse(store.meta(catalogKey(agent)) || "null");
+  } catch {
+    return null;
+  }
+};
 /** An agent's models as it last listed them: [{ id, about?, efforts, defaultEffort?, context?, price? }]. */
-export const catalogOf = (store, agent) => listingOf(store, agent)?.models ?? [];
+export const catalogOf = (store, agent) =>
+  listingOf(store, agent)?.models ?? [];
 
 /**
  * Asks each installed agent for its models, all at once so one slow agent holds up no other, and keeps the lists; an
  * agent whose command fails keeps its last list and the error.
  */
 export async function refreshCatalogs(store, agents, log = () => {}) {
-  await Promise.all(agents.filter((name) => LISTS[name]).map(async (agent) => {
-    try {
-      const models = LISTS[agent].parse(await capture(LISTS[agent].command, LISTS[agent].args)).filter((entry) => entry.id);
-      store.meta(catalogKey(agent), JSON.stringify({ at: new Date().toISOString(), models }));
-    } catch (error) {
-      log(`model list for ${agent}: ${error.message}`);
-      store.meta(catalogKey(agent), JSON.stringify({ ...(listingOf(store, agent) ?? { models: [] }), error: error.message.slice(0, 300) }));
-    }
-  }));
+  await Promise.all(
+    agents
+      .filter((name) => LISTS[name])
+      .map(async (agent) => {
+        try {
+          const models = LISTS[agent]
+            .parse(await capture(LISTS[agent].command, LISTS[agent].args))
+            .filter((entry) => entry.id);
+          store.meta(
+            catalogKey(agent),
+            JSON.stringify({ at: new Date().toISOString(), models })
+          );
+        } catch (error) {
+          log(`model list for ${agent}: ${error.message}`);
+          store.meta(
+            catalogKey(agent),
+            JSON.stringify({
+              ...(listingOf(store, agent) ?? { models: [] }),
+              error: error.message.slice(0, 300),
+            })
+          );
+        }
+      })
+  );
   if (agents.includes("claude-code")) {
-    const anthropic = [...new Set(["pi", "opencode"].flatMap((other) => catalogOf(store, other)).map((entry) => entry.id).filter((id) => id.startsWith("anthropic/")).map((id) => id.slice("anthropic/".length)))];
-    const models = [...CLAUDE_NAMES.map((name) => ({ id: name, about: `Claude Code's name for its newest ${name[0].toUpperCase()}${name.slice(1)} model.`, efforts: CLAUDE_EFFORTS })),
-      ...anthropic.sort().map((id) => ({ id, efforts: CLAUDE_EFFORTS }))];
-    store.meta(catalogKey("claude-code"), JSON.stringify({ at: new Date().toISOString(), models }));
+    const anthropic = [
+      ...new Set(
+        ["pi", "opencode"]
+          .flatMap((other) => catalogOf(store, other))
+          .map((entry) => entry.id)
+          .filter((id) => id.startsWith("anthropic/"))
+          .map((id) => id.slice("anthropic/".length))
+      ),
+    ];
+    const models = [
+      ...CLAUDE_NAMES.map((name) => ({
+        id: name,
+        about: `Claude Code's name for its newest ${name[0].toUpperCase()}${name.slice(1)} model.`,
+        efforts: CLAUDE_EFFORTS,
+      })),
+      ...anthropic.sort().map((id) => ({ id, efforts: CLAUDE_EFFORTS })),
+    ];
+    store.meta(
+      catalogKey("claude-code"),
+      JSON.stringify({ at: new Date().toISOString(), models })
+    );
   }
 }
 
 /** One model in an agent's list, written as its id or, when only one model has it, the part after the provider. */
 function findModel(store, agent, written) {
   const models = catalogOf(store, agent);
-  if (!models.length) return { error: `the relay has no model list for ${nameOf(agent)} yet` };
-  const wanted = String(written || "").trim().toLowerCase();
+  if (!models.length)
+    return { error: `the relay has no model list for ${nameOf(agent)} yet` };
+  const wanted = String(written || "")
+    .trim()
+    .toLowerCase();
   const exact = models.find((entry) => entry.id.toLowerCase() === wanted);
   if (exact) return { entry: exact };
-  const tail = models.filter((entry) => entry.id.toLowerCase().split("/").pop() === wanted);
+  const tail = models.filter(
+    (entry) => entry.id.toLowerCase().split("/").pop() === wanted
+  );
   if (tail.length === 1) return { entry: tail[0] };
-  return { error: tail.length ? `"${written}" could be ${tail.map((entry) => entry.id).join(" or ")}; name one` : `"${written}" is not one of ${nameOf(agent)}'s models (npx tryharness agents models ${agent})` };
+  return {
+    error: tail.length
+      ? `"${written}" could be ${tail.map((entry) => entry.id).join(" or ")}; name one`
+      : `"${written}" is not one of ${nameOf(agent)}'s models (npx tryharness agents models ${agent})`,
+  };
 }
 
 /** A model and effort for an agent, checked against its list: { ok, pick } with pick { model, effort } or { auto: true }. */
 export function resolvePick(store, agent, model, effort = null) {
-  if (String(model || "").trim().toLowerCase() === "auto") return { ok: true, pick: { auto: true } };
+  if (
+    String(model || "")
+      .trim()
+      .toLowerCase() === "auto"
+  )
+    return { ok: true, pick: { auto: true } };
   const found = findModel(store, agent, model);
   if (found.error) return { ok: false, error: found.error };
   const level = effort ? String(effort).trim().toLowerCase() : null;
   if (level && !found.entry.efforts.includes(level)) {
-    return { ok: false, error: found.entry.efforts.length ? `${found.entry.id} takes effort ${found.entry.efforts.join(", ")}, not "${effort}"` : `${found.entry.id} has no effort setting in ${nameOf(agent)}` };
+    return {
+      ok: false,
+      error: found.entry.efforts.length
+        ? `${found.entry.id} takes effort ${found.entry.efforts.join(", ")}, not "${effort}"`
+        : `${found.entry.id} has no effort setting in ${nameOf(agent)}`,
+    };
   }
   return { ok: true, pick: { model: found.entry.id, effort: level } };
 }
@@ -126,7 +252,15 @@ export function resolveSpec(store, agent, spec) {
   const whole = resolvePick(store, agent, spec);
   if (whole.ok) return whole;
   const cut = spec.lastIndexOf("/");
-  if (cut > 0) { const split = resolvePick(store, agent, spec.slice(0, cut), spec.slice(cut + 1)); if (split.ok) return split; }
+  if (cut > 0) {
+    const split = resolvePick(
+      store,
+      agent,
+      spec.slice(0, cut),
+      spec.slice(cut + 1)
+    );
+    if (split.ok) return split;
+  }
   return whole;
 }
 
@@ -141,7 +275,8 @@ export function picksIn(store, text, agents) {
     if (!agents.includes(agent) || !PICKABLE.has(agent)) continue;
     const spec = match[2].replace(/[.,;:!?)]+$/, "");
     const resolved = resolveSpec(store, agent, spec);
-    if (!resolved.ok) return { picks, error: `@${match[1]}:${spec}: ${resolved.error}` };
+    if (!resolved.ok)
+      return { picks, error: `@${match[1]}:${spec}: ${resolved.error}` };
     picks.set(agent, resolved.pick);
   }
   return { picks };
@@ -149,33 +284,73 @@ export function picksIn(store, text, agents) {
 
 const choiceKey = (agent) => `model-choice:${agent}`;
 /** The person's choice for an agent: { model, effort }, { auto: true }, or null for the agent's own settings. */
-export const choiceOf = (store, agent) => { try { return JSON.parse(store.meta(choiceKey(agent)) || "null"); } catch { return null; } };
+export const choiceOf = (store, agent) => {
+  try {
+    return JSON.parse(store.meta(choiceKey(agent)) || "null");
+  } catch {
+    return null;
+  }
+};
 export function setChoice(store, agent, model, effort = null) {
-  if (!model || String(model).toLowerCase() === "default") { store.meta(choiceKey(agent), ""); return { ok: true, choice: null }; }
+  if (!model || String(model).toLowerCase() === "default") {
+    store.meta(choiceKey(agent), "");
+    return { ok: true, choice: null };
+  }
   const resolved = resolvePick(store, agent, model, effort);
   if (!resolved.ok) return resolved;
   store.meta(choiceKey(agent), JSON.stringify(resolved.pick));
   return { ok: true, choice: resolved.pick };
 }
 export const routingOf = (store) => store.meta("routing-note") || null;
-export const setRouting = (store, text) => { const note = String(text || "").replace(/\s+/g, " ").trim().slice(0, 1_000); store.meta("routing-note", note); return note || null; };
+export const setRouting = (store, text) => {
+  const note = String(text || "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 1_000);
+  store.meta("routing-note", note);
+  return note || null;
+};
 
 /** A pick in words: "gpt-6-luna, effort high", "Auto". */
-export const describePick = (pick) => !pick ? null : pick.auto ? "Auto" : [pick.model, pick.effort && `effort ${pick.effort}`].filter(Boolean).join(", ");
+export const describePick = (pick) =>
+  !pick
+    ? null
+    : pick.auto
+      ? "Auto"
+      : [pick.model, pick.effort && `effort ${pick.effort}`]
+          .filter(Boolean)
+          .join(", ");
 
 /** The pick named for one run (a post, post_task), kept with the run until it starts. */
-export const setRunPick = (store, runId, pick, by) => store.meta(`model:${runId}`, JSON.stringify({ ...pick, by }));
-const runPick = (store, runId) => { try { return JSON.parse(store.meta(`model:${runId}`) || "null"); } catch { return null; } };
+export const setRunPick = (store, runId, pick, by) =>
+  store.meta(`model:${runId}`, JSON.stringify({ ...pick, by }));
+const runPick = (store, runId) => {
+  try {
+    return JSON.parse(store.meta(`model:${runId}`) || "null");
+  } catch {
+    return null;
+  }
+};
 
-const median = (values) => { const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); return sorted.length ? sorted[Math.floor(sorted.length / 2)] : null; };
+const median = (values) => {
+  const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
+  return sorted.length ? sorted[Math.floor(sorted.length / 2)] : null;
+};
 /** How each of an agent's models did on this board: finished, failed, and the median minutes and dollars of a finished run. */
 function recordByModel(store, agent) {
   const byModel = new Map();
   for (const row of store.modelWork(agent)) {
-    const entry = byModel.get(row.model) ?? { done: 0, failed: 0, minutes: [], cost: [] };
+    const entry = byModel.get(row.model) ?? {
+      done: 0,
+      failed: 0,
+      minutes: [],
+      cost: [],
+    };
     if (row.status === "done") {
       entry.done++;
-      entry.minutes.push((Date.parse(row.reported_at) - Date.parse(row.started_at)) / 60_000);
+      entry.minutes.push(
+        (Date.parse(row.reported_at) - Date.parse(row.started_at)) / 60_000
+      );
       entry.cost.push(Number(row.usage?.cost_usd));
     } else if (row.status === "failed") entry.failed++;
     byModel.set(row.model, entry);
@@ -183,15 +358,36 @@ function recordByModel(store, agent) {
   return (model) => {
     const entry = byModel.get(model);
     if (!entry) return null;
-    const minutes = median(entry.minutes), cost = median(entry.cost);
-    return { done: entry.done, failed: entry.failed, medianMinutes: minutes === null ? null : Math.round(minutes * 10) / 10, medianCostUsd: cost === null ? null : Math.round(cost * 10_000) / 10_000 };
+    const minutes = median(entry.minutes),
+      cost = median(entry.cost);
+    return {
+      done: entry.done,
+      failed: entry.failed,
+      medianMinutes: minutes === null ? null : Math.round(minutes * 10) / 10,
+      medianCostUsd: cost === null ? null : Math.round(cost * 10_000) / 10_000,
+    };
   };
 }
 
 /** What Auto sends Harness for one run (src/server/swarm/route.ts): the work, the person's words, every listed model with its record here. */
-export const routeInput = ({ agent, work, transcript, routing, ownModel, models, record }) => ({
-  agent, name: nameOf(agent), work, transcript, routing: routing ?? DEFAULT_ROUTING, ownModel,
-  models: models.slice(0, 250).map((entry) => ({ ...entry, record: record(entry.id) })),
+export const routeInput = ({
+  agent,
+  work,
+  transcript,
+  routing,
+  ownModel,
+  models,
+  record,
+}) => ({
+  agent,
+  name: nameOf(agent),
+  work,
+  transcript,
+  routing: routing ?? DEFAULT_ROUTING,
+  ownModel,
+  models: models
+    .slice(0, 250)
+    .map((entry) => ({ ...entry, record: record(entry.id) })),
 });
 
 /**
@@ -199,35 +395,101 @@ export const routeInput = ({ agent, work, transcript, routing, ownModel, models,
  * who chose: "you" (the person's post), another agent's id, "your choice" (the person's setting) or "auto".
  * Auto asks Harness only when the person allows it (not the privacy setting) and Harness can answer.
  */
-export function createModelRouter({ store, askHarness, canAsk, privacy, ownModel, workOf, log = () => {} }) {
+export function createModelRouter({
+  store,
+  askHarness,
+  canAsk,
+  privacy,
+  ownModel,
+  workOf,
+  log = () => {},
+}) {
   return async function modelForRun(task, agent) {
     const named = runPick(store, task.id);
-    const pick = named ?? (choiceOf(store, agent) ? { ...choiceOf(store, agent), by: "your choice" } : null);
+    const pick =
+      named ??
+      (choiceOf(store, agent)
+        ? { ...choiceOf(store, agent), by: "your choice" }
+        : null);
     if (!pick) return null;
     const by = named ? (named.by === "user" ? "you" : named.by) : pick.by;
     if (!pick.auto) {
       // The list may have changed since the pick: a model the agent no longer lists runs on its own settings, said so.
       const checked = resolvePick(store, agent, pick.model, pick.effort);
-      return checked.ok ? { ...checked.pick, by } : { model: null, effort: null, by, note: `${checked.error}; it ran on its own settings` };
+      return checked.ok
+        ? { ...checked.pick, by }
+        : {
+            model: null,
+            effort: null,
+            by,
+            note: `${checked.error}; it ran on its own settings`,
+          };
     }
-    const fallback = (note) => ({ model: null, effort: null, by: "auto", note });
-    if (privacy()) return fallback("Auto is off on the privacy setting; it ran on its own settings");
-    if (!canAsk()) return fallback("Harness could not pick a model just now; it ran on its own settings");
+    const fallback = (note) => ({
+      model: null,
+      effort: null,
+      by: "auto",
+      note,
+    });
+    if (privacy())
+      return fallback(
+        "Auto is off on the privacy setting; it ran on its own settings"
+      );
+    if (!canAsk())
+      return fallback(
+        "Harness could not pick a model just now; it ran on its own settings"
+      );
     const models = catalogOf(store, agent);
-    if (!models.length) return fallback(`the relay has no model list for ${nameOf(agent)} yet; it ran on its own settings`);
+    if (!models.length)
+      return fallback(
+        `the relay has no model list for ${nameOf(agent)} yet; it ran on its own settings`
+      );
     try {
-      const reply = await askHarness("route_model", routeInput({ agent, ...workOf(task), routing: routingOf(store), ownModel: ownModel(agent), models, record: recordByModel(store, agent) }), null);
-      if (!reply.ok) throw new Error(reply.error || "Harness did not pick a model");
+      const reply = await askHarness(
+        "route_model",
+        routeInput({
+          agent,
+          ...workOf(task),
+          routing: routingOf(store),
+          ownModel: ownModel(agent),
+          models,
+          record: recordByModel(store, agent),
+        }),
+        null
+      );
+      if (!reply.ok)
+        throw new Error(reply.error || "Harness did not pick a model");
       // The bridge hands back Harness's answer with its fields beside ok.
       const picked = reply;
-      if (!picked.model) return fallback("Harness picked no model; it ran on its own settings");
-      const checked = resolvePick(store, agent, picked.model, picked.effort && models.find((entry) => entry.id === picked.model)?.efforts.includes(picked.effort) ? picked.effort : null);
-      if (!checked.ok) return fallback(`Auto picked ${picked.model}, which ${nameOf(agent)} does not list; it ran on its own settings`);
+      if (!picked.model)
+        return fallback("Harness picked no model; it ran on its own settings");
+      const checked = resolvePick(
+        store,
+        agent,
+        picked.model,
+        picked.effort &&
+          models
+            .find((entry) => entry.id === picked.model)
+            ?.efforts.includes(picked.effort)
+          ? picked.effort
+          : null
+      );
+      if (!checked.ok)
+        return fallback(
+          `Auto picked ${picked.model}, which ${nameOf(agent)} does not list; it ran on its own settings`
+        );
       // Jev's confidence in the pick stays with the run, to see later how sure the picks that worked were.
-      return { ...checked.pick, by: "auto", judgments: picked.judgments ?? null, estimatedMicroUsd: picked.estimatedMicroUsd ?? null };
+      return {
+        ...checked.pick,
+        by: "auto",
+        judgments: picked.judgments ?? null,
+        estimatedMicroUsd: picked.estimatedMicroUsd ?? null,
+      };
     } catch (error) {
       log(`auto model for ${task.id}: ${error.message}`);
-      return fallback(`Harness could not pick a model (${error.message}); it ran on its own settings`);
+      return fallback(
+        `Harness could not pick a model (${error.message}); it ran on its own settings`
+      );
     }
   };
 }