total-recall-brain 3.43.3 → 3.45.0

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/session-jobs.mjs

@@ -0,0 +1,72 @@
+/** Continue a stored job on its branch, without reusing a deleted checkout. */
+import fs from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { configDir } from './paths.mjs';
+import { resolveJob } from './jobs.mjs';
+import { appendCommand, jobPaths } from './control.mjs';
+import { createWorktree, removeWorktree, commitAll, isDirty, provideDependencies } from './worktree.mjs';
+import { dispatchWithReceipt } from './dispatch.mjs';
+
+export async function continueJob(target, text, { fork = false, atTurn, dispatchFn, execFileFn = promisify(execFile), timeoutMs = 30000 } = {}) {
+  const job = resolveJob(target);
+  if (!fork && !job.result) throw new Error('Resume requires a finished job; use reply for a running job.');
+  if (typeof text !== 'string' || !text.trim()) throw new Error('A continuation message is required.');
+  const owner = job.result?.execution?.node || job.record?.node;
+  if (owner && owner !== 'local') {
+    const quote = value => `'${String(value).replaceAll("'", "'\\''")}'`;
+    const argv = [fork ? 'fork' : 'resume', job.result?.execution?.remoteJob || job.record?.remoteJob || job.id, ...(fork ? atTurn ? ['--at', atTurn] : [] : [text]), '--json'];
+    const report = await execFileFn(process.env.TOTAL_RECALL_BIN || 'total-recall', ['mesh', 'exec', owner,
+      `total-recall meta-harness ${argv.map(quote).join(' ')}`], { encoding: 'utf8', timeout: 600000 });
+    return JSON.parse(report.stdout);
+  }
+  const attempt = job.result?.attempts?.at(-1) || job.result;
+  const file = attempt?.progressFile || job.result?.progressFile || path.join(configDir(), 'jobs', `${job.id}.events.jsonl`);
+  let saved, status;
+  try {
+    saved = JSON.parse(fs.readFileSync(file.replace('.events.jsonl', '.session.json'), 'utf8'));
+    status = JSON.parse(fs.readFileSync(file.replace('.events.jsonl', '.status.json'), 'utf8'));
+  } catch { throw new Error('This job has no stored session.'); }
+  let sessionId = attempt?.sessionId || status.session_id;
+  if (!sessionId) throw new Error('This job has no resumable session ID.');
+  if (fork && !job.result) {
+    const delivery = await appendCommand(job.id, { type: 'fork', atTurn });
+    const deadline = Date.now() + timeoutMs;
+    let event;
+    while (!event && Date.now() < deadline) {
+      const rows = fs.readFileSync(jobPaths(job.id).events, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse);
+      const ack = rows.find(row => row.commandId === delivery.command.id && row.kind === 'ack');
+      if (ack?.ok === false) throw new Error(ack.text);
+      event = rows.find(row => row.commandId === delivery.command.id && row.kind === 'fork');
+      if (!event) await new Promise(resolve => setTimeout(resolve, 20));
+    }
+    if (!event?.sessionId) throw new Error('Fork did not return a session ID before the timeout.');
+    sessionId = event.sessionId;
+  }
+  const parentBranch = attempt?.branch || job.result?.branch || saved.branch;
+  if (!fork && (!parentBranch || !parentBranch.startsWith('meta-harness/'))) {
+    // A dispatch job worked in its own folder and has no job branch: resume it there, as it ran (A-123).
+    if (!saved.workspace) throw new Error('This job has no saved job branch or workspace.');
+    const result = await dispatchWithReceipt({ ...saved.request, sessionId, cwd: saved.workspace, contextCwd: saved.workspace,
+      contextTask: text, prompt: text, permissions: saved.request?.permissions || { read: true, write: true } },
+      { dispatchFn, role: 'Resume' });
+    return { ...result, parentJob: job.id, workspace: saved.workspace };
+  }
+  const branch = fork ? `meta-harness/fork-${randomUUID()}` : parentBranch;
+  const cwd = saved.workspace;
+  const directory = createWorktree(cwd, { branch, existingBranch: !fork, ref: parentBranch || 'HEAD' });
+  let result, keep = false;
+  try {
+    provideDependencies(cwd, directory);
+    result = await dispatchWithReceipt({ ...saved.request, sessionId, forkSession: fork && Boolean(job.result), atTurn,
+      cwd: directory, contextCwd: cwd, contextTask: text, prompt: text, isolated: true,
+      permissions: { read: true, write: true } }, { dispatchFn, role: fork ? 'Fork' : 'Resume' });
+    const commit = commitAll(directory, `meta-harness: ${fork ? 'fork' : 'resume'} ${job.id}`);
+    return { ...result, parentJob: job.id, branch, ...(commit ? result.ok ? { commit } : { wip: commit } : {}) };
+  } catch (error) {
+    keep = isDirty(directory);
+    return { ok: false, error: error.message, branch, ...(keep ? { keptWorktree: directory } : {}) };
+  } finally { if (!keep) removeWorktree(cwd, directory); }
+}

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/setup.mjs

@@ -26,10 +26,13 @@ export async function setupIntegrations({ node, cwd = process.cwd(), env = proce
     const target=mesh.findMeshNode(node);
     if(!target)throw new Error('Maintenance node is not recorded in the mesh');
     const plugin=getPlugin('meta-harness',cwd) || getPlugin('cli-agents',cwd);
-    if(!plugin?.valid)throw new Error('Meta-harness (cli-agents) is not installed in the selected workspace');
-    await ensurePluginRecord(plugin);
+    if(!plugin?.valid)throw new Error('Metaharness (cli-agents) is not installed in the selected workspace');
+    const record=await ensurePluginRecord(plugin);
     placement=target.hostname;
-    await patchPluginRecord(plugin,{task_nodes:{maintenance:placement}});
+    const manifest=JSON.parse(fs.readFileSync(new URL('../plugin.json',import.meta.url),'utf8'));
+    const taskNodes={...(record?.task_nodes || {})};
+    for(const task of manifest.tasks.filter(task=>task.placement==='selected-node'))taskNodes[task.command]=placement;
+    await patchPluginRecord(plugin,{task_nodes:taskNodes});
   }
   return {ok:true,connected,universalInstructions:'AGENTS.md; every dispatch also receives a freshly generated bootstrap',maintenanceNode:placement};
 }

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/termination.mjs

@@ -1,5 +1,5 @@
-/** A stopped run takes its agents, checkouts, unfinished branches and open threads with it. */
-import { stopActiveProcesses } from './worker.mjs';
+/** Stop agents first. Let the operation save its work and return before the process exits. */
+import { cancelExecution, cancellationResult } from './worker.mjs';
 import { removeActiveWorktrees } from './worktree.mjs';
 import { failOpenThreads } from './coordinator.mjs';
 
@@ -9,16 +9,20 @@ let installed = false;
 export function stopOnTermination({ graceMs = 500 } = {}) {
   if (installed) return;
   installed = true;
-  let stopping = false;
   for (const [signal, code] of Object.entries(SIGNALS)) process.on(signal, () => {
-    if (stopping) return;
-    stopping = true;
-    stopActiveProcesses('SIGTERM');
-    setTimeout(() => {
-      stopActiveProcesses('SIGKILL');
-      removeActiveWorktrees();
-      failOpenThreads();
-      process.exit(code);
-    }, graceMs);
+    if (cancellationResult()) return;
+    cancelExecution(signal, graceMs);
+    process.exitCode = code;
   });
+  // Also cover callers that use the lifecycle helper without the CLI wrapper.
+  process.once('beforeExit', () => { if (cancellationResult()) finishCancellation(); });
+}
+
+export function finishCancellation(result) {
+  const cancelled = cancellationResult();
+  if (!cancelled) return result;
+  const saved = removeActiveWorktrees({ preserveChanges: true });
+  failOpenThreads('The run was cancelled.');
+  process.exitCode = cancelled.exitCode;
+  return { ...result, ...cancelled, ...(saved.length ? { saved } : {}) };
 }

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/threads.mjs

@@ -48,7 +48,7 @@ function mutate(work) {
   for (let attempt = 0; attempt < 8; attempt += 1) {
     const store = storeForThreads();
     const state = readState(store);
-    const result = work(state.threads);
+    const result = work(state.threads, state);
     validateThreads(state.threads);
     try {
       store.write({ ...state, version: '2.0.0', updatedAt: new Date().toISOString(), threads: state.threads });
@@ -117,3 +117,34 @@ export function deleteThread(id) {
     return true;
   });
 }
+
+// Exact titles from test/threads.test.mjs and test/coordinator.test.mjs.
+// Real multiply-job threads do not match, including failed end-to-end attempts.
+const FIXTURE_TITLES = new Set([
+  'Implement Multi-Platform Threads', 'Canonical state', 'concurrent',
+  'Implement token bucket', 'task one', 'task two', 'task three', 'task four', 'failed verification',
+]);
+const CLEANUP_ID = 'a118-fixture-threads-v1';
+
+/** Delete the known fixture titles once. The same SSSS operation records every removal. */
+export function cleanupFixtureThreads({ dryRun = false } = {}) {
+  const plan = (threads, state) => {
+    const previous = state.cleanups?.find(row => row.id === CLEANUP_ID);
+    if (previous) return { ok: true, alreadyApplied: true, record: previous };
+    const removed = threads.filter(thread => FIXTURE_TITLES.has(thread.title))
+      .map(({ id, title, status }) => ({ id, title, status }));
+    const kept = threads.filter(thread => !FIXTURE_TITLES.has(thread.title)).map(thread => thread.id);
+    return { ok: true, record: { id: CLEANUP_ID, timestamp: new Date().toISOString(), sources: ['test/threads.test.mjs', 'test/coordinator.test.mjs'], removed, kept } };
+  };
+  const state = readState(storeForThreads());
+  const preview = plan(state.threads, state);
+  if (dryRun || preview.alreadyApplied) return { ...preview, dryRun };
+  return mutate((threads, current) => {
+    const result = plan(threads, current);
+    if (result.alreadyApplied) return result;
+    const ids = new Set(result.record.removed.map(row => row.id));
+    threads.splice(0, threads.length, ...threads.filter(thread => !ids.has(thread.id)));
+    current.cleanups = [...(current.cleanups || []), result.record];
+    return result;
+  });
+}

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/usage-queue.mjs

@@ -0,0 +1,110 @@
+/** SSSS-backed queue for work deferred by included usage limits. */
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { configDir } from './paths.mjs';
+import { destinationDispatchRequest } from './dispatch-request.mjs';
+import { readIncludedPolicy } from './included-usage.mjs';
+
+async function queueStore() {
+  const { PluginStateStore } = await import('./ssss-store.mjs');
+  return new PluginStateStore(path.join(configDir(), 'usage-queue.json'), 'meta-harness');
+}
+
+function readRows(store) {
+  const jobs = store.read([]);
+  if (!Array.isArray(jobs) || jobs.some(job => !job?.id || typeof job.task !== 'string' || !path.isAbsolute(job.workspace || '') ||
+      !['waiting', 'running', 'completed', 'cancelled', 'failed'].includes(job.status))) {
+    throw new Error('Invalid included usage queue.');
+  }
+  return jobs;
+}
+
+async function mutateQueue(change) {
+  for (let attempt = 0; attempt < 8; attempt += 1) {
+    const store = await queueStore();
+    const jobs = readRows(store);
+    const result = change(jobs);
+    try { store.write(jobs); return result; }
+    catch (error) { if (!error.message.includes('version conflict') || attempt === 7) throw error; }
+  }
+}
+
+function publicJob({ replay, task, ...job }) {
+  return { ...job, task: task.slice(0, 160), canRetry: Boolean(replay) && !job.needsReview };
+}
+
+export async function queueIncludedJob(request, result, { role, replay, launchPrevented = false } = {}) {
+  const workspace = request.contextCwd || request.cwd;
+  const direct = !replay && role === 'Dispatch' && workspace === request.cwd;
+  if (direct) replay = { kind: 'dispatch', request: destinationDispatchRequest(request) };
+  const job = {
+    id: randomUUID(), status: 'waiting', agent: request.agent, workspace, role: role || 'Dispatch',
+    task: String(request.contextTask || request.prompt).slice(0, direct ? 160 : 65536),
+    reason: 'included-usage-exhausted',
+    resetsAt: Number.isFinite(Date.parse(result.resetsAt)) ? result.resetsAt : null,
+    queuedAt: new Date().toISOString(), attempts: 0,
+    needsReview: !replay || (!launchPrevented && request.permissions?.write === true),
+    replay: replay || null,
+  };
+  return mutateQueue(jobs => {
+    if (jobs.filter(item => ['waiting', 'running'].includes(item.status)).length >= 200) throw new Error('Included usage queue is full.');
+    const history = jobs.filter(item => !['waiting', 'running'].includes(item.status)).slice(0, -50);
+    for (const item of history) jobs.splice(jobs.indexOf(item), 1);
+    jobs.push(job);
+    if (Buffer.byteLength(JSON.stringify(jobs)) > 4 * 1024 * 1024) throw new Error('Included usage queue exceeds its storage bound.');
+    return publicJob(job);
+  });
+}
+
+export async function listIncludedJobs() {
+  const jobs = readRows(await queueStore());
+  return jobs.map(publicJob);
+}
+
+function updateJob(jobs, id, change) {
+  const index = jobs.findIndex(job => job.id === id);
+  if (index < 0) throw new Error('Included usage job was not found.');
+  return jobs[index] = change(jobs[index]);
+}
+
+export async function cancelIncludedJob(id) {
+  return mutateQueue(jobs => publicJob(updateJob(jobs, id, job => {
+    if (job.status !== 'waiting') throw new Error('Only a waiting job can be cancelled.');
+    return { ...job, status: 'cancelled', replay: null };
+  })));
+}
+
+/** Claim atomically before retry, so two callers cannot start the same job. */
+export async function retryIncludedJob(id, { dispatch, route, now = Date.now() } = {}) {
+  if (!(await readIncludedPolicy()).enabled) throw new Error('Included usage policy must be enabled to retry.');
+  const claimed = await mutateQueue(jobs => updateJob(jobs, id, job => {
+    if (job.status !== 'waiting') throw new Error('Only a waiting job can be retried.');
+    if (job.needsReview || !job.replay) throw new Error('Review the prior attempt and resubmit this task from its original workspace.');
+    if (Date.parse(job.resetsAt) > now) throw new Error('The provider-reported reset time has not arrived.');
+    return { ...job, status: 'running', attempts: job.attempts + 1, startedAt: new Date(now).toISOString() };
+  }));
+  let result;
+  try {
+    if (claimed.replay.kind === 'dispatch') {
+      dispatch ??= (await import('./dispatch.mjs')).dispatchWithReceipt;
+      result = await dispatch(claimed.replay.request, { role: 'Dispatch', retryId: id });
+    } else if (claimed.replay.kind === 'route') {
+      route ??= (await import('./cascade.mjs')).executeRoutedTask;
+      result = await route(claimed.task, { ...claimed.replay.controls, cwd: claimed.workspace, gate: claimed.replay.gate, gateOptions: claimed.replay.gateOptions,
+        route: { selected: claimed.agent }, pool: [claimed.agent], retryId: id });
+    } else throw new Error('Unknown included usage job operation.');
+  } catch (error) {
+    result = { ok: false, failureKind: 'execution', error: error.message };
+  }
+  const held = ['quota', 'billing-policy', 'auth'].includes(result.failureKind);
+  const updated = await mutateQueue(jobs => updateJob(jobs, id, job => {
+    if (job.status !== 'running') throw new Error('Included usage job lost its claim.');
+    return { ...job, status: result.ok ? 'completed' : held ? 'waiting' : 'failed',
+      resetsAt: held && Number.isFinite(Date.parse(result.resetsAt)) ? result.resetsAt : null,
+      needsReview: job.needsReview || result.needsReview === true ||
+        (held && !result.launchPrevented && (job.replay.kind === 'route' || job.replay.request?.permissions?.write === true)),
+      lastError: result.ok ? null : result.error,
+      replay: held ? job.replay : null, finishedAt: new Date().toISOString() };
+  }));
+  return { ...result, job: publicJob(updated) };
+}

package/scaffold/.agent/skills/total-recall/plugins/meta-harness/src/verify-ssss.mjs

@@ -47,8 +47,8 @@ async function verifyEvents(documentPath, primitive, expectedCount) {
 async function verifyTarget(targetArgument) {
   const target = fs.realpathSync(path.resolve(targetArgument));
   const readJson = file => JSON.parse(fs.readFileSync(readTargetFile(target, file), 'utf8'));
-  assert.equal(readJson('package.json').name, 'tr-plugin-meta-harness', 'Select a Meta Harness checkout');
-  assert.equal(readJson('plugin.json').id, owner, 'The target must own Meta Harness state');
+  assert.equal(readJson('package.json').name, 'tr-plugin-meta-harness', 'Select a Metaharness checkout');
+  assert.equal(readJson('plugin.json').id, owner, 'The target must own Metaharness state');
 
   const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'meta-harness-ssss-gate-'));
   const previousBrain = process.env.TR_SECRETS_BRAIN;
@@ -94,7 +94,7 @@ async function verifyTarget(targetArgument) {
 
 try {
   const [flag, target, ...rest] = process.argv.slice(2);
-  if (flag !== '--target' || !target || rest.length) throw new Error('Supply an explicit Meta Harness checkout with --target <directory>');
+  if (flag !== '--target' || !target || rest.length) throw new Error('Supply an explicit Metaharness checkout with --target <directory>');
   process.stdout.write(`${JSON.stringify(await verifyTarget(target))}\n`);
 } catch (error) {
   process.stdout.write(`${JSON.stringify({ ok: false, error: error.message })}\n`);