parallelclaw 1.2.1 → 1.3.0

package/CHANGELOG.md

@@ -2,6 +2,46 @@
 
 Notable changes to parallelclaw (formerly memex-mvp). Older history lives in the git log.
 
+## 1.3.0 — instant delegation: push-on-write (no waiting for the sync schedule)
+
+Delegations, claims, and results now fly to the mesh the moment they're written,
+instead of waiting up to a full sync interval. Every task write on an initiating
+node fires a detached `sync-run --all` immediately — fire-and-forget,
+best-effort, a no-op when sync isn't configured (or `PARALLELCLAW_NO_PUSH=1`).
+Wired into `task-delegate` / `task-claim` / `task-update`, the
+`parallelclaw_delegate` MCP tool, and the Mac headless executor's results.
+Live-verified: a delegation pushed to the hub in ~1s (last-sync jumped from 82m
+ago to 0m ago) with no manual sync. +1 guard test.
+
+This removes the **requester→hub** leg of the latency. Two legs remain
+schedule-bound (tuned by interval, not code): the **hub→executor** forward (the
+hub syncs the executor on its own timer) and the **executor's inbox poll**. Drop
+those to ~30–60s for near-instant end-to-end; true sub-second needs hub
+push-routing + an event-driven executor (next).
+
+Also: **`task-list --inbox --quiet`** — a no-output exit-code gate (0 = has work,
+1 = empty) so a 1-min executor cron can pre-check with a free shell call and only
+spin up the (token-costly) LLM agentTurn when a task is actually waiting.
+
+## 1.2.2 — durable execution: claim lease + auto-requeue for dead executors
+
+Found live: a research task outran Kimi's 240-second cron tick. The executor had
+already written `working` (the claim), but its agentTurn was killed before it
+could write a terminal event → the task was stranded in `working` forever. Fixes:
+- **Claim lease + lazy auto-requeue (#3b).** A `working` claim is leased (default
+  15 min). A stale `working` (claimer died — cron timeout, crash, laptop sleep)
+  becomes claimable again and reappears in `task-list --inbox`, so the next
+  poller retries it. No task is lost to a dead executor, and no sweeper is needed.
+  `claimTask`/`listTasks` gain `now`/`leaseMs` knobs; the Mac executor passes them.
+- **Executor prompt + docs:** read the FULL prompt from the `task-claim` output
+  (or `task-claim <id> --json`), never the 80-char `task-list` display line; on a
+  too-big task write `failed` with partial progress instead of spinning until the
+  run is killed or closing `done` with a placeholder (which would block requeue —
+  the exact thing that lost the first research delegation).
+
+Tests: +2 (stale working is re-claimable and back in inbox; terminal never
+requeues). Full npm test green.
+
 ## 1.2.1 — patch: make the Mac executor's `claude -p` actually complete
 
 The Phase 2b headless executor invoked `claude -p` but every run hung until the

package/lib/executor.js

@@ -25,7 +25,7 @@ import { writeFileSync, existsSync, mkdirSync, unlinkSync } from 'node:fs';
 import { execFile, execSync } from 'node:child_process';
 
 import { getOrigin } from './config.js';
-import { listTasks, claimTask, updateTask } from './tasks.js';
+import { listTasks, claimTask, updateTask, triggerSyncPush } from './tasks.js';
 
 const HOME = homedir();
 const MEMEX_DIR = process.env.MEMEX_DIR || join(HOME, '.memex');
@@ -104,7 +104,9 @@ export async function runInboxOnce({
   const me = getOrigin();
   const exec = runner || claudeRunner;
   const acted = [];
-  let inbox = listTasks({ inbox: true, db, limit: 50 });
+  // Passing `now` keeps the lease/stale-working check on the same clock as claimTask
+  // below, and lets the executor pick up tasks abandoned by a dead claimer (#3b).
+  let inbox = listTasks({ inbox: true, db, limit: 50, now });
   inbox = inbox.filter((t) => t.to === me || (includeBroadcast && t.to === 'any'));
   for (const t of inbox) {
     if (!allowKinds.includes(t.kind)) {
@@ -126,6 +128,9 @@ export async function runInboxOnce({
       acted.push({ id: t.id, status: 'failed' });
     }
   }
+  // Fly results back to the requester now (claimTask/updateTask here are lib-level
+  // and don't push on their own). One push covers the whole drain pass.
+  if (acted.some((a) => a.status === 'done' || a.status === 'failed')) triggerSyncPush();
   return acted;
 }
 

package/lib/tasks.js

@@ -27,13 +27,27 @@ import { homedir } from 'node:os';
 import { join, dirname } from 'node:path';
 import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
 import Database from 'better-sqlite3';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
 
 import { getOrigin } from './config.js';
+import { syncExperimentEnabled } from './sync/config.js';
 
 const SOURCE = 'agent-task';
 const STATUSES = ['submitted', 'working', 'done', 'failed'];
 const TERMINAL = new Set(['done', 'failed']);
 
+// A claimed ('working') task whose claimer dies (cron agentTurn timeout, crash,
+// laptop sleep) would otherwise be stranded in 'working' forever. We LEASE the
+// claim: a 'working' event older than LEASE_MS is treated as abandoned and
+// becomes re-claimable by the next poller (lazy requeue — no sweeper, no extra
+// event type). Generous so modest cross-node clock skew can't expire a LIVE
+// claim. Tunable per call via opts.leaseMs.
+const LEASE_MS = 15 * 60 * 1000;
+function isStaleWorking(status, eventTsSec, nowMs, leaseMs) {
+  return status === 'working' && (nowMs - (eventTsSec || 0) * 1000) > leaseMs;
+}
+
 // Pick the CURRENT event of a task WITHOUT trusting wall-clock ts — clock skew
 // across nodes regressed state on the live mesh (a 'done' on a fast clock could
 // sort before a later 'working'). Order by the monotonic per-task `seq`, then a
@@ -62,6 +76,26 @@ function openDb() {
   return db;
 }
 
+const INGEST_PATH = join(dirname(fileURLToPath(import.meta.url)), '..', 'ingest.js');
+
+/**
+ * Push the just-written task event toward the mesh NOW — so a delegation, claim,
+ * or result propagates in seconds instead of waiting for the next scheduled sync.
+ * Fire-and-forget: detached, best-effort, never blocks or throws. No-op when sync
+ * isn't configured (or when disabled via PARALLELCLAW_NO_PUSH=1). Returns the
+ * spawned child (or null if skipped) — for tests.
+ */
+export function triggerSyncPush() {
+  try {
+    if (process.env.PARALLELCLAW_NO_PUSH === '1') return null;
+    if (!syncExperimentEnabled()) return null; // no mesh configured → nothing to push to
+    const child = spawn(process.execPath, [INGEST_PATH, 'sync-run', '--all'],
+      { detached: true, stdio: 'ignore', env: process.env });
+    child.unref();
+    return child;
+  } catch (_) { return null; }
+}
+
 /** Append one event row for a task. Returns the row's msg_id. */
 function writeEvent(db, { taskId, status, seq, envelope, text, origin, now }) {
   const ts = Math.floor(now / 1000);
@@ -141,14 +175,18 @@ export function updateTask(id, status, { result = null, db = null, now = Date.no
  * the task wasn't claimable (already taken / terminal) or another node won.
  * opts: { db?, now? }.
  */
-export function claimTask(id, { db = null, now = Date.now() } = {}) {
+export function claimTask(id, { db = null, now = Date.now(), leaseMs = LEASE_MS } = {}) {
   const ownDb = !db;
   db = db || openDb();
   try {
     const me = getOrigin();
     const prev = latestEvent(db, id);
     if (!prev) throw new Error(`claimTask: no task "${id}"`);
-    if (prev.envelope.status !== 'submitted') return null; // already working/terminal — not claimable
+    const status = prev.envelope.status;
+    // Claimable if fresh-submitted OR a STALE 'working' (its claimer died — the
+    // lease expired). A fresh 'working' or a terminal state is not claimable.
+    const reclaimStale = isStaleWorking(status, prev.ts, now, leaseMs);
+    if (status !== 'submitted' && !reclaimStale) return null;
     const seq = (prev.envelope.seq ?? 0) + 1;
     const env = { ...prev.envelope, status: 'working', seq, result: prev.envelope.result ?? null };
     writeEvent(db, { taskId: id, status: 'working', seq, origin: me, now, text: env.prompt || '', envelope: env });
@@ -180,7 +218,7 @@ function latestEvent(db, id) {
  *   inbox  — tasks addressed to me AND currently 'submitted' (ready to take)
  * Returns [{ id, from, to, kind, status, prompt, result, ts }] newest-first.
  */
-export function listTasks({ forOrigin = null, status = null, mine = false, inbox = false, limit = 50, db = null } = {}) {
+export function listTasks({ forOrigin = null, status = null, mine = false, inbox = false, limit = 50, db = null, now = Date.now(), leaseMs = LEASE_MS } = {}) {
   const ownDb = !db;
   db = db || openDb();
   try {
@@ -201,7 +239,9 @@ export function listTasks({ forOrigin = null, status = null, mine = false, inbox
     }
     let out = [...latest.values()];
     // #2 'any' is a broadcast: it lands in EVERY node's inbox (claim resolves the winner).
-    if (inbox) out = out.filter((t) => (t.to === me || t.to === 'any') && t.status === 'submitted');
+    // inbox = ready to run: 'submitted', OR a stale 'working' whose claimer died.
+    if (inbox) out = out.filter((t) => (t.to === me || t.to === 'any')
+      && (t.status === 'submitted' || isStaleWorking(t.status, t.ts, now, leaseMs)));
     if (mine) out = out.filter((t) => t.from === me);
     if (forOrigin) out = out.filter((t) => t.to === forOrigin || t.from === forOrigin);
     if (status) out = out.filter((t) => t.status === status);
@@ -282,6 +322,7 @@ export function cmdTaskDelegate() {
   });
   console.log(`✓ delegated task ${id} → ${to} (from ${getOrigin()})`);
   console.log(`  track: task-list --mine`);
+  triggerSyncPush(); // fly it to the mesh now, don't wait for the schedule
   process.exit(0);
 }
 
@@ -296,6 +337,7 @@ export function cmdTaskUpdate() {
     const env = updateTask(id, status, { result: typeof args['--result'] === 'string' ? args['--result'] : null });
     console.log(`✓ task ${id} → ${env.status}${env.result ? ' (result attached)' : ''}`);
   } catch (e) { console.error(`✗ ${e.message}`); process.exit(1); }
+  triggerSyncPush(); // status transition flies back to the requester now
   process.exit(0);
 }
 
@@ -316,6 +358,7 @@ export function cmdTaskClaim() {
     else console.log(`· task ${id} not claimable (already taken or terminal) — skip`);
     process.exit(3);
   }
+  triggerSyncPush(); // propagate the claim immediately (shrinks the double-claim window)
   if ('--json' in args) { console.log(JSON.stringify({ claimed: true, id, ...env })); process.exit(0); }
   console.log(`✓ claimed ${id} → working (by ${getOrigin()})`);
   console.log(`  from ${env.from}  ·  kind: ${env.kind || 'general'}`);
@@ -334,6 +377,10 @@ export function cmdTaskList() {
     inbox: '--inbox' in args,
     limit: parseInt(args['--limit'] || '', 10) || 50,
   });
+  // Cheap exit-code gate for cron polling: no output, exit 0 if there ARE tasks,
+  // 1 if none. Lets a scheduler do `task-list --inbox --quiet && run-executor`
+  // so the (token-costly) LLM agentTurn only fires when there's actual work.
+  if ('--quiet' in args) process.exit(tasks.length ? 0 : 1);
   if (!tasks.length) { console.log('No tasks.'); process.exit(0); }
   const icon = { submitted: '○', working: '◐', done: '✓', failed: '✗' };
   for (const t of tasks) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parallelclaw",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Local-first personal AI ops layer. Shared verbatim memory across your agents (Claude Code, Cursor, OpenClaw, Hermes, Obsidian, Telegram) in one SQLite + FTS5 corpus — plus a coordination layer where any of your agents can delegate tasks to any other. Searchable from any MCP-compatible client.",
   "type": "module",
   "main": "server.js",

package/server.js

@@ -44,7 +44,7 @@ import {
   KNOWN_SOURCES,
   CONFIG_PATH,
 } from './lib/config.js';
-import { createTask, listTasks, listTaskResults } from './lib/tasks.js';
+import { createTask, listTasks, listTaskResults, triggerSyncPush } from './lib/tasks.js';
 import {
   canonicalize as canonicalizeUrl,
   extractDomain,
@@ -3632,6 +3632,7 @@ server.setRequestHandler(CallToolRequestSchema, async (req) => {
       const kind = (typeof args.kind === 'string' && args.kind.trim()) ? args.kind.trim() : 'general';
       const content = typeof args.content === 'string' ? args.content : null;
       const { id } = createTask({ prompt, to, kind, content });
+      triggerSyncPush(); // deliver the delegation to the mesh now, not on the next schedule
       return textResult(
         `✓ Delegated task \`${id}\` → **${to}** (from ${getOrigin()}).\n` +
         `Runs when ${to === 'any' ? 'an always-on agent' : to} is available; the result returns to you ` +