total-recall-brain 3.43.3 → 3.44.0

package/src/core/server-restart.mjs

@@ -12,9 +12,14 @@
  * Exiting an unsupervised server is not a restart, it is an outage, so
  * supervision is proven (the job's reported PID is ours) rather than assumed.
  */
-import { execFileSync } from 'node:child_process';
+import { execFileSync, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
+import { processIdentity } from './startup-health.mjs';
+import { runningServerStatus } from './running-server.mjs';
+
+const GLOBAL_PACKAGE = 'total-recall-brain';
+const HOLD_LIMIT_MS = 6 * 60 * 60 * 1000;
 
 const LAUNCHD_LABELS = ['com.totalrecall.brain', 'com.totalrecall.server'];
 const SYSTEMD_UNITS = ['total-recall-server', 'total-recall-server.service'];
@@ -47,30 +52,33 @@ function systemdPidFor(unit) {
 
 /**
  * Prove that a supervisor owns THIS process and will restart it.
- * @returns {{supervised: boolean, kind: string|null, label: string|null, reason: string}}
+ * `scope` says whose supervisor it is: `user` belongs to one login, `system` to the machine.
+ * @returns {{supervised: boolean, kind: string|null, label: string|null, scope: string|null, reason: string}}
  */
 export function detectSupervisor(pid = process.pid) {
   // An explicit opt-in for hosts running under a supervisor we cannot query
   // (docker restart policies, pm2, runit). The operator asserts respawn.
   if (process.env.TR_SUPERVISED === '1') {
-    return { supervised: true, kind: 'declared', label: 'TR_SUPERVISED=1', reason: 'declared by operator' };
+    return { supervised: true, kind: 'declared', label: 'TR_SUPERVISED=1', scope: null, reason: 'declared by operator' };
   }
   if (process.platform === 'darwin') {
     for (const label of LAUNCHD_LABELS) {
       const jobPid = launchdPidFor(label);
       if (jobPid === pid) {
-        return { supervised: true, kind: 'launchd', label, reason: `launchd job ${label} owns pid ${pid}` };
+        // launchctl answers for the caller's domain, so as root this is a system daemon.
+        const scope = process.getuid?.() === 0 ? 'system' : 'user';
+        return { supervised: true, kind: 'launchd', label, scope, reason: `launchd job ${label} owns pid ${pid}` };
       }
     }
-    return { supervised: false, kind: null, label: null, reason: 'no launchd job reports this pid' };
+    return { supervised: false, kind: null, label: null, scope: null, reason: 'no launchd job reports this pid' };
   }
   for (const unit of SYSTEMD_UNITS) {
     const found = systemdPidFor(unit);
     if (found && found.pid === pid) {
-      return { supervised: true, kind: 'systemd', label: unit, reason: `systemd ${found.scope} unit ${unit} owns pid ${pid}` };
+      return { supervised: true, kind: 'systemd', label: unit, scope: found.scope, reason: `systemd ${found.scope} unit ${unit} owns pid ${pid}` };
     }
   }
-  return { supervised: false, kind: null, label: null, reason: 'no systemd unit reports this pid' };
+  return { supervised: false, kind: null, label: null, scope: null, reason: 'no systemd unit reports this pid' };
 }
 
 /**
@@ -128,3 +136,116 @@ export function packageVersionOnDisk(rootDir) {
     return null;
   }
 }
+
+/**
+ * Can maintenance replace the server on this host, and how?
+ *
+ * `npm install --global` moves the files under a running server and nothing else
+ * moves the process, so a host that serves from that install stays on the release it
+ * booted on. Replacing it has a gap, so maintenance does it only where a gap is allowed
+ * and a restart is certain to help:
+ *   - a per-user supervisor owns the process. A LaunchAgent or user unit belongs to one
+ *     login; a system service is production, and `deploy release` replaces that with no gap.
+ *   - the process runs the global install's entry, and that install is not a checkout
+ *     (auto-pull owns a checkout) and is at the release. Otherwise a restart starts the
+ *     same old code, or code nobody released.
+ *
+ * @returns {{method: 'restart', kind: string, label: string, pid: number}|{method: 'deploy'|'none', reason: string}}
+ */
+export function planServerReplacement({ brainDir, globalRoot, latest, env = process.env, identity = processIdentity, supervisor = detectSupervisor, versionOnDisk = packageVersionOnDisk } = {}) {
+  const leave = (reason, method = 'none') => ({ method, reason });
+  if (env.TR_SERVER_AUTO_RESTART === '0') return leave('TR_SERVER_AUTO_RESTART=0 leaves this server to the operator.');
+  if (!globalRoot) return leave('The global npm root could not be read, so what a restart would start is unknown.');
+  const install = path.join(globalRoot, GLOBAL_PACKAGE);
+  const observed = identity(path.join(brainDir, 'server.pid'), path.join(install, 'src', 'server', 'index.mjs'));
+  if (!observed.pid) return leave('The process serving this brain could not be identified.');
+  const owner = supervisor(observed.pid);
+  if (owner.supervised && owner.scope === 'system') return leave(`${owner.label} is a system service: replace it with total-recall deploy release.`, 'deploy');
+  if (observed.status !== 'running') return leave(`The server does not run from the global install${observed.entry ? ` (it runs ${observed.entry})` : ''}, so a restart would not bring it to the release.`);
+  if (fs.existsSync(path.join(install, '.git'))) return leave('The global install is a link to a git checkout, which auto-pull keeps current.');
+  const onDisk = versionOnDisk(install);
+  if (onDisk !== latest) return leave(`The global install is ${onDisk ?? 'unreadable'} and the release is ${latest}.`);
+  if (!owner.supervised || owner.scope !== 'user') return leave(`Nothing maintenance can drive would bring the server back: ${owner.reason}.`);
+  return { method: 'restart', kind: owner.kind, label: owner.label, pid: observed.pid };
+}
+
+/** Connections open to the server right now, or null when that cannot be read. */
+export function openServerConnections({ pid, port, platform = process.platform, run = spawnSync } = {}) {
+  const options = { encoding: 'utf8', timeout: 10000 };
+  if (platform === 'linux') {
+    const result = run('ss', ['-tnH', 'state', 'established', 'sport', '=', `:${port}`], options);
+    if (result.error || result.status !== 0) return null;
+    return result.stdout.split('\n').filter((line) => line.trim()).length;
+  }
+  const result = run('lsof', ['-nP', '-a', '-p', String(pid), '-iTCP', '-sTCP:ESTABLISHED', '-Fn'], options);
+  if (result.error) return null;
+  // lsof exits 1 when nothing matches, and also on a real failure, which is the one that writes to stderr.
+  if (result.status !== 0 && (result.stdout?.trim() || result.stderr?.trim())) return null;
+  return (result.stdout || '').split('\n').filter((line) => line.startsWith('n') && line.split('->')[0].endsWith(`:${port}`)).length;
+}
+
+/** The command that stops the server and starts it again, or null when the job is not there. */
+function restartCommand(plan, { run, uid }) {
+  if (plan.kind === 'systemd') return ['systemctl', ['--user', 'restart', plan.label]];
+  // `gui` exists while the user is logged in; a Mac with no login session has only `user`.
+  const target = [`gui/${uid}/${plan.label}`, `user/${uid}/${plan.label}`]
+    .find((candidate) => run('launchctl', ['print', candidate], { encoding: 'utf8', timeout: 10000 }).status === 0);
+  // kickstart -k delivers SIGTERM and waits (measured, launchd does not document it), so the
+  // server drains and releases its PID lock, and it needs no KeepAlive to come back.
+  return target ? ['launchctl', ['kickstart', '-k', target]] : null;
+}
+
+/**
+ * Bring a stale server to the release, when `planServerReplacement` allows it.
+ *
+ * An open connection holds this back so nobody's work is cut off, for as long as
+ * `holdLimitMs`: an open dashboard tab must not keep a host on an old release forever.
+ * Replaced means /health reports the release, never that the command returned.
+ *
+ * @returns the server status to report, with `replacement` saying what happened.
+ */
+export async function replaceStaleServer({
+  brainDir, server, globalRoot, previous, uid = process.getuid?.(), plan = planServerReplacement, connections = openServerConnections,
+  run = spawnSync, observe = runningServerStatus, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), now = Date.now,
+  idleWaitMs = 30000, healthWaitMs = 120000, pollMs = 2000, holdLimitMs = HOLD_LIMIT_MS,
+} = {}) {
+  let replacement;
+  try {
+    replacement = plan({ brainDir, globalRoot, latest: server.latest });
+    if (replacement.method !== 'restart') return { ...server, replacement };
+
+    const port = new URL(server.url).port;
+    const idleBy = now() + idleWaitMs;
+    let open = connections({ pid: replacement.pid, port });
+    while (open !== 0 && now() < idleBy) {
+      await sleep(pollMs);
+      open = connections({ pid: replacement.pid, port });
+    }
+    if (open !== 0) {
+      const heldSince = previous?.replacement?.state === 'held' ? previous.replacement.held_since : now();
+      if (now() - heldSince < holdLimitMs) {
+        const reason = open === null ? 'Open connections could not be read.' : `${open} connection(s) are open on the server.`;
+        return { ...server, replacement: { ...replacement, state: 'held', held_since: heldSince, reason } };
+      }
+    }
+
+    const command = restartCommand(replacement, { run, uid });
+    if (!command) return { ...server, replacement: { ...replacement, state: 'failed', reason: `launchd has no job ${replacement.label} for this user.` } };
+    const restarted = run(command[0], command[1], { encoding: 'utf8', timeout: 60000 });
+    if (restarted.error || restarted.status !== 0) {
+      const reason = `${command[0]} ${command[1].join(' ')} failed: ${restarted.error?.message || restarted.stderr?.trim() || `exit ${restarted.status}`}`;
+      return { ...server, replacement: { ...replacement, state: 'failed', reason } };
+    }
+
+    const healthyBy = now() + healthWaitMs;
+    let after;
+    do {
+      await sleep(pollMs);
+      after = await observe({ brainDir, latest: server.latest, timeoutMs: 5000 });
+    } while (after.status !== 'current' && now() < healthyBy);
+    if (after.status === 'current') return { ...after, replacement: { ...replacement, state: 'replaced', from: server.running } };
+    return { ...after, replacement: { ...replacement, state: 'failed', reason: `/health did not report ${server.latest} within ${healthWaitMs / 1000} seconds of the restart.` } };
+  } catch (error) {
+    return { ...server, replacement: { ...replacement, state: 'failed', reason: error.message } };
+  }
+}

package/src/core/ssss-kernel-bridge.mjs

@@ -235,9 +235,8 @@ export function isProtocolPath(vfsPath = '') {
 }
 
 function stringifyDocument(data, body) {
-  // Prefer gray-matter when available; fall back to simple frontmatter dump.
   try {
-    return matter.stringify(body || '', data);
+    return matter.stringify({ content: body || '', data });
   } catch {
     const yaml = Object.entries(data)
       .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)

package/src/core/surface.mjs

@@ -620,7 +620,7 @@ const CLIENT_SHIMS = {
   'hermes-agent':     ['AGENTS.md'],
   dsh:                ['AGENTS.md'],
   'deepseek-harness': ['AGENTS.md'],
-  openclaw:           ['MEMORY.md', 'AGENTS.md']
+  openclaw:           ['AGENTS.md']
 };
 
 /**
@@ -739,13 +739,13 @@ async function compilePointers(instructionsFile, skillsDir, nodes = [], { vaultD
  */
 /**
  * The rule nodes a project's instruction surfaces are built from: the global
- * brain's rules plus the project's own. A project node with the same slug
+ * brain's rules plus the project's own. A project rule with the same slug
  * wins, so a repo can override a global rule; everything else in the global
  * vault stays search-only. Without this every repo carried only its own vault,
  * and a rule saved with `remember --global` reached no repo at all.
  */
 export function mergeGlobalRuleNodes(projectNodes, globalNodes) {
-  const own = new Set(projectNodes.map((n) => n.slug));
+  const own = new Set(projectNodes.filter((n) => RULE_CATEGORIES.has(n.category)).map((n) => n.slug));
   const inherited = globalNodes
     .filter((n) => RULE_CATEGORIES.has(n.category) && !own.has(n.slug))
     .map((n) => ({ ...n, _layer: 'global' }));

package/src/core/task-context.mjs

@@ -9,7 +9,9 @@ import fs from 'node:fs';
 const hash = text => crypto.createHash('sha256').update(String(text)).digest('hex');
 const STOP = new Set('a an and are as at be before by can do for from how i in is it of on or our should so that the their this to use we what when which with you'.split(' '));
 export const taskTerms = task => [...new Set((String(task).toLowerCase().match(/[\p{L}\p{N}_-]{3,}/gu) || []).filter(term => !STOP.has(term)))];
-const overlap = (terms, text) => terms.filter(term => String(text).toLowerCase().includes(term)).length / Math.max(1, terms.length);
+// Task terms contain only letters, digits, underscores and hyphens. Hyphens also delimit source-reference segments.
+const taskMatchers = task => taskTerms(task).map(term => new RegExp(`(?<![\\p{L}\\p{N}_])${term}(?![\\p{L}\\p{N}_])`, 'iu'));
+const overlap = (matchers, text) => matchers.filter(matcher => matcher.test(text)).length / Math.max(1, matchers.length);
 export const summarizeRetrieval = summary => summary ? { mode: summary.mode, selected: summary.selected,
   candidates: summary.candidates || 0, providers: (summary.coverage || []).map(row => row.provider),
   ...(summary.duplicatePassages ? { duplicatePassages: summary.duplicatePassages } : {}),
@@ -18,11 +20,11 @@ export const summarizeRetrieval = summary => summary ? { mode: summary.mode, sel
 
 /** Preserve the best matching line and bounded neighboring text, including a match late in a long body. */
 export function extractTaskPassage(text, task, { maxChars = 1000, anchorLine } = {}) {
-  const lines = String(text || '').split('\n'), terms = taskTerms(task);
+  const lines = String(text || '').split('\n'), matchers = taskMatchers(task);
   let anchor = Number.isInteger(anchorLine) ? Math.max(0, Math.min(lines.length - 1, anchorLine - 1)) : 0;
   if (!Number.isInteger(anchorLine)) {
     let best = -1;
-    lines.forEach((line, index) => { const score = overlap(terms, line); if (score > best) { best = score; anchor = index; } });
+    lines.forEach((line, index) => { const score = overlap(matchers, line); if (score > best) { best = score; anchor = index; } });
   }
   let start = Math.max(0, anchor - 2), end = Math.min(lines.length, anchor + 3);
   while (lines.slice(start, end).join('\n').length > maxChars && (start < anchor || end > anchor + 1)) {
@@ -32,8 +34,7 @@ export function extractTaskPassage(text, task, { maxChars = 1000, anchorLine } =
   let passage = lines.slice(start, end).join('\n');
   let startColumn = 1;
   if (passage.length > maxChars) {
-    const lower = passage.toLowerCase();
-    const hits = terms.map(term => lower.indexOf(term)).filter(index => index >= 0);
+    const hits = matchers.map(matcher => passage.search(matcher)).filter(index => index >= 0);
     const focus = hits.length ? Math.min(...hits) : 0;
     const offset = Math.max(0, Math.min(passage.length - maxChars, focus - Math.floor(maxChars / 3)));
     passage = passage.slice(offset, offset + maxChars); startColumn += offset;
@@ -50,8 +51,8 @@ function renderEvidence(item) {
 export async function retrieveTaskKnowledge({ query, nodes = [], derivedDir, indexScopes,
   projectRoot = process.cwd(), actions = [], semantic = true, budget = 1200, limit = 6,
   embedding = getEmbedding, loadIndex = loadEmbeddingsIndex, providers = retrievePluginContext } = {}) {
-  const terms = taskTerms(query);
-  if (!terms.length) return { contributions: [], summary: { mode: 'none', selected: 0, diagnostics: [], coverage: [] }, queryVector: null };
+  const matchers = taskMatchers(query);
+  if (!matchers.length) return { contributions: [], summary: { mode: 'none', selected: 0, diagnostics: [], coverage: [] }, queryVector: null };
   const diagnostics = [], indexes = new Map();
   const scopes = indexScopes || (derivedDir ? [{ layer: '*', derivedDir }] : []);
   for (const scope of scopes) {
@@ -77,7 +78,7 @@ export async function retrieveTaskKnowledge({ query, nodes = [], derivedDir, ind
     if (stale) staleVectors++;
     const valid = !stale && Array.isArray(entry?.embedding) && Array.isArray(queryVector) && entry.embedding.length === queryVector.length;
     if (valid) vectors++;
-    const lexical = overlap(terms, `${node.title || ''}\n${body}`);
+    const lexical = overlap(matchers, `${node.title || ''}\n${body}`);
     const similarity = valid ? cosineSimilarity(queryVector, entry.embedding) : 0;
     const score = valid ? Math.max(0, similarity) * .75 + lexical * .25 : lexical * .7;
     return { node, layer, body, score, relevant: lexical > 0 || similarity >= .35 };

package/src/core/validated-write.mjs

@@ -15,6 +15,8 @@ import { processOperationAsync } from './operation-validator.mjs';
 import { safeStringify, isSafeVaultName } from './vault.mjs';
 import { invalidate } from './vault-cache.mjs';
 
+export const supportsExpectedBeforeHash = true;
+
 /**
  * Normalize a vault node so package universal frontmatter + TR memory v2 pass.
  */
@@ -101,6 +103,7 @@ function buildEnvelope(node, options = {}) {
   const {
     workspaceId = 'default',
     idempotencyKey,
+    expectedBeforeHash,
     dryRun = false,
     path: pathOverride,
   } = options;
@@ -124,6 +127,7 @@ function buildEnvelope(node, options = {}) {
     path: vfsPath,
     workspace_id: workspaceId,
     content,
+    ...(expectedBeforeHash === undefined ? {} : { expected_before_hash: expectedBeforeHash }),
     dry_run: !!dryRun,
     actor: { role: options.agentRole || 'system' },
   };

package/src/core/vault-files.mjs

@@ -8,6 +8,12 @@ export function isSafeVaultName(name) {
 
 export function atomicWrite(filePath, content) {
   const tmpPath = `${filePath}.tmp.${process.pid}`;
-  fs.writeFileSync(tmpPath, content, 'utf8');
-  fs.renameSync(tmpPath, filePath);
+  try {
+    fs.writeFileSync(tmpPath, content, 'utf8');
+    fs.renameSync(tmpPath, filePath);
+  } catch (error) {
+    // A leftover copy is untracked and unignored, so a commit of everything would publish it.
+    fs.rmSync(tmpPath, { force: true });
+    throw error;
+  }
 }

package/src/core/vault.mjs

@@ -33,7 +33,7 @@ function stripUndefined(obj) {
  * @returns {string}        The serialized markdown with frontmatter
  */
 export function safeStringify(content, data) {
-  return matter.stringify(content ?? '', stripUndefined(data ?? {}));
+  return matter.stringify({ content: content ?? '', data: stripUndefined(data ?? {}) });
 }
 
 /**

package/src/server/index.mjs

@@ -34,7 +34,7 @@ import { readProcessCommand, entryPathHint, shouldHonorPidLock } from '../core/p
 import { createCoverageCache } from './health-coverage.mjs';
 import { drainActiveEmbeddings } from './routes/sessions.mjs';
 import { agentDir as configAgentDir, brainDir as configBrainDir, port as configPort, host as configHost, nodeEnv, instructionsFilePath } from '../core/config.mjs';
-import { getDaemonStatus, ensureDaemonRunning } from '../core/daemon-control.mjs';
+import { getDaemonStatus, ensureDaemonRunning, clearStaleDaemonLock } from '../core/daemon-control.mjs';
 import { deploymentHealth } from '../core/running-server.mjs';
 
 
@@ -1004,6 +1004,8 @@ if (HOST !== '127.0.0.1' && HOST !== '0.0.0.0') {
     
     // Unref the interval so it doesn't block process shutdown
     daemonWatchdogInterval.unref();
+  } else if (nodeEnv !== 'test') {
+    clearStaleDaemonLock();
   }
 
   // ─── Cloudflare Tunnel Auto-Start ───