total-recall-brain 3.30.0 → 3.30.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.30.0",
+  "version": "3.30.2",
   "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
   "type": "module",
   "workspaces": [

package/scaffold/.agent/skills/total-recall/SKILL.md

@@ -181,6 +181,10 @@ npx total-recall compile
 ```
 Synchronously rebuild the entire memory index (derived embeddings) and regenerate prompt instruction shims (such as `INSTRUCTIONS.md`, `GEMINI.md`, `AGENTS.md`, etc.).
 
+*   **Rules are global + local.** Every project's shims carry the global brain's invariants, preferences and corrections plus the project's own; a project node with the same slug overrides the global one.
+*   `compile --global` (what `remember --global` runs) and `compile --all` also recompile every registered project, so a global rule reaches every repo at once.
+*   A client gets a shim only once it is connected (`total-recall connect claude-code|codex|antigravity|…` in that repo). No `CLAUDE.md` in a repo means Claude Code was never connected there.
+
 ### 5. Ingest OKF Bundle
 ```bash
 npx total-recall ingest okf <path> [options]
@@ -394,11 +398,23 @@ transcript. `set`, `rotate` and `rekey` all accept it.
 ### Master password and `rekey`
 
 The store is AES-256-GCM; the key is scrypt-derived (N=2^16, ~64 MB) from
-`TR_SECRETS_PASSWORD`. On macOS the password belongs in the **Keychain**
-(`security add-generic-password -s total-recall-secrets`), not in a dotfile —
-`rekey` picks the Keychain entry up automatically and rotates every carrier
-(LaunchAgent plist, env file, Keychain) in one transaction, verifying the
-retired password no longer decrypts before it commits.
+the master password, resolved in this order:
+
+1. `TR_SECRETS_PASSWORD` / `TR_MASTER_PASSWORD` in the environment (a
+   LaunchAgent plist injects it for launchd jobs);
+2. macOS: the **Keychain** entry `total-recall-secrets`
+   (`security add-generic-password -s total-recall-secrets`). An ssh or mesh
+   session cannot read a locked login Keychain;
+3. Linux and headless hosts: the env file `~/.agent/tr.env` (or `TR_ENV_FILE`)
+   containing `export TR_SECRETS_PASSWORD='…'`. It is used only when it is a
+   regular file, mode 0600, owned by you, in a directory (keep `~/.agent` 0700)
+   nobody else can write. Do not also export it from `.bashrc` — every process
+   started from the shell would inherit it.
+
+`rekey` rotates every carrier (LaunchAgent plist, `--env-file`, Keychain) in one
+transaction, verifying the retired password no longer decrypts before it
+commits. No carrier on a machine means every command fails with "Secrets file
+is not valid JSON" and `mesh` falls back to local discovery.
 
 > **Processes hold the old password until restarted.** After a rekey, anything
 > still running with the retired value fails with `Failed to decrypt secrets
@@ -584,7 +600,7 @@ npm test                    # vitest run — whole suite
 npx vitest run <path>       # one spec file
 ```
 
-Current baseline: **288 spec files / 1493 tests, ~160 s** on the Mac Mini.
+Current baseline (3.30.1): **349 spec files / 2024 tests, ~160 s** on the Mac Mini.
 
 > **Never run the full suite, a full typecheck, or a production build on the
 > laptop.** These go to the Mac Mini or the droplet — local runs cause slowdowns

package/src/cli/connect.mjs

@@ -369,10 +369,19 @@ function apiDetails(opts) {
     models_url: `${base}/v1/models`,
     ssss_manifest_url: `${base}/api/ssss`,
     discovery_url: `${base}/.well-known/total-recall.json`,
-    authorization_header: opts.token ? `Authorization: Bearer ${opts.token}` : 'Authorization: Bearer <PAT>'
+    // Never the token itself: this goes to stdout and --json, where it lands in
+    // terminal scrollback, CI logs and agent transcripts. Clients read the
+    // token from brain.json (0600).
+    authorization_header: opts.token ? `Authorization: Bearer ${maskToken(opts.token)}` : 'Authorization: Bearer <PAT>'
   };
 }
 
+export function maskToken(token) {
+  const value = String(token || '');
+  if (value.length <= 8) return '<PAT>';
+  return `${value.slice(0, 6)}… (full token in config/brain.json)`;
+}
+
 function detectObsidianVault() {
   // macOS: Obsidian stores vault paths in its app support config
   const obsidianConfig = path.join(

package/src/cli/update.mjs

@@ -6,6 +6,7 @@
  *   npx total-recall update --apply      # install latest where needed
  *   npx total-recall update --apply --force
  *   npx total-recall update --repo /path/to/app
+ *   npx total-recall update --install-autopull [--port N] [--no-build]
  */
 
 import {
@@ -15,6 +16,7 @@ import {
   PACKAGE_NAME,
 } from '../core/package-auto-update.mjs';
 import { brainDir } from '../core/config.mjs';
+import { installAutoPull } from '../core/autopull-install.mjs';
 
 /** @param {string[]} args */
 export function parseArgs(args) {
@@ -24,6 +26,9 @@ export function parseArgs(args) {
     dryRun: false,
     help: false,
     repos: [],
+    installAutopull: false,
+    port: null,
+    noBuild: false,
   };
   for (let i = 0; i < args.length; i++) {
     const a = args[i];
@@ -32,6 +37,9 @@ export function parseArgs(args) {
     else if (a === '--force') opts.force = true;
     else if (a === '--dry-run') opts.dryRun = true;
     else if (a === '--repo' && args[i + 1]) opts.repos.push(args[++i]);
+    else if (a === '--install-autopull') opts.installAutopull = true;
+    else if (a === '--port' && args[i + 1]) opts.port = Number(args[++i]);
+    else if (a === '--no-build') opts.noBuild = true;
   }
   return opts;
 }
@@ -48,6 +56,15 @@ function printHelp() {
     total-recall update --apply --force Ignore throttle / disabled flag
     total-recall update --repo <path>   Include extra root (repeatable)
 
+  This brain's own checkout (git):
+    total-recall update --install-autopull [--port N] [--no-build] [--dry-run]
+        Every 5 minutes fast-forward this checkout to origin/main, install and
+        rebuild what changed, restart the brain and verify /health reports the
+        new version (scripts/auto-pull.sh). macOS: LaunchAgent
+        com.totalrecall.autopull; Linux: crontab. A dev checkout (other branch,
+        uncommitted or unpushed work) is never touched. --no-build: never build
+        the dashboard on this host (laptops).
+
   Env:
     TR_AUTO_UPDATE_PACKAGE=0   Disable daemon cron auto-update (default: on)
     TR_SYNC_REPOS=/a:/b       Extra project roots
@@ -63,6 +80,18 @@ export default async function update(args = []) {
     return;
   }
 
+  if (opts.installAutopull) {
+    const result = installAutoPull({ port: opts.port || undefined, noBuild: opts.noBuild, dryRun: opts.dryRun });
+    if (opts.dryRun) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    const where = result.plistPath ? `LaunchAgent ${result.plistPath}` : `crontab: ${result.cronLine}`;
+    console.log(`\n  ✅ Auto-pull installed (${where}), checking port ${result.port} every 5 minutes.`);
+    console.log('  Log: ~/.agent/logs/auto-pull.log\n');
+    return;
+  }
+
   const latest = fetchLatestNpmVersion();
   console.log(`\n  Package: ${PACKAGE_NAME}`);
   console.log(`  Latest on npm: ${latest || '(unavailable)'}`);

package/src/core/autopull-install.mjs

@@ -0,0 +1,177 @@
+/**
+ * Install the auto-pull timer on a host that serves a brain from a git checkout.
+ *
+ * The daemon's package auto-update only reaches npm installs in registered
+ * projects; it deliberately skips source checkouts. A brain running from a
+ * checkout therefore never updated unless someone pulled and restarted it by
+ * hand, and the one host with a timer (a droplet cron) pointed at a script that
+ * could not restart anything. This installs scripts/auto-pull.sh — the repo's
+ * own copy, so the updater updates itself — on a 5-minute timer:
+ *   - macOS: a LaunchAgent (com.totalrecall.autopull) with StartInterval;
+ *   - Linux: a crontab line, replacing any earlier auto-pull line.
+ *
+ * Nothing here names a machine or a repository: paths come from where this
+ * package is running, ports from flags, the environment or the brain's own
+ * LaunchAgent.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+export const AUTOPULL_LABEL = 'com.totalrecall.autopull';
+export const AUTOPULL_INTERVAL_SECONDS = 300;
+
+/** The checkout this CLI runs from (…/src/core → repo root). */
+export function packageRoot() {
+  return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+}
+
+function xmlEscape(value) {
+  return String(value)
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+/** LaunchAgent that runs the checkout's auto-pull.sh every `interval` seconds. */
+export function buildLaunchAgentPlist({ script, env = {}, logFile, interval = AUTOPULL_INTERVAL_SECONDS, label = AUTOPULL_LABEL }) {
+  const envEntries = Object.entries(env)
+    .filter(([, v]) => v !== undefined && v !== null && v !== '')
+    .map(([k, v]) => `\t\t<key>${xmlEscape(k)}</key>\n\t\t<string>${xmlEscape(v)}</string>`)
+    .join('\n');
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+\t<key>Label</key>
+\t<string>${xmlEscape(label)}</string>
+\t<key>ProgramArguments</key>
+\t<array>
+\t\t<string>/bin/bash</string>
+\t\t<string>${xmlEscape(script)}</string>
+\t</array>
+\t<key>EnvironmentVariables</key>
+\t<dict>
+${envEntries}
+\t</dict>
+\t<key>StartInterval</key>
+\t<integer>${Number(interval)}</integer>
+\t<key>RunAtLoad</key>
+\t<true/>
+\t<key>StandardOutPath</key>
+\t<string>${xmlEscape(logFile)}</string>
+\t<key>StandardErrorPath</key>
+\t<string>${xmlEscape(logFile)}</string>
+</dict>
+</plist>
+`;
+}
+
+function shellQuote(value) {
+  return `'${String(value).replace(/'/g, `'\\''`)}'`;
+}
+
+/** The crontab line that runs auto-pull.sh every 5 minutes. */
+export function buildCronLine({ script, env = {} }) {
+  const assignments = Object.entries(env)
+    .filter(([, v]) => v !== undefined && v !== null && v !== '')
+    .map(([k, v]) => `${k}=${shellQuote(v)}`)
+    .join(' ');
+  return `*/5 * * * * ${assignments ? `${assignments} ` : ''}/bin/bash ${shellQuote(script)} >/dev/null 2>&1`;
+}
+
+/** Replace every earlier auto-pull line with `line`; keep everything else. */
+export function mergeCrontab(existing, line) {
+  const kept = String(existing || '')
+    .split('\n')
+    .filter((l) => l.trim() && !/auto-pull\.sh/.test(l));
+  return `${[...kept, line].join('\n')}\n`;
+}
+
+/**
+ * The port the brain from `repoDir` serves: the PORT of the LaunchAgent that
+ * runs it (macOS), else null.
+ */
+export function detectLaunchAgentPort(repoDir, { home = os.homedir(), readPlist = readPlistValue } = {}) {
+  const dir = path.join(home, 'Library', 'LaunchAgents');
+  if (!fs.existsSync(dir)) return null;
+  for (const entry of fs.readdirSync(dir)) {
+    if (!entry.endsWith('.plist') || entry === `${AUTOPULL_LABEL}.plist`) continue;
+    const file = path.join(dir, entry);
+    const cwd = readPlist(file, 'WorkingDirectory');
+    const args = readPlist(file, 'ProgramArguments') || '';
+    if (cwd !== repoDir && !args.includes(`${repoDir}/`)) continue;
+    const port = readPlist(file, 'EnvironmentVariables:PORT');
+    if (port && /^\d+$/.test(port.trim())) return Number(port.trim());
+  }
+  return null;
+}
+
+function readPlistValue(file, key) {
+  try {
+    return execFileSync('/usr/libexec/PlistBuddy', ['-c', `Print :${key}`, file], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Install (or refresh) the timer. Returns what it did; `dryRun` returns the
+ * plan without writing.
+ */
+export function installAutoPull({
+  repoDir = packageRoot(),
+  port,
+  noBuild = false,
+  platform = process.platform,
+  home = os.homedir(),
+  dryRun = false,
+} = {}) {
+  const script = path.join(repoDir, 'scripts', 'auto-pull.sh');
+  if (!fs.existsSync(path.join(repoDir, '.git'))) {
+    throw new Error(`${repoDir} is not a git checkout; npm installs update through the daemon (total-recall update --apply).`);
+  }
+  if (!fs.existsSync(script)) throw new Error(`Missing ${script}`);
+
+  const resolvedPort =
+    port || Number(process.env.TR_PORT) || (platform === 'darwin' ? detectLaunchAgentPort(repoDir, { home }) : null) || 3000;
+  const env = {
+    TR_REPO_DIR: repoDir,
+    TR_PORT: String(resolvedPort),
+    TR_AUTOPULL_NO_BUILD: noBuild ? '1' : undefined,
+  };
+  const logDir = path.join(home, '.agent', 'logs');
+
+  if (platform === 'darwin') {
+    const plistPath = path.join(home, 'Library', 'LaunchAgents', `${AUTOPULL_LABEL}.plist`);
+    const plist = buildLaunchAgentPlist({
+      script,
+      env: { ...env, HOME: home, PATH: process.env.PATH },
+      logFile: path.join(logDir, 'auto-pull.launchd.log'),
+    });
+    if (dryRun) return { platform, plistPath, plist, port: resolvedPort };
+    fs.mkdirSync(logDir, { recursive: true });
+    fs.mkdirSync(path.dirname(plistPath), { recursive: true });
+    fs.writeFileSync(plistPath, plist, { mode: 0o644 });
+    const domain = `gui/${process.getuid()}`;
+    spawnSync('launchctl', ['bootout', `${domain}/${AUTOPULL_LABEL}`], { stdio: 'ignore' });
+    const boot = spawnSync('launchctl', ['bootstrap', domain, plistPath], { encoding: 'utf8' });
+    if (boot.status !== 0) throw new Error(`launchctl bootstrap failed: ${(boot.stderr || '').trim()}`);
+    return { platform, plistPath, port: resolvedPort, installed: true };
+  }
+
+  const line = buildCronLine({ script, env });
+  const current = spawnSync('crontab', ['-l'], { encoding: 'utf8' });
+  const merged = mergeCrontab(current.status === 0 ? current.stdout : '', line);
+  if (dryRun) return { platform, cronLine: line, crontab: merged, port: resolvedPort };
+  const write = spawnSync('crontab', ['-'], { input: merged, encoding: 'utf8' });
+  if (write.status !== 0) throw new Error(`crontab write failed: ${(write.stderr || '').trim()}`);
+  return { platform, cronLine: line, port: resolvedPort, installed: true };
+}

package/src/core/secrets-store.mjs

@@ -43,7 +43,8 @@ export function resolveUsagePath(brainDir) {
 /**
  * The master password: the environment first, then — on macOS — the Keychain
  * entry `secret rekey` already maintains (service `total-recall-secrets`,
- * overridable with TR_SECRETS_KEYCHAIN_SERVICE).
+ * overridable with TR_SECRETS_KEYCHAIN_SERVICE), then the host env file
+ * (see readTrEnvFilePassword).
  *
  * Only an interactive shell profile used to read the Keychain, so a
  * non-interactive process on the same Mac — a mesh `exec`, a release script —
@@ -52,13 +53,56 @@ export function resolveUsagePath(brainDir) {
  * disables the fallback.
  */
 let keychainPassword;
-export function secretsPassword({ env = process.env, readKeychain = readTrKeychainPassword } = {}) {
+export function secretsPassword({
+  env = process.env,
+  readKeychain = readTrKeychainPassword,
+  readEnvFile = readTrEnvFilePassword,
+} = {}) {
   const fromEnv = env.TR_SECRETS_PASSWORD || env.TR_MASTER_PASSWORD;
   if (fromEnv) return fromEnv;
-  if (env.TR_SECRETS_NO_KEYCHAIN === '1') return null;
-  if (readKeychain !== readTrKeychainPassword) return readKeychain(env) || null;
-  if (keychainPassword === undefined) keychainPassword = readKeychain(env) || null;
-  return keychainPassword;
+  if (env.TR_SECRETS_NO_KEYCHAIN !== '1') {
+    let fromKeychain;
+    if (readKeychain !== readTrKeychainPassword) fromKeychain = readKeychain(env) || null;
+    else {
+      if (keychainPassword === undefined) keychainPassword = readKeychain(env) || null;
+      fromKeychain = keychainPassword;
+    }
+    if (fromKeychain) return fromKeychain;
+  }
+  return readEnvFile(env) || null;
+}
+
+/**
+ * The host's env-file carrier: `TR_ENV_FILE`, else `~/.agent/tr.env` — the file
+ * `auto-pull.sh` sources and `secret rekey --env-file` rotates. Only a shell
+ * that sources it used to see the password, so a non-interactive process on a
+ * Linux host (an agent's tool shell, cron, a systemd unit) could not open the
+ * store its own user owns. The file is honoured only when it is a regular file
+ * (not a symlink) that this user owns and no one else can read or write, in a
+ * directory this user owns that no one else can write to.
+ */
+export function readTrEnvFilePassword(env = process.env) {
+  const file = env.TR_ENV_FILE || path.join(env.HOME || os.homedir(), '.agent', 'tr.env');
+  try {
+    // lstat: a symlink could point the carrier at a file someone else controls.
+    const stat = fs.lstatSync(file);
+    if (!stat.isFile() || (stat.mode & 0o077) !== 0) return null;
+    // A directory others can write to lets them replace the file.
+    const dir = fs.statSync(path.dirname(file));
+    if ((dir.mode & 0o022) !== 0) return null;
+    const uid = typeof process.getuid === 'function' ? process.getuid() : null;
+    if (uid !== null && (stat.uid !== uid || dir.uid !== uid)) return null;
+    const line = fs
+      .readFileSync(file, 'utf8')
+      .split(/\r?\n/)
+      .find((entry) => /^\s*(?:export\s+)?TR_SECRETS_PASSWORD\s*=/.test(entry));
+    if (!line) return null;
+    let value = line.slice(line.indexOf('=') + 1).trim();
+    if (value.length >= 2 && /^(['"]).*\1$/.test(value)) value = value.slice(1, -1);
+    return value || null;
+  } catch {
+    return null;
+  }
 }
 
 function readTrKeychainPassword(env) {

package/src/server/index.mjs

@@ -280,7 +280,10 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   const unembeddedBrains = (embeddingCoverage || []).filter(c => c.nodes > 0 && c.embedded === 0);
 
   // Determine overall status
-  const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
+  // No agent CLIs on PATH is normal for a headless server brain (a droplet has
+  // none), so it is reported in cli_agents but does not make the brain
+  // "degraded" — that status is for things that break memory itself.
+  const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead'
     || unembeddedBrains.length > 0;
 
   // Check Caddy and Cloudflare status