total-recall-brain 3.39.7 → 3.40.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.39.7",
+  "version": "3.40.1",
   "description": "Portable SSSS memory, local retrieval and task instructions with explicit plugin loading.",
   "type": "module",
   "bin": {
@@ -75,7 +75,7 @@
   "homepage": "https://github.com/gregiteen/total-recall#readme",
   "dependencies": {
     "@simplewebauthn/server": "^13.3.2",
-    "@ssss/cli": "npm:@gregiteen/ssss-cli@0.11.0",
+    "@ssss/cli": "npm:@gregiteen/ssss-cli@0.11.2",
     "bcrypt": "^6.0.0",
     "better-sqlite3": "^12.11.1",
     "cookie-parser": "^1.4.7",

package/src/cli/backup.mjs

@@ -17,6 +17,7 @@
  */
 
 import { spawnSync } from 'node:child_process';
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -64,8 +65,22 @@ function parseArgs(args) {
   return opts;
 }
 
+/**
+ * The backup repo's git directory lives OUTSIDE the project tree, under
+ * ~/.agent/brain-backups. A `.git` inside `<project>/.agent/skills/total-recall`
+ * makes the project repo see an embedded repository: `git add -A` records it as a
+ * gitlink (mode 160000) with no .gitmodules, which cannot be cloned back.
+ */
+export function brainGitDir(skillDir) {
+  const abs = path.resolve(skillDir);
+  const owner = path.basename(path.resolve(abs, '..', '..', '..')) || 'brain';
+  const tag = crypto.createHash('sha1').update(abs).digest('hex').slice(0, 8);
+  return path.join(os.homedir(), '.agent', 'brain-backups', `${owner}-${tag}.git`);
+}
+
 function git(cwd, ...gitArgs) {
-  const result = spawnSync('git', gitArgs, { cwd, stdio: 'pipe' });
+  const env = { ...process.env, GIT_DIR: brainGitDir(cwd), GIT_WORK_TREE: path.resolve(cwd) };
+  const result = spawnSync('git', gitArgs, { cwd, env, stdio: 'pipe' });
   return {
     ok: result.status === 0,
     stdout: result.stdout?.toString().trim() || '',
@@ -73,6 +88,56 @@ function git(cwd, ...gitArgs) {
   };
 }
 
+/**
+ * Move a legacy in-tree `<brain>/.git` to the external git dir, and keep the
+ * enclosing project repo from absorbing the brain directory. Idempotent.
+ */
+export function adoptExternalGitDir(skillDir) {
+  const inTree = path.join(skillDir, '.git');
+  const external = brainGitDir(skillDir);
+  let moved = false;
+  if (fs.existsSync(inTree) && fs.statSync(inTree).isDirectory() && !fs.existsSync(external)) {
+    fs.mkdirSync(path.dirname(external), { recursive: true });
+    try {
+      fs.renameSync(inTree, external);
+    } catch {
+      fs.cpSync(inTree, external, { recursive: true });
+      fs.rmSync(inTree, { recursive: true, force: true });
+    }
+    spawnSync('git', ['--git-dir', external, 'config', 'core.worktree', path.resolve(skillDir)], { stdio: 'pipe' });
+    moved = true;
+  }
+  if (fs.existsSync(external) && !fs.existsSync(inTree)) excludeFromParentRepo(skillDir);
+  return moved;
+}
+
+/** Add the brain dir to the enclosing repo's local exclude unless that repo tracks it on purpose. */
+function excludeFromParentRepo(skillDir) {
+  try {
+    const top = spawnSync('git', ['-C', skillDir, 'rev-parse', '--show-toplevel'], { stdio: 'pipe' });
+    if (top.status !== 0) return;
+    const root = top.stdout.toString().trim();
+    const rel = path.relative(fs.realpathSync(root), fs.realpathSync(skillDir)).split(path.sep).join('/');
+    if (!rel || rel.startsWith('..')) return;
+    const tracked = spawnSync('git', ['-C', root, 'ls-files', '-s', '--', rel], { stdio: 'pipe' }).stdout.toString().trim();
+    if (/^160000 /.test(tracked) && !tracked.includes('\n')) {
+      // A stale gitlink (embedded repo with no .gitmodules) cannot be cloned back: unstage it.
+      spawnSync('git', ['-C', root, 'rm', '--cached', '-f', '-q', '--', rel], { stdio: 'pipe' });
+    } else if (tracked) {
+      return; // the project repo versions its brain files itself
+    }
+    const excludePath = path.join(root, '.git', 'info', 'exclude');
+    try {
+      const entry = `/${rel}/`;
+      const current = fs.existsSync(excludePath) ? fs.readFileSync(excludePath, 'utf8') : '';
+      if (!current.split('\n').includes(entry)) {
+        fs.mkdirSync(path.dirname(excludePath), { recursive: true });
+        fs.appendFileSync(excludePath, `${current && !current.endsWith('\n') ? '\n' : ''}${entry}\n`);
+      }
+    } catch { /* parent exclude is best effort */ }
+  } catch { /* best effort: never fail a backup over the parent's exclude file */ }
+}
+
 function defaultOutput(encrypt) {
   const stamp = new Date().toISOString().replace(/[:.]/g, '-');
   const suffix = encrypt ? '.tar.gz.gpg' : '.tar.gz';
@@ -169,8 +234,10 @@ async function pushGitBackup(remote, layer = 'auto') {
   // rule are corrected on their next backup.
   ensureBrainGitignore(SKILL_DIR);
 
-  // Initialise the skill folder as a git repo if needed
-  const gitDir = path.join(SKILL_DIR, '.git');
+  // Initialise the skill folder as a git repo if needed. The git dir lives outside
+  // the project tree; a legacy in-tree .git is moved there first.
+  adoptExternalGitDir(SKILL_DIR);
+  const gitDir = brainGitDir(SKILL_DIR);
   if (!fs.existsSync(gitDir)) {
     console.error(`  🔧 Initialising git repo in ${SKILL_DIR}`);
     const init = git(SKILL_DIR, 'init', '-b', 'main');
@@ -178,6 +245,7 @@ async function pushGitBackup(remote, layer = 'auto') {
       git(SKILL_DIR, 'init');
       git(SKILL_DIR, 'checkout', '-b', 'main');
     }
+    excludeFromParentRepo(SKILL_DIR);
   }
 
   // Resolve the remote: if the argument is an existing remote name, use its URL.

package/src/cli/connect.mjs

@@ -43,7 +43,15 @@ export const CLIENTS = {
     globalMemoryPath: path.join(os.homedir(), '.dsh', 'memory', 'MEMORY.md'),
     globalUserPath:   path.join(os.homedir(), '.dsh', 'memory', 'user.md'),
     localTarget: 'AGENTS.md',
-    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') }
+    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') },
+    // DSH's default agent preset mounts dsh-skill-filesystem with no config, so
+    // the roots it scans are its own defaults: <dshHome>/skills (user-dsh) plus
+    // <repo>/.agents/skills and <repo>/.dsh/skills. Its host-plane
+    // `skill-filesystem` row is disabled because presets own local discovery, so
+    // a profile patch cannot configure these roots. Projecting Total Recall's
+    // authored skills into <dshHome>/skills is therefore the supported way to
+    // make them reach the `/` slash menu, globally and for every repo.
+    globalSkillsProjection: { dir: path.join('.dsh', 'skills') }
   },
   'deepseek-harness': {
     label: 'DeepSeek Harness',
@@ -51,7 +59,8 @@ export const CLIENTS = {
     globalMemoryPath: path.join(os.homedir(), '.dsh', 'memory', 'MEMORY.md'),
     globalUserPath:   path.join(os.homedir(), '.dsh', 'memory', 'user.md'),
     localTarget: 'AGENTS.md',
-    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') }
+    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') },
+    globalSkillsProjection: { dir: path.join('.dsh', 'skills') }
   },
   openclaw: {
     label: 'OpenClaw',
@@ -1036,6 +1045,29 @@ export default async function connect(args) {
     }
   }
 
+  // Project GLOBAL skills into the target IDE's home-scoped Agent Skills
+  // directory. Some harnesses read only a home root for skills and never scan a
+  // repo directory, so a client that declares `globalSkillsProjection` needs this
+  // as well as the project projection above. `connect <ide>` must leave a client
+  // usable for both scopes; `init` reaches the same destination through
+  // projectSkillsForScope().
+  if (preset.globalSkillsProjection) {
+    const globalAgentDir = path.join(os.homedir(), '.agent');
+    const skills = discoverRepoSkills(globalAgentDir);
+    if (skills.length > 0) {
+      const destDir = path.join(os.homedir(), preset.globalSkillsProjection.dir);
+      const skillResults = projectSkillsAsCommands(destDir, skills, opts);
+      result.global_skill_commands = skillResults;
+      const fresh = skillResults.filter(r => r.action === 'linked' || r.action === 'copied');
+      if (fresh.length > 0) {
+        result.notes.push(
+          `  Global skills projected → ${destDir}/:\n` +
+          fresh.map(r => `    ${r.name}`).join('\n')
+        );
+      }
+    }
+  }
+
   // Record this client in the registry so `status` can report freshness
   registerClient(brainDir, opts.client, preset, result.projection?.targetPath || null);
 

package/src/cli/init.mjs

@@ -318,7 +318,18 @@ export default async function init(args) {
     for (const id of (fs.existsSync(installedPluginsDir) ? fs.readdirSync(installedPluginsDir) : [])) {
       const target = path.join(installedPluginsDir, id);
       if (!fs.existsSync(path.join(target, 'package.json'))) continue;
-      preparePluginDependencies(target);
+      // One plugin whose declared dependency cannot be wired must not abort the
+      // whole run: the steps after this one — global skill projection into each
+      // IDE, then the vault compile — are independent of it. Before this guard a
+      // single unresolvable optional dependency (observed: '@gregiteen/ssss-cli'
+      // in a vendored plugin) killed `init` before skill projection, so no IDE
+      // ever received the global skills and their slash menus stayed empty with
+      // no error the user could connect to the cause.
+      try {
+        preparePluginDependencies(target);
+      } catch (err) {
+        logWarn(`Could not wire dependencies for plugin '${id}': ${err.message}`);
+      }
     }
   }
   logStep('2.5/4', 'Creating user data directories inside meta-skill');

package/src/core/brain-updates.mjs

@@ -48,7 +48,13 @@ export function discoverBrainConsumers(roots, { onError } = {}) {
       if (deps.length) consumers.push({ root: real, file, pkg, deps });
     }
     for (const entry of fs.readdirSync(real, { withFileTypes: true })) {
-      if (entry.isDirectory() && !entry.name.startsWith('.') && !ignored.has(entry.name)) walk(path.join(real, entry.name));
+      if (!entry.isDirectory() || ignored.has(entry.name)) continue;
+      // Dot-directories are skipped to avoid descending into .git and similar,
+      // but .agent holds the brain and its plugin installs. A plugin that runs
+      // `npm install` keeps its own node_modules there, and skipping the whole
+      // dot tree left those copies stranded on an old version.
+      if (entry.name.startsWith('.') && entry.name !== '.agent') continue;
+      walk(path.join(real, entry.name));
     }
   }
   for (const root of roots) walk(root);

package/src/core/frontmatter.mjs

@@ -1,18 +1,21 @@
-/** Markdown metadata is data: YAML/JSON only, with no executable engines. */
-import * as yamlModule from 'js-yaml';
-const yaml = yamlModule.default || yamlModule;
-// Existing source checkouts can still have the legacy CommonJS parser installed.
-const load = yaml.safeLoad || yaml.load;
-const dump = yaml.safeDump || yaml.dump;
-// Preserve the existing YAML 1.2 scalar rules plus dates/merge/collection tags.
-const schema = yaml.CORE_SCHEMA?.withTags
-  ? yaml.CORE_SCHEMA.withTags(yaml.timestampTag, yaml.mergeTag, yaml.binaryTag, yaml.omapTag, yaml.pairsTag, yaml.setTag)
-  : yaml.DEFAULT_SAFE_SCHEMA;
+/**
+ * Markdown metadata is data: YAML/JSON only, with no executable engines.
+ *
+ * YAML frontmatter is read and written by the SSSS parser (@ssss/cli/frontmatter),
+ * the single implementation shared with every SSSS host, so Total Recall and SSSS
+ * can never disagree about a vault file. This wrapper keeps the gray-matter-style
+ * API, the fence/language checks and the CRLF handling.
+ */
+import { parseFrontmatter, serializeDocument, repairEscapedNewlines } from '@ssss/cli/frontmatter';
+
+// YAML tags (`!!js/function`, `!!python/object`, ...) can request code; SSSS reads
+// them as plain text, so refuse them up front instead of storing them as data.
+const YAML_TAG = /(?:^|[:,\[{-])[ \t]*!!\S/m;
 
 export default function matter(input) {
   const original = Buffer.isBuffer(input) ? input.toString('utf8') : typeof input === 'string' ? input : input?.content;
   if (typeof original !== 'string') throw new TypeError('Markdown content must be a string');
-  const text = original.replace(/^\uFEFF/, '');
+  const text = repairEscapedNewlines(original.replace(/^/, ''));
   const file = { data: {}, content: text, excerpt: '', orig: original };
   const opening = /^---([^\r\n]*)\r?\n/.exec(text);
   if (!opening || opening[1].startsWith('-')) return file;
@@ -22,7 +25,8 @@ export default function matter(input) {
   const remainder = text.slice(opening[0].length);
   const closing = /^---(?:\r?\n|$)/m.exec(remainder);
   const metadata = closing ? remainder.slice(0, closing.index) : remainder;
-  file.data = (language === 'json' ? JSON.parse(metadata || '{}') : load(metadata, { schema })) || {};
+  if (language !== 'json' && YAML_TAG.test(metadata)) throw new Error('YAML tags are not allowed in frontmatter');
+  file.data = (language === 'json' ? JSON.parse(metadata || '{}') : parseFrontmatter(metadata.replace(/\r\n/g, '\n'), { strict: true })) || {};
   file.content = closing ? remainder.slice(closing.index + closing[0].length) : '';
   file.matter = metadata;
   file.language = language;
@@ -30,11 +34,14 @@ export default function matter(input) {
 }
 
 const newline = value => value.endsWith('\n') ? value : value + '\n';
-matter.stringify = (input, data, options = {}) => {
+// Dates become ISO strings (SSSS keeps timestamps as strings); undefined is dropped.
+const plain = value => JSON.parse(JSON.stringify(value, (_key, v) => (v instanceof Date ? v.toISOString() : v)));
+
+matter.stringify = (input, data) => {
   if (typeof input === 'string' && data == null) return input;
   const file = typeof input === 'string' ? matter(input) : input;
   if (!file || typeof file.content !== 'string') throw new TypeError('Markdown content must be a string');
-  const metadata = { ...file.data, ...(data ?? {}) };
-  const serialized = dump(metadata, options).trim();
-  return (serialized === '{}' ? '' : `---\n${serialized}\n---\n`) + newline(file.content);
+  const metadata = plain({ ...file.data, ...(data ?? {}) });
+  const head = Object.keys(metadata).length ? serializeDocument(metadata, '') : '';
+  return head + newline(file.content);
 };

package/src/core/plugin-state-store.mjs

@@ -71,8 +71,22 @@ export async function executePluginState(request) {
   }
   // A guard serializes state read/compare/commit across native worker processes.
   const lock = path.join(root, '.mutation-lock');
-  try { fs.mkdirSync(lock, { mode: 0o700 }); }
-  catch { throw new Error('Plugin state mutation is busy or its ownership guard is unreadable'); }
+  try {
+    fs.mkdirSync(lock, { mode: 0o700 });
+  } catch (err) {
+    // Distinguish a genuine concurrent mutation from an environment failure so
+    // callers are not told the store is busy when it is actually inaccessible.
+    if (err && err.code === 'EEXIST') {
+      throw new Error('Plugin state mutation is busy; another mutation holds the guard');
+    }
+    if (err && (err.code === 'EACCES' || err.code === 'EPERM' || err.code === 'EROFS')) {
+      throw new Error(
+        `Plugin state store is not writable (${err.code}): cannot create the mutation guard at ${lock}. ` +
+        'Grant write access to this brain root, or point TR_SECRETS_BRAIN at a store you can write to.'
+      );
+    }
+    throw new Error(`Plugin state mutation guard could not be created at ${lock}: ${err?.message || err}`);
+  }
   try {
     const durableKeys = new FileIdempotencyStore(path.join(root, '.idempotency'));
     const intentHash = crypto.createHash('sha256').update(JSON.stringify({ path: relative, action, value, expectedVersion, format: request.format })).digest('hex');

package/src/core/surface.mjs

@@ -758,7 +758,10 @@ export async function compileSurface({ vaultDir, skillsDir, derivedDir, instruct
     logger.info('surface', 'Semantic embeddings explicitly disabled; local indexes updated.', { derivedDir });
   } else try {
     const { buildEmbeddingsIndex } = await import('./embeddings.mjs');
-    const built = await buildEmbeddingsIndex(nodes, derivedDir);
+    // Forward `force`: the builder is content-hash incremental and skips a node
+    // whose text is unchanged, so without this `compile --force` rebuilt nothing
+    // and rows kept an outdated model label.
+    const built = await buildEmbeddingsIndex(nodes, derivedDir, { force });
     semanticResult = {
       indexed: built.built,
       skipped: built.skipped,