total-recall-brain 3.39.6 → 3.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.39.6",
+  "version": "3.40.0",
   "description": "Portable SSSS memory, local retrieval and task instructions with explicit plugin loading.",
   "type": "module",
   "bin": {
@@ -75,7 +75,7 @@
   "homepage": "https://github.com/gregiteen/total-recall#readme",
   "dependencies": {
     "@simplewebauthn/server": "^13.3.2",
-    "@ssss/cli": "npm:@gregiteen/ssss-cli@0.11.0",
+    "@ssss/cli": "npm:@gregiteen/ssss-cli@0.11.2",
     "bcrypt": "^6.0.0",
     "better-sqlite3": "^12.11.1",
     "cookie-parser": "^1.4.7",

package/src/cli/connect.mjs

@@ -43,7 +43,15 @@ export const CLIENTS = {
     globalMemoryPath: path.join(os.homedir(), '.dsh', 'memory', 'MEMORY.md'),
     globalUserPath:   path.join(os.homedir(), '.dsh', 'memory', 'user.md'),
     localTarget: 'AGENTS.md',
-    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') }
+    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') },
+    // DSH's default agent preset mounts dsh-skill-filesystem with no config, so
+    // the roots it scans are its own defaults: <dshHome>/skills (user-dsh) plus
+    // <repo>/.agents/skills and <repo>/.dsh/skills. Its host-plane
+    // `skill-filesystem` row is disabled because presets own local discovery, so
+    // a profile patch cannot configure these roots. Projecting Total Recall's
+    // authored skills into <dshHome>/skills is therefore the supported way to
+    // make them reach the `/` slash menu, globally and for every repo.
+    globalSkillsProjection: { dir: path.join('.dsh', 'skills') }
   },
   'deepseek-harness': {
     label: 'DeepSeek Harness',
@@ -51,7 +59,8 @@ export const CLIENTS = {
     globalMemoryPath: path.join(os.homedir(), '.dsh', 'memory', 'MEMORY.md'),
     globalUserPath:   path.join(os.homedir(), '.dsh', 'memory', 'user.md'),
     localTarget: 'AGENTS.md',
-    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') }
+    skillsProjection: { scope: 'project', dir: path.join('.agents', 'skills') },
+    globalSkillsProjection: { dir: path.join('.dsh', 'skills') }
   },
   openclaw: {
     label: 'OpenClaw',
@@ -1036,6 +1045,29 @@ export default async function connect(args) {
     }
   }
 
+  // Project GLOBAL skills into the target IDE's home-scoped Agent Skills
+  // directory. Some harnesses read only a home root for skills and never scan a
+  // repo directory, so a client that declares `globalSkillsProjection` needs this
+  // as well as the project projection above. `connect <ide>` must leave a client
+  // usable for both scopes; `init` reaches the same destination through
+  // projectSkillsForScope().
+  if (preset.globalSkillsProjection) {
+    const globalAgentDir = path.join(os.homedir(), '.agent');
+    const skills = discoverRepoSkills(globalAgentDir);
+    if (skills.length > 0) {
+      const destDir = path.join(os.homedir(), preset.globalSkillsProjection.dir);
+      const skillResults = projectSkillsAsCommands(destDir, skills, opts);
+      result.global_skill_commands = skillResults;
+      const fresh = skillResults.filter(r => r.action === 'linked' || r.action === 'copied');
+      if (fresh.length > 0) {
+        result.notes.push(
+          `  Global skills projected → ${destDir}/:\n` +
+          fresh.map(r => `    ${r.name}`).join('\n')
+        );
+      }
+    }
+  }
+
   // Record this client in the registry so `status` can report freshness
   registerClient(brainDir, opts.client, preset, result.projection?.targetPath || null);
 

package/src/cli/init.mjs

@@ -249,6 +249,9 @@ export default async function init(args) {
   if (!fs.existsSync(path.join(authSource, 'plugin.json'))) throw new Error('Required auth scaffold is missing');
   if (!fs.existsSync(authTarget) && !opts.dryRun) fs.cpSync(authSource, authTarget, { recursive: true, errorOnExist: true, force: false });
 
+  // Every scaffold plugin's node_modules are wired in Step 2.5, after the skill
+  // tree has been copied into the brain (see the note there).
+
   // ── Step 2: Seed core skills ──
   logStep('2/4', 'Installing core skills into .agent/skills/');
 
@@ -304,9 +307,30 @@ export default async function init(args) {
   }
 
   // ── Step 2.5: Create data directories inside the meta-skill (brain dir) ──
+  // Wire every scaffold plugin's dependencies AFTER the skill copy above: the
+  // plugin directories only exist once the skill tree has been copied, so an
+  // earlier pass silently skipped everything except auth (copied separately).
+  // A plugin shipped without its node_modules aborts plugin loading for the whole
+  // scope — the symptom was 'Cannot find package express' on an unrelated command.
   if (!opts.dryRun) {
     const { preparePluginDependencies } = await import('../core/plugin-dependencies.mjs');
-    preparePluginDependencies(path.join(brainDir, 'plugins', 'auth'));
+    const installedPluginsDir = path.join(brainDir, 'plugins');
+    for (const id of (fs.existsSync(installedPluginsDir) ? fs.readdirSync(installedPluginsDir) : [])) {
+      const target = path.join(installedPluginsDir, id);
+      if (!fs.existsSync(path.join(target, 'package.json'))) continue;
+      // One plugin whose declared dependency cannot be wired must not abort the
+      // whole run: the steps after this one — global skill projection into each
+      // IDE, then the vault compile — are independent of it. Before this guard a
+      // single unresolvable optional dependency (observed: '@gregiteen/ssss-cli'
+      // in a vendored plugin) killed `init` before skill projection, so no IDE
+      // ever received the global skills and their slash menus stayed empty with
+      // no error the user could connect to the cause.
+      try {
+        preparePluginDependencies(target);
+      } catch (err) {
+        logWarn(`Could not wire dependencies for plugin '${id}': ${err.message}`);
+      }
+    }
   }
   logStep('2.5/4', 'Creating user data directories inside meta-skill');
 

package/src/cli/plugin/create.mjs

@@ -300,6 +300,22 @@ export async function run(argv = []) {
   const args = Array.isArray(argv) ? argv.slice(3) : [];
   const sub = args[0] || "status";
   const json = args.includes("--json");
+  const wantsHelp = args.includes("--help") || args.includes("-h");
+
+  if (wantsHelp || sub === "help") {
+    const subs = ["status", ...Object.keys(IMPLEMENTED)];
+    console.log(\`${pluginName} v${manifest.version}
+
+Usage: total-recall ${id} <subcommand> [options]
+
+Subcommands:
+\${subs.map((s) => \`  \${s}\`).join("\\n")}
+
+Options:
+  --json      Machine-readable output
+  --help, -h  Show this help\`);
+    return;
+  }
 
   if (sub === "status") {
     const status = { plugin: ${JSON.stringify(id)}, implemented: Object.keys(IMPLEMENTED) };
@@ -312,7 +328,7 @@ export async function run(argv = []) {
 
   if (IMPLEMENTED[sub]) return IMPLEMENTED[sub](args.slice(1));
 
-  console.error(\`${pluginName}: "\${sub}" is not implemented.\`);
+  console.error(\`${pluginName}: "\${sub}" is not implemented. Run 'total-recall ${id} --help' for available subcommands.\`);
   process.exitCode = 2;
 }
 export default run;
@@ -494,7 +510,7 @@ const manifest = JSON.parse(fs.readFileSync(path.join(root, 'plugin.json'), 'utf
 
 test('manifest identifies the plugin', () => {
   assert.equal(manifest.id, ${JSON.stringify(id)});
-  assert.match(manifest.version, /^\\\\d+\\\\.\\\\d+\\\\.\\\\d+/);
+  assert.match(manifest.version, /^\\d+\\.\\d+\\.\\d+/);
   assert.equal(manifest.license, 'MIT');
 });
 

package/src/cli/plugin/install.mjs

@@ -33,7 +33,17 @@ export async function installPlugin(args = []) {
       return;
     }
     const { autoCompileSurfaces } = await import('../../core/auto-compile.mjs');
-    if (!await autoCompileSurfaces({ layer: isGlobal ? 'global' : 'project', semantic: false, force: true })) process.exitCode = 1;
+    // Compile the layer the plugin actually landed in. `scope` is not enough on
+    // its own: from a directory with no project brain, projectPluginsDir(cwd)
+    // resolves to the global plugins dir, so a plugin installed into the global
+    // brain still reports scope 'project'. Ask the resolver whether a project
+    // brain exists at all, and otherwise fall back to the global layer — the one
+    // that was really written. Assuming 'project' compiled nothing and printed
+    // "Auto-compile failed: No project brain found" after a successful install.
+    const { resolveBrainDir } = await import('../agent-dir.mjs');
+    let targetLayer = 'global';
+    try { resolveBrainDir('project'); targetLayer = 'project'; } catch { targetLayer = 'global'; }
+    if (!await autoCompileSurfaces({ layer: targetLayer, semantic: false, force: true })) process.exitCode = 1;
   } catch (err) {
     console.error(`❌ ${err.message}`);
     process.exit(1);

package/src/core/brain-updates.mjs

@@ -48,7 +48,13 @@ export function discoverBrainConsumers(roots, { onError } = {}) {
       if (deps.length) consumers.push({ root: real, file, pkg, deps });
     }
     for (const entry of fs.readdirSync(real, { withFileTypes: true })) {
-      if (entry.isDirectory() && !entry.name.startsWith('.') && !ignored.has(entry.name)) walk(path.join(real, entry.name));
+      if (!entry.isDirectory() || ignored.has(entry.name)) continue;
+      // Dot-directories are skipped to avoid descending into .git and similar,
+      // but .agent holds the brain and its plugin installs. A plugin that runs
+      // `npm install` keeps its own node_modules there, and skipping the whole
+      // dot tree left those copies stranded on an old version.
+      if (entry.name.startsWith('.') && entry.name !== '.agent') continue;
+      walk(path.join(real, entry.name));
     }
   }
   for (const root of roots) walk(root);

package/src/core/embeddings.mjs

@@ -316,6 +316,23 @@ async function getOpenAIEmbedding(text, model = 'text-embedding-3-small') {
 }
 
 export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
+  const { embedding } = await getEmbeddingWithProvenance(text, model);
+  return embedding;
+}
+
+/**
+ * Resolve an embedding together with the provider and model that actually
+ * served it.
+ *
+ * The requested model name is not the serving model: Ollama is tried first and
+ * answers with a model discovered from the server, so a row labelled with the
+ * hosted default would misreport where the vector came from and whether the
+ * text ever left the machine. Persist the returned `provider`/`model`, never
+ * the requested default.
+ *
+ * @returns {Promise<{embedding: number[], provider: string, model: string}>}
+ */
+export async function getEmbeddingWithProvenance(text, model = DEFAULT_EMBED_MODEL) {
   if (process.env.TR_EMBEDDINGS_DISABLED === '1') {
     throw new Error('Semantic embeddings explicitly disabled; local recall remains available');
   }
@@ -325,6 +342,8 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
   const openaiKey = resolveOpenAiApiKey();
 
   let embedding;
+  let provider = null;
+  let servedBy = null;
 
   // Local/mesh Ollama first: no key, no spend, no rate limit, and unaffected by
   // a hosted provider running out of credit. The model is discovered from the
@@ -342,6 +361,7 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
           model: ollamaModel,
           dims: EMBEDDING_DIMS,
         });
+        if (embedding) { provider = 'ollama'; servedBy = ollamaModel; }
       }
     }
   } catch (err) {
@@ -354,6 +374,7 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
   if (!embedding && openRouterKey) {
     try {
       embedding = await getOpenRouterEmbedding(input);
+      if (embedding) { provider = 'openrouter'; servedBy = 'openai/text-embedding-3-small'; }
     } catch (err) {
       logger.debug('embeddings: OpenRouter failed, trying Google fallback', { err: err.message });
     }
@@ -362,6 +383,9 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
   if (!embedding && googleKey) {
     try {
       embedding = await getGoogleEmbedding(input, model);
+      // getGoogleEmbedding resolves the concrete model internally; record that,
+      // not the requested default, so provenance is exact.
+      if (embedding) { provider = 'google'; servedBy = await resolveEmbeddingModel(model); }
     } catch (err) {
       logger.debug('embeddings: Google API failed, trying OpenAI fallback', { err: err.message });
     }
@@ -370,6 +394,7 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
   if (!embedding && openaiKey) {
     try {
       embedding = await getOpenAIEmbedding(input);
+      if (embedding) { provider = 'openai'; servedBy = 'text-embedding-3-small'; }
     } catch (err) {
       logger.debug('embeddings: OpenAI fallback also failed', { err: err.message });
       throw err;
@@ -383,7 +408,7 @@ export async function getEmbedding(text, _unused, model = DEFAULT_EMBED_MODEL) {
     );
   }
 
-  return embedding;
+  return { embedding, provider, model: servedBy };
 }
 
 export function cosineSimilarity(a, b) {
@@ -511,7 +536,10 @@ export async function buildEmbeddingsIndex(nodes, derivedDir, opts = {}) {
     }
 
     try {
-      const parentEmbedding = await getEmbedding(text, undefined, model);
+      // Record the provider/model that actually served this vector. The row used
+      // to be stamped with the requested default even when local Ollama answered,
+      // which made provenance — and "did this text leave the machine?" — unreadable.
+      const { embedding: parentEmbedding, model: servedModel } = await getEmbeddingWithProvenance(text, model);
       const now = new Date().toISOString();
       const embeddingJson = JSON.stringify(parentEmbedding);
 
@@ -524,7 +552,7 @@ export async function buildEmbeddingsIndex(nodes, derivedDir, opts = {}) {
             generated_at=excluded.generated_at,
             content_sha256=excluded.content_sha256,
             embedding_json=excluded.embedding_json
-        `).run(node.slug, model, now, contentHash, embeddingJson);
+        `).run(node.slug, servedModel || model, now, contentHash, embeddingJson);
         
         const rowId = db.prepare('SELECT rowid FROM vault_embeddings WHERE slug = ?').get(node.slug).rowid;
         vssReplaceVault(db, rowId, embeddingJson);
@@ -658,7 +686,7 @@ export async function buildSessionEmbeddingsIndex(sessionsDir, derivedDir, opts
 
       for (let i = 0; i < chunks.length; i++) {
         const key = chunks.length === 1 ? sessionId : `${sessionId}:chunk-${i}`;
-        const embedding = await getEmbedding(chunks[i], undefined, model);
+        const { embedding, model: servedModel } = await getEmbeddingWithProvenance(chunks[i], model);
         const snippet = chunks[i].slice(0, 300);
         
         db.transaction(() => {
@@ -670,7 +698,7 @@ export async function buildSessionEmbeddingsIndex(sessionsDir, derivedDir, opts
               model=excluded.model,
               generated_at=excluded.generated_at,
               embedding_json=excluded.embedding_json
-          `).run(key, sessionId, i, chunks.length, snippet, model, new Date().toISOString(), JSON.stringify(embedding));
+          `).run(key, sessionId, i, chunks.length, snippet, servedModel || model, new Date().toISOString(), JSON.stringify(embedding));
           
           const rowId = db.prepare('SELECT rowid FROM session_embeddings WHERE key = ?').get(key).rowid;
           vssReplaceSession(db, rowId, JSON.stringify(embedding));

package/src/core/frontmatter.mjs

@@ -1,18 +1,21 @@
-/** Markdown metadata is data: YAML/JSON only, with no executable engines. */
-import * as yamlModule from 'js-yaml';
-const yaml = yamlModule.default || yamlModule;
-// Existing source checkouts can still have the legacy CommonJS parser installed.
-const load = yaml.safeLoad || yaml.load;
-const dump = yaml.safeDump || yaml.dump;
-// Preserve the existing YAML 1.2 scalar rules plus dates/merge/collection tags.
-const schema = yaml.CORE_SCHEMA?.withTags
-  ? yaml.CORE_SCHEMA.withTags(yaml.timestampTag, yaml.mergeTag, yaml.binaryTag, yaml.omapTag, yaml.pairsTag, yaml.setTag)
-  : yaml.DEFAULT_SAFE_SCHEMA;
+/**
+ * Markdown metadata is data: YAML/JSON only, with no executable engines.
+ *
+ * YAML frontmatter is read and written by the SSSS parser (@ssss/cli/frontmatter),
+ * the single implementation shared with every SSSS host, so Total Recall and SSSS
+ * can never disagree about a vault file. This wrapper keeps the gray-matter-style
+ * API, the fence/language checks and the CRLF handling.
+ */
+import { parseFrontmatter, serializeDocument, repairEscapedNewlines } from '@ssss/cli/frontmatter';
+
+// YAML tags (`!!js/function`, `!!python/object`, ...) can request code; SSSS reads
+// them as plain text, so refuse them up front instead of storing them as data.
+const YAML_TAG = /(?:^|[:,\[{-])[ \t]*!!\S/m;
 
 export default function matter(input) {
   const original = Buffer.isBuffer(input) ? input.toString('utf8') : typeof input === 'string' ? input : input?.content;
   if (typeof original !== 'string') throw new TypeError('Markdown content must be a string');
-  const text = original.replace(/^\uFEFF/, '');
+  const text = repairEscapedNewlines(original.replace(/^/, ''));
   const file = { data: {}, content: text, excerpt: '', orig: original };
   const opening = /^---([^\r\n]*)\r?\n/.exec(text);
   if (!opening || opening[1].startsWith('-')) return file;
@@ -22,7 +25,8 @@ export default function matter(input) {
   const remainder = text.slice(opening[0].length);
   const closing = /^---(?:\r?\n|$)/m.exec(remainder);
   const metadata = closing ? remainder.slice(0, closing.index) : remainder;
-  file.data = (language === 'json' ? JSON.parse(metadata || '{}') : load(metadata, { schema })) || {};
+  if (language !== 'json' && YAML_TAG.test(metadata)) throw new Error('YAML tags are not allowed in frontmatter');
+  file.data = (language === 'json' ? JSON.parse(metadata || '{}') : parseFrontmatter(metadata.replace(/\r\n/g, '\n'), { strict: true })) || {};
   file.content = closing ? remainder.slice(closing.index + closing[0].length) : '';
   file.matter = metadata;
   file.language = language;
@@ -30,11 +34,14 @@ export default function matter(input) {
 }
 
 const newline = value => value.endsWith('\n') ? value : value + '\n';
-matter.stringify = (input, data, options = {}) => {
+// Dates become ISO strings (SSSS keeps timestamps as strings); undefined is dropped.
+const plain = value => JSON.parse(JSON.stringify(value, (_key, v) => (v instanceof Date ? v.toISOString() : v)));
+
+matter.stringify = (input, data) => {
   if (typeof input === 'string' && data == null) return input;
   const file = typeof input === 'string' ? matter(input) : input;
   if (!file || typeof file.content !== 'string') throw new TypeError('Markdown content must be a string');
-  const metadata = { ...file.data, ...(data ?? {}) };
-  const serialized = dump(metadata, options).trim();
-  return (serialized === '{}' ? '' : `---\n${serialized}\n---\n`) + newline(file.content);
+  const metadata = plain({ ...file.data, ...(data ?? {}) });
+  const head = Object.keys(metadata).length ? serializeDocument(metadata, '') : '';
+  return head + newline(file.content);
 };

package/src/core/plugin-state-store.mjs

@@ -71,8 +71,22 @@ export async function executePluginState(request) {
   }
   // A guard serializes state read/compare/commit across native worker processes.
   const lock = path.join(root, '.mutation-lock');
-  try { fs.mkdirSync(lock, { mode: 0o700 }); }
-  catch { throw new Error('Plugin state mutation is busy or its ownership guard is unreadable'); }
+  try {
+    fs.mkdirSync(lock, { mode: 0o700 });
+  } catch (err) {
+    // Distinguish a genuine concurrent mutation from an environment failure so
+    // callers are not told the store is busy when it is actually inaccessible.
+    if (err && err.code === 'EEXIST') {
+      throw new Error('Plugin state mutation is busy; another mutation holds the guard');
+    }
+    if (err && (err.code === 'EACCES' || err.code === 'EPERM' || err.code === 'EROFS')) {
+      throw new Error(
+        `Plugin state store is not writable (${err.code}): cannot create the mutation guard at ${lock}. ` +
+        'Grant write access to this brain root, or point TR_SECRETS_BRAIN at a store you can write to.'
+      );
+    }
+    throw new Error(`Plugin state mutation guard could not be created at ${lock}: ${err?.message || err}`);
+  }
   try {
     const durableKeys = new FileIdempotencyStore(path.join(root, '.idempotency'));
     const intentHash = crypto.createHash('sha256').update(JSON.stringify({ path: relative, action, value, expectedVersion, format: request.format })).digest('hex');

package/src/core/surface.mjs

@@ -758,7 +758,10 @@ export async function compileSurface({ vaultDir, skillsDir, derivedDir, instruct
     logger.info('surface', 'Semantic embeddings explicitly disabled; local indexes updated.', { derivedDir });
   } else try {
     const { buildEmbeddingsIndex } = await import('./embeddings.mjs');
-    const built = await buildEmbeddingsIndex(nodes, derivedDir);
+    // Forward `force`: the builder is content-hash incremental and skips a node
+    // whose text is unchanged, so without this `compile --force` rebuilt nothing
+    // and rows kept an outdated model label.
+    const built = await buildEmbeddingsIndex(nodes, derivedDir, { force });
     semanticResult = {
       indexed: built.built,
       skipped: built.skipped,