total-recall-brain 3.46.1 → 3.47.0

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/config.mjs

@@ -6,10 +6,13 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 
+// No model default: models and prices change too often to fix one in code, and a stale
+// default (deepseek/deepseek-chat) cost about 5x a current model of the same family.
+// A refresh with no configured model stops before it calls a provider.
 export const DEFAULT_OPENWIKI_CONFIG = Object.freeze({
   provider: 'openrouter',
-  model: 'deepseek/deepseek-chat',
-  fallback_model: 'deepseek/deepseek-chat',
+  model: null,
+  fallback_model: null,
   max_tokens: 4096,
   timeout_ms: 300000
 });
@@ -69,8 +72,8 @@ export function saveOpenWikiConfig(config, options = {}) {
   const filePath = resolveConfigPath(options);
   const validated = {
     provider: String(config.provider || DEFAULT_OPENWIKI_CONFIG.provider).toLowerCase(),
-    model: String(config.model || DEFAULT_OPENWIKI_CONFIG.model).trim(),
-    fallback_model: String(config.fallback_model || DEFAULT_OPENWIKI_CONFIG.fallback_model).trim(),
+    model: config.model ? String(config.model).trim() || null : null,
+    fallback_model: config.fallback_model ? String(config.fallback_model).trim() || null : null,
     max_tokens: Number.isInteger(Number(config.max_tokens)) ? Math.max(512, Math.min(131072, Number(config.max_tokens))) : DEFAULT_OPENWIKI_CONFIG.max_tokens,
     timeout_ms: Number.isInteger(Number(config.timeout_ms)) ? Math.max(30000, Math.min(1800000, Number(config.timeout_ms))) : DEFAULT_OPENWIKI_CONFIG.timeout_ms
   };
@@ -78,9 +81,6 @@ export function saveOpenWikiConfig(config, options = {}) {
   if (!ALLOWED_PROVIDERS.includes(validated.provider)) {
     throw new Error(`Invalid provider: ${validated.provider}. Allowed: ${ALLOWED_PROVIDERS.join(', ')}`);
   }
-  if (!validated.model) {
-    throw new Error('Model identifier cannot be empty');
-  }
 
   const dir = path.dirname(filePath);
   if (!fs.existsSync(dir)) {

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/generate.mjs

@@ -8,7 +8,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { spawn } from 'node:child_process';
 import { loadOpenWikiConfig } from './config.mjs';
-import { readProjects, resolveGenerator, resolveKey } from './refresh.mjs';
+import { buildGeneratorInvocation, ensureDefaultIgnores, readProjects, redact, resolveGenerator, resolveKey } from './refresh.mjs';
 import { execOnMeshNode, getMeshNodes } from './mesh-vaults.mjs';
 
 export async function generateWiki({
@@ -92,31 +92,30 @@ export async function generateWiki({
   }
 
   const gen = resolveGenerator();
-  const generatorArgs = [...gen.args];
-
-  // Pass --init or update mode
-  const isInit = !fs.existsSync(outPath) || force;
-  if (isInit) {
-    generatorArgs.push('--init');
+  if (!gen.command) {
+    throw new Error('No openwiki generator available (missing bundled generator, local .bin, PATH entry, or npx)');
   }
 
-  const env = {
-    ...process.env,
-    OPENROUTER_API_KEY: config.provider === 'openrouter' ? key : process.env.OPENROUTER_API_KEY,
-    GOOGLE_API_KEY: config.provider === 'google' ? key : process.env.GOOGLE_API_KEY,
-    OPENWIKI_MODEL: config.model,
-    OPENWIKI_TIMEOUT: String(config.timeout_ms || 300000),
-  };
+  const mode = (!fs.existsSync(outPath) || force) ? 'init' : 'update';
+  const timeoutMs = config.timeout_ms || 300000;
+  const { cmdArgs, childEnv } = buildGeneratorInvocation({
+    baseArgs: gen.args,
+    mode,
+    config,
+    key,
+  });
+  ensureDefaultIgnores(resolvedTarget);
 
   const startedAt = Date.now();
   if (typeof onProgress === 'function') {
-    onProgress(`Starting generator for ${repoName} (${gen.via})...`);
+    onProgress(`Starting generator for ${repoName} (${gen.via}, ${mode})...`);
   }
 
   return new Promise((resolve, reject) => {
-    const child = spawn(gen.command, generatorArgs, {
+    const child = spawn(gen.command, cmdArgs, {
       cwd: resolvedTarget,
-      env,
+      env: childEnv,
+      timeout: timeoutMs,
       stdio: ['ignore', 'pipe', 'pipe'],
     });
 
@@ -127,7 +126,7 @@ export async function generateWiki({
       const str = chunk.toString();
       stdout += str;
       if (typeof onProgress === 'function') {
-        onProgress(str.trim());
+        onProgress(redact(str.trim(), key));
       }
     });
 
@@ -135,7 +134,7 @@ export async function generateWiki({
       stderr += chunk.toString();
     });
 
-    child.on('close', (code) => {
+    child.on('close', (code, signal) => {
       const durationMs = Date.now() - startedAt;
       if (code === 0) {
         resolve({
@@ -144,17 +143,19 @@ export async function generateWiki({
           targetPath: resolvedTarget,
           outputDir: outPath,
           generatorVia: gen.via,
+          mode,
           durationMs,
           credential: from,
-          stdout: stdout.trim(),
+          stdout: redact(stdout.trim(), key),
         });
       } else {
-        reject(new Error(`Generator exited with code ${code}: ${stderr || stdout || 'Unknown error'}`));
+        const detail = redact(stderr || stdout || (signal ? `terminated by ${signal}` : 'Unknown error'), key);
+        reject(new Error(`Generator exited with code ${code ?? signal ?? 1}: ${detail}`));
       }
     });
 
     child.on('error', (err) => {
-      reject(new Error(`Failed to spawn generator: ${err.message}`));
+      reject(new Error(`Failed to spawn generator: ${redact(err.message, key)}`));
     });
   });
 }

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/graph.mjs

@@ -5,8 +5,8 @@
  * across local workspace, all vaults, or remote mesh nodes. Formats to JSON, Mermaid, or DOT.
  */
 import fs from 'node:fs';
-import path from 'node:path';
 import { discoverAllHubs, execOnMeshNode } from './mesh-vaults.mjs';
+import { extractMarkdownLinks, indexHubsForResolution, resolveLinkTarget } from './markdown-links.mjs';
 
 /**
  * Builds knowledge graph from markdown hubs.
@@ -16,6 +16,7 @@ export async function buildKnowledgeGraph({
   node = null,
   format = 'json',
   cwd = process.cwd(),
+  nested = true,
 } = {}) {
   // If remote mesh node specified, dispatch via mesh
   if (node && node !== 'local' && node !== 'self') {
@@ -33,10 +34,11 @@ export async function buildKnowledgeGraph({
     }
   }
 
-  const hubsResult = await discoverAllHubs({ allVaults, cwd });
+  const hubsResult = await discoverAllHubs({ allVaults, cwd, nested });
   const nodes = [];
   const edges = [];
-  const nodeMap = new Map();
+  const nodeById = new Map();
+  const nodeByPath = new Map();
 
   // Create nodes
   for (const hub of hubsResult.hubs) {
@@ -53,13 +55,12 @@ export async function buildKnowledgeGraph({
       outDegree: 0,
     };
     nodes.push(nodeObj);
-    nodeMap.set(nodeId, nodeObj);
-    // Also index by fileName and title for fuzzy link target resolution
-    nodeMap.set(hub.fileName.toLowerCase(), nodeObj);
-    nodeMap.set(hub.fileName.replace(/\.md$/, '').toLowerCase(), nodeObj);
-    nodeMap.set(hub.title.toLowerCase(), nodeObj);
+    nodeById.set(nodeId, nodeObj);
+    nodeByPath.set(hub.path, nodeObj);
   }
 
+  const index = indexHubsForResolution(hubsResult.hubs);
+
   // Extract edges
   for (const hub of hubsResult.hubs) {
     let content;
@@ -70,27 +71,18 @@ export async function buildKnowledgeGraph({
     }
 
     const sourceId = `${hub.repoName}:${hub.fileName.replace(/\.md$/, '')}`;
-    const sourceNode = nodeMap.get(sourceId);
-
-    // 1. Match Markdown links: [Text](target.md) or [Text](./target.md)
-    const mdLinkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
-    let match;
-    while ((match = mdLinkRegex.exec(content)) !== null) {
-      const anchorText = match[1].trim();
-      const targetUrl = match[2].trim().split('#')[0]; // strip hash anchor
-      if (!targetUrl || targetUrl.startsWith('http://') || targetUrl.startsWith('https://')) continue;
-
-      const targetFileName = path.basename(targetUrl);
-      if (!targetFileName.endsWith('.md')) continue;
-
-      const targetNode = nodeMap.get(targetFileName.toLowerCase()) ||
-                         nodeMap.get(targetFileName.replace(/\.md$/, '').toLowerCase());
+    const sourceNode = nodeById.get(sourceId);
+    const { markdownLinks, wikilinks } = extractMarkdownLinks(content);
 
+    for (const link of markdownLinks) {
+      if (!link.target.endsWith('.md')) continue;
+      const resolved = resolveLinkTarget(hub, link.target, index, { isWikilink: false });
+      const targetNode = resolved.hub ? nodeByPath.get(resolved.hub.path) : null;
       if (targetNode && targetNode.id !== sourceId) {
         edges.push({
           from: sourceId,
           to: targetNode.id,
-          label: anchorText.length > 30 ? anchorText.slice(0, 27) + '...' : anchorText,
+          label: link.text.length > 30 ? link.text.slice(0, 27) + '...' : link.text,
           type: 'markdown-link',
         });
         if (sourceNode) sourceNode.outDegree++;
@@ -98,19 +90,14 @@ export async function buildKnowledgeGraph({
       }
     }
 
-    // 2. Match Wikilinks: [[Target Page]] or [[Target Page|Anchor]]
-    const wikiLinkRegex = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
-    while ((match = wikiLinkRegex.exec(content)) !== null) {
-      const targetName = match[1].trim();
-      const anchor = (match[2] || targetName).trim();
-      const targetNode = nodeMap.get(targetName.toLowerCase()) ||
-                         nodeMap.get(`${targetName}.md`.toLowerCase());
-
+    for (const link of wikilinks) {
+      const resolved = resolveLinkTarget(hub, link.target, index, { isWikilink: true });
+      const targetNode = resolved.hub ? nodeByPath.get(resolved.hub.path) : null;
       if (targetNode && targetNode.id !== sourceId) {
         edges.push({
           from: sourceId,
           to: targetNode.id,
-          label: anchor.length > 30 ? anchor.slice(0, 27) + '...' : anchor,
+          label: link.anchor.length > 30 ? link.anchor.slice(0, 27) + '...' : link.anchor,
           type: 'wikilink',
         });
         if (sourceNode) sourceNode.outDegree++;

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/lint.mjs

@@ -8,24 +8,34 @@
  * - Orphaned hubs with zero inbound or outbound links
  */
 import fs from 'node:fs';
-import path from 'node:path';
-import { discoverAllHubs } from './mesh-vaults.mjs';
+import { discoverAllHubs, execOnMeshNode } from './mesh-vaults.mjs';
+import { extractMarkdownLinks, indexHubsForResolution, resolveLinkTarget } from './markdown-links.mjs';
 
 export async function lintWikiHubs({
   allVaults = false,
   node = null,
   cwd = process.cwd(),
+  nested = true,
 } = {}) {
-  const hubsResult = await discoverAllHubs({ allVaults, node, cwd });
-  const allHubs = hubsResult.hubs;
-
-  const existingFileSet = new Set(allHubs.map((h) => path.resolve(h.path)));
-  const baseNameMap = new Map();
-  for (const h of allHubs) {
-    baseNameMap.set(h.fileName.toLowerCase(), h.path);
-    baseNameMap.set(h.fileName.replace(/\.md$/, '').toLowerCase(), h.path);
+  // Delegate to remote mesh node when requested, matching search and graph.
+  if (node && node !== 'local' && node !== 'self') {
+    const meshRes = execOnMeshNode(node, [
+      'lint', ...(allVaults ? ['--all-vaults'] : []), '--json',
+    ]);
+    if (meshRes.ok) {
+      try {
+        return JSON.parse(meshRes.stdout);
+      } catch {
+        throw new Error(`Invalid JSON from mesh node ${node}: ${meshRes.stdout}`);
+      }
+    }
+    throw new Error(`Mesh lint failed on node ${node}: ${meshRes.stderr || meshRes.error}`);
   }
 
+  const hubsResult = await discoverAllHubs({ allVaults, cwd, nested });
+  const allHubs = hubsResult.hubs;
+  const index = indexHubsForResolution(allHubs);
+
   const issues = [];
   let totalLinks = 0;
   let brokenLinksCount = 0;
@@ -63,61 +73,44 @@ export async function lintWikiHubs({
       });
     }
 
-    const hubDir = path.dirname(hub.path);
-
-    // 1. Check relative markdown links: [Text](target.md)
-    const mdLinkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
-    let match;
-    while ((match = mdLinkRegex.exec(content)) !== null) {
-      const linkText = match[1];
-      const rawUrl = match[2].trim().split('#')[0]; // exclude anchor
-      if (!rawUrl || rawUrl.startsWith('http://') || rawUrl.startsWith('https://') || rawUrl.startsWith('mailto:')) {
-        continue;
-      }
+    const { markdownLinks, wikilinks } = extractMarkdownLinks(content);
 
+    for (const link of markdownLinks) {
       totalLinks++;
       outboundCount.set(hub.path, (outboundCount.get(hub.path) || 0) + 1);
 
-      // Resolve relative to hub file directory
-      const resolvedTarget = path.resolve(hubDir, rawUrl);
-      if (fs.existsSync(resolvedTarget)) {
-        inboundCount.set(resolvedTarget, (inboundCount.get(resolvedTarget) || 0) + 1);
-      } else {
-        // Fallback: check if matches another hub by basename
-        const targetBase = path.basename(rawUrl).toLowerCase();
-        if (baseNameMap.has(targetBase)) {
-          const alternate = baseNameMap.get(targetBase);
-          inboundCount.set(alternate, (inboundCount.get(alternate) || 0) + 1);
-        } else {
-          brokenLinksCount++;
-          issues.push({
-            type: 'broken_link',
-            file: hub.path,
-            linkText,
-            target: rawUrl,
-            message: `Target file not found: ${rawUrl}`,
-          });
+      const resolved = resolveLinkTarget(hub, link.target, index, { isWikilink: false });
+      if (resolved.existsOnDisk) {
+        const targetPath = resolved.hub?.path || resolved.resolvedPath;
+        if (targetPath) {
+          inboundCount.set(targetPath, (inboundCount.get(targetPath) || 0) + 1);
         }
+      } else {
+        brokenLinksCount++;
+        issues.push({
+          type: 'broken_link',
+          file: hub.path,
+          linkText: link.text,
+          target: link.target,
+          message: `Target file not found: ${link.target}`,
+        });
       }
     }
 
-    // 2. Check wikilinks: [[Target]]
-    const wikiRegex = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
-    while ((match = wikiRegex.exec(content)) !== null) {
-      const targetName = match[1].trim().toLowerCase();
+    for (const link of wikilinks) {
       totalLinks++;
       outboundCount.set(hub.path, (outboundCount.get(hub.path) || 0) + 1);
 
-      if (baseNameMap.has(targetName) || baseNameMap.has(`${targetName}.md`)) {
-        const targetPath = baseNameMap.get(targetName) || baseNameMap.get(`${targetName}.md`);
-        inboundCount.set(targetPath, (inboundCount.get(targetPath) || 0) + 1);
+      const resolved = resolveLinkTarget(hub, link.target, index, { isWikilink: true });
+      if (resolved.existsOnDisk && resolved.hub) {
+        inboundCount.set(resolved.hub.path, (inboundCount.get(resolved.hub.path) || 0) + 1);
       } else {
         brokenLinksCount++;
         issues.push({
           type: 'broken_wikilink',
           file: hub.path,
-          target: match[1].trim(),
-          message: `Wikilink target not found: [[${match[1].trim()}]]`,
+          target: link.target,
+          message: `Wikilink target not found: [[${link.target}]]`,
         });
       }
     }

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/markdown-links.mjs

@@ -0,0 +1,112 @@
+/**
+ * Shared Markdown link extraction and hub target resolution for OpenWiki.
+ *
+ * Strips fenced code blocks and inline code spans so code examples never create
+ * false-positive broken links or graph edges, and resolves targets with
+ * same-file-directory and same-repository priority before global fallback.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Remove fenced code blocks (``` / ~~~) and inline code spans (`...`).
+ */
+export function stripMarkdownCode(content) {
+  const withoutFences = String(content || '')
+    .replace(/^([ \t]*)(`{3,}|~{3,})[^\n]*\n[\s\S]*?^\1\2[ \t]*$/gm, '');
+  return withoutFences.replace(/`[^`\n]+`/g, '');
+}
+
+/**
+ * Extract Markdown links and Wikilinks from prose (excluding code blocks/spans).
+ */
+export function extractMarkdownLinks(content) {
+  const clean = stripMarkdownCode(content);
+  const markdownLinks = [];
+  const wikilinks = [];
+
+  const mdLinkRegex = /(?<!!)\[([^\]]+)\]\(([^)]+)\)/g;
+  let match;
+  while ((match = mdLinkRegex.exec(clean)) !== null) {
+    const text = match[1].trim();
+    const rawUrl = match[2].trim().split('#')[0].trim();
+    if (!rawUrl || /^(https?:|mailto:|tel:|data:|#)/i.test(rawUrl)) continue;
+    markdownLinks.push({ text, target: rawUrl });
+  }
+
+  const wikiRegex = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
+  while ((match = wikiRegex.exec(clean)) !== null) {
+    const target = match[1].trim();
+    const anchor = (match[2] || target).trim();
+    if (target) wikilinks.push({ target, anchor });
+  }
+
+  return { markdownLinks, wikilinks };
+}
+
+/**
+ * Build a multi-level lookup index for hubs so same-repo targets beat
+ * cross-repo basename collisions.
+ */
+export function indexHubsForResolution(hubs) {
+  const byPath = new Map();
+  const byRepoKey = new Map();
+  const byGlobalKey = new Map();
+
+  for (const hub of hubs) {
+    const resolved = path.resolve(hub.path);
+    byPath.set(resolved, hub);
+
+    const relLower = hub.fileName.toLowerCase();
+    const relNoExt = relLower.replace(/\.md$/i, '');
+    const baseLower = path.basename(hub.fileName).toLowerCase();
+    const baseNoExt = baseLower.replace(/\.md$/i, '');
+    const titleLower = (hub.title || '').toLowerCase();
+
+    const keys = [relLower, relNoExt, baseLower, baseNoExt, titleLower].filter(Boolean);
+    for (const k of keys) {
+      const repoKey = `${hub.repoName}:${k}`;
+      if (!byRepoKey.has(repoKey)) byRepoKey.set(repoKey, hub);
+      if (!byGlobalKey.has(k)) byGlobalKey.set(k, hub);
+    }
+  }
+
+  return { byPath, byRepoKey, byGlobalKey };
+}
+
+/**
+ * Resolve a Markdown link or wikilink target from `sourceHub` using `index`.
+ */
+export function resolveLinkTarget(sourceHub, rawTarget, index, { isWikilink = false } = {}) {
+  const sourceDir = path.dirname(sourceHub.path);
+
+  if (!isWikilink) {
+    const resolved = path.resolve(sourceDir, rawTarget);
+    if (index.byPath.has(resolved)) {
+      return { hub: index.byPath.get(resolved), existsOnDisk: true };
+    }
+    if (fs.existsSync(resolved)) {
+      return { hub: null, existsOnDisk: true, resolvedPath: resolved };
+    }
+  }
+
+  const normalized = rawTarget.replace(/^\.\//, '').toLowerCase();
+  const noExt = normalized.replace(/\.md$/i, '');
+  const base = path.basename(normalized);
+  const baseNoExt = base.replace(/\.md$/i, '');
+  const candidates = isWikilink
+    ? [normalized, `${normalized}.md`, base, `${base}.md`]
+    : [normalized, noExt, base, baseNoExt];
+
+  for (const key of candidates) {
+    const repoMatch = index.byRepoKey.get(`${sourceHub.repoName}:${key}`);
+    if (repoMatch) return { hub: repoMatch, existsOnDisk: true };
+  }
+
+  for (const key of candidates) {
+    const globalMatch = index.byGlobalKey.get(key);
+    if (globalMatch) return { hub: globalMatch, existsOnDisk: true };
+  }
+
+  return { hub: null, existsOnDisk: false };
+}

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/mesh-vaults.mjs

@@ -257,16 +257,17 @@ export async function discoverAllHubs({
       const projHubs = scanDirectoryHubs(vault.openwikiPath, {
         label: 'project-openwiki',
         repoName: vault.name,
+        nested,
       });
       allHubs.push(...projHubs);
     } else if (vault.type === 'global') {
       const gHubsDir = path.join(vault.root, 'openwiki-hubs');
       const gArticlesDir = path.join(vault.root, 'openwiki-articles');
       if (fs.existsSync(gHubsDir)) {
-        allHubs.push(...scanDirectoryHubs(gHubsDir, { label: 'global-vault-hubs', repoName: 'global-vault' }));
+        allHubs.push(...scanDirectoryHubs(gHubsDir, { label: 'global-vault-hubs', repoName: 'global-vault', nested }));
       }
       if (fs.existsSync(gArticlesDir)) {
-        allHubs.push(...scanDirectoryHubs(gArticlesDir, { label: 'global-vault-articles', repoName: 'global-vault' }));
+        allHubs.push(...scanDirectoryHubs(gArticlesDir, { label: 'global-vault-articles', repoName: 'global-vault', nested }));
       }
     }
   }