total-recall-brain 3.41.3 → 3.41.5

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/generate.mjs

@@ -0,0 +1,159 @@
+/**
+ * Targeted On-Demand Generator for OpenWiki.
+ *
+ * Generates or regenerates documentation for a specific repository or workspace
+ * using the hermetic bundled generator `vendor/openwiki.mjs`.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { loadOpenWikiConfig } from './config.mjs';
+import { readProjects, resolveGenerator, resolveKey } from './refresh.mjs';
+import { execOnMeshNode, getMeshNodes } from './mesh-vaults.mjs';
+
+export async function generateWiki({
+  repo = null,
+  targetPath = null,
+  outputDir = null,
+  focus = 'all',
+  force = false,
+  dryRun = false,
+  node = null,
+  onProgress = null,
+} = {}) {
+  // If remote mesh node specified, dispatch via mesh
+  if (node && node !== 'local' && node !== 'self') {
+    const nodes = getMeshNodes();
+    const selfNode = nodes.find((n) => n.self);
+    const isSelf = selfNode && (selfNode.ip === node || selfNode.hostname.toLowerCase() === node.toLowerCase());
+
+    if (!isSelf) {
+      const escapedRepo = repo ? `--repo="${repo}"` : '';
+      const cmd = `total-recall openwiki generate ${escapedRepo} ${force ? '--force' : ''} --json`;
+      const meshRes = execOnMeshNode(node, cmd);
+      if (meshRes.ok) {
+        try {
+          return JSON.parse(meshRes.stdout);
+        } catch {
+          return { ok: true, node, output: meshRes.stdout };
+        }
+      } else {
+        throw new Error(`Mesh generation failed on node ${node}: ${meshRes.stderr || meshRes.error}`);
+      }
+    }
+  }
+
+  // Resolve target directory
+  let resolvedTarget = process.cwd();
+  let repoName = path.basename(resolvedTarget);
+
+  if (repo) {
+    let projects = [];
+    try {
+      projects = readProjects();
+    } catch {
+      projects = [];
+    }
+    const found = projects.find((p) => p.name === repo || p.root === repo);
+    if (!found) {
+      throw new Error(`Project '${repo}' not found in registered projects.`);
+    }
+    resolvedTarget = found.root;
+    repoName = found.name;
+  } else if (targetPath) {
+    resolvedTarget = path.resolve(process.cwd(), targetPath);
+    repoName = path.basename(resolvedTarget);
+  }
+
+  if (!fs.existsSync(resolvedTarget)) {
+    throw new Error(`Target path does not exist: ${resolvedTarget}`);
+  }
+
+  const outPath = outputDir ? path.resolve(resolvedTarget, outputDir) : path.join(resolvedTarget, 'openwiki');
+
+  if (dryRun) {
+    return {
+      ok: true,
+      dryRun: true,
+      repoName,
+      targetPath: resolvedTarget,
+      outputDir: outPath,
+      focus,
+      message: `[Dry Run] Would generate documentation for ${repoName} at ${outPath}`,
+    };
+  }
+
+  const config = loadOpenWikiConfig();
+  const { key, from } = resolveKey({ provider: config.provider });
+  if (!key) {
+    const keyName = config.provider === 'google' ? 'GOOGLE_API_KEY' : 'OPENROUTER_API_KEY';
+    throw new Error(`No API credential found for provider '${config.provider}'. Save via: total-recall auth secret set ${keyName} --stdin`);
+  }
+
+  const gen = resolveGenerator();
+  const generatorArgs = [...gen.args];
+
+  // Pass --init or update mode
+  const isInit = !fs.existsSync(outPath) || force;
+  if (isInit) {
+    generatorArgs.push('--init');
+  }
+
+  const env = {
+    ...process.env,
+    OPENROUTER_API_KEY: config.provider === 'openrouter' ? key : process.env.OPENROUTER_API_KEY,
+    GOOGLE_API_KEY: config.provider === 'google' ? key : process.env.GOOGLE_API_KEY,
+    OPENWIKI_MODEL: config.model,
+    OPENWIKI_TIMEOUT: String(config.timeout_ms || 300000),
+  };
+
+  const startedAt = Date.now();
+  if (typeof onProgress === 'function') {
+    onProgress(`Starting generator for ${repoName} (${gen.via})...`);
+  }
+
+  return new Promise((resolve, reject) => {
+    const child = spawn(gen.command, generatorArgs, {
+      cwd: resolvedTarget,
+      env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout.on('data', (chunk) => {
+      const str = chunk.toString();
+      stdout += str;
+      if (typeof onProgress === 'function') {
+        onProgress(str.trim());
+      }
+    });
+
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+
+    child.on('close', (code) => {
+      const durationMs = Date.now() - startedAt;
+      if (code === 0) {
+        resolve({
+          ok: true,
+          repoName,
+          targetPath: resolvedTarget,
+          outputDir: outPath,
+          generatorVia: gen.via,
+          durationMs,
+          credential: from,
+          stdout: stdout.trim(),
+        });
+      } else {
+        reject(new Error(`Generator exited with code ${code}: ${stderr || stdout || 'Unknown error'}`));
+      }
+    });
+
+    child.on('error', (err) => {
+      reject(new Error(`Failed to spawn generator: ${err.message}`));
+    });
+  });
+}

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/graph.mjs

@@ -0,0 +1,197 @@
+/**
+ * OpenWiki Knowledge Topology & Graph Extractor.
+ *
+ * Extracts nodes, edges, cross-document links, and architectural relationships
+ * across local workspace, all vaults, or remote mesh nodes. Formats to JSON, Mermaid, or DOT.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { discoverAllHubs, execOnMeshNode } from './mesh-vaults.mjs';
+
+/**
+ * Builds knowledge graph from markdown hubs.
+ */
+export async function buildKnowledgeGraph({
+  allVaults = false,
+  node = null,
+  format = 'json',
+  cwd = process.cwd(),
+} = {}) {
+  // If remote mesh node specified, dispatch via mesh
+  if (node && node !== 'local' && node !== 'self') {
+    const meshRes = execOnMeshNode(
+      node,
+      `total-recall openwiki graph ${allVaults ? '--all-vaults' : ''} --format=${format} --json`
+    );
+    if (meshRes.ok) {
+      try {
+        return JSON.parse(meshRes.stdout);
+      } catch {
+        throw new Error(`Invalid JSON from mesh node ${node}: ${meshRes.stdout}`);
+      }
+    } else {
+      throw new Error(`Mesh graph failed on node ${node}: ${meshRes.stderr || meshRes.error}`);
+    }
+  }
+
+  const hubsResult = await discoverAllHubs({ allVaults, cwd });
+  const nodes = [];
+  const edges = [];
+  const nodeMap = new Map();
+
+  // Create nodes
+  for (const hub of hubsResult.hubs) {
+    const nodeId = `${hub.repoName}:${hub.fileName.replace(/\.md$/, '')}`;
+    const nodeObj = {
+      id: nodeId,
+      label: hub.title || hub.fileName,
+      fileName: hub.fileName,
+      repoName: hub.repoName,
+      path: hub.path,
+      lines: hub.lines,
+      size_bytes: hub.size_bytes,
+      inDegree: 0,
+      outDegree: 0,
+    };
+    nodes.push(nodeObj);
+    nodeMap.set(nodeId, nodeObj);
+    // Also index by fileName and title for fuzzy link target resolution
+    nodeMap.set(hub.fileName.toLowerCase(), nodeObj);
+    nodeMap.set(hub.fileName.replace(/\.md$/, '').toLowerCase(), nodeObj);
+    nodeMap.set(hub.title.toLowerCase(), nodeObj);
+  }
+
+  // Extract edges
+  for (const hub of hubsResult.hubs) {
+    let content;
+    try {
+      content = fs.readFileSync(hub.path, 'utf8');
+    } catch {
+      continue;
+    }
+
+    const sourceId = `${hub.repoName}:${hub.fileName.replace(/\.md$/, '')}`;
+    const sourceNode = nodeMap.get(sourceId);
+
+    // 1. Match Markdown links: [Text](target.md) or [Text](./target.md)
+    const mdLinkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+    let match;
+    while ((match = mdLinkRegex.exec(content)) !== null) {
+      const anchorText = match[1].trim();
+      const targetUrl = match[2].trim().split('#')[0]; // strip hash anchor
+      if (!targetUrl || targetUrl.startsWith('http://') || targetUrl.startsWith('https://')) continue;
+
+      const targetFileName = path.basename(targetUrl);
+      if (!targetFileName.endsWith('.md')) continue;
+
+      const targetNode = nodeMap.get(targetFileName.toLowerCase()) ||
+                         nodeMap.get(targetFileName.replace(/\.md$/, '').toLowerCase());
+
+      if (targetNode && targetNode.id !== sourceId) {
+        edges.push({
+          from: sourceId,
+          to: targetNode.id,
+          label: anchorText.length > 30 ? anchorText.slice(0, 27) + '...' : anchorText,
+          type: 'markdown-link',
+        });
+        if (sourceNode) sourceNode.outDegree++;
+        targetNode.inDegree++;
+      }
+    }
+
+    // 2. Match Wikilinks: [[Target Page]] or [[Target Page|Anchor]]
+    const wikiLinkRegex = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
+    while ((match = wikiLinkRegex.exec(content)) !== null) {
+      const targetName = match[1].trim();
+      const anchor = (match[2] || targetName).trim();
+      const targetNode = nodeMap.get(targetName.toLowerCase()) ||
+                         nodeMap.get(`${targetName}.md`.toLowerCase());
+
+      if (targetNode && targetNode.id !== sourceId) {
+        edges.push({
+          from: sourceId,
+          to: targetNode.id,
+          label: anchor.length > 30 ? anchor.slice(0, 27) + '...' : anchor,
+          type: 'wikilink',
+        });
+        if (sourceNode) sourceNode.outDegree++;
+        targetNode.inDegree++;
+      }
+    }
+  }
+
+  // Deduplicate edges
+  const edgeKeySet = new Set();
+  const dedupedEdges = [];
+  for (const edge of edges) {
+    const key = `${edge.from}->${edge.to}`;
+    if (!edgeKeySet.has(key)) {
+      edgeKeySet.add(key);
+      dedupedEdges.push(edge);
+    }
+  }
+
+  // Identify orphans (disconnected nodes)
+  const orphans = nodes.filter((n) => n.inDegree === 0 && n.outDegree === 0).map((n) => n.id);
+
+  const payload = {
+    ok: true,
+    scope: hubsResult.scope,
+    counts: {
+      nodes: nodes.length,
+      edges: dedupedEdges.length,
+      orphans: orphans.length,
+    },
+    nodes,
+    edges: dedupedEdges,
+    orphans,
+  };
+
+  if (format === 'mermaid') {
+    payload.mermaid = generateMermaidDiagram(nodes, dedupedEdges);
+  } else if (format === 'dot') {
+    payload.dot = generateDotDiagram(nodes, dedupedEdges);
+  }
+
+  return payload;
+}
+
+function sanitizeId(id) {
+  return id.replace(/[^a-zA-Z0-9_]/g, '_');
+}
+
+export function generateMermaidDiagram(nodes, edges) {
+  const lines = ['flowchart TD'];
+  for (const node of nodes) {
+    const cleanId = sanitizeId(node.id);
+    const cleanLabel = (node.label || node.fileName).replace(/["[\]()]/g, '');
+    lines.push(`  ${cleanId}["${cleanLabel}"]`);
+  }
+  for (const edge of edges) {
+    const fromId = sanitizeId(edge.from);
+    const toId = sanitizeId(edge.to);
+    if (edge.label && edge.label !== edge.to) {
+      const cleanEdgeLabel = edge.label.replace(/["[\]()]/g, '');
+      lines.push(`  ${fromId} -->|"${cleanEdgeLabel}"| ${toId}`);
+    } else {
+      lines.push(`  ${fromId} --> ${toId}`);
+    }
+  }
+  return lines.join('\n');
+}
+
+export function generateDotDiagram(nodes, edges) {
+  const lines = ['digraph OpenWikiGraph {', '  rankdir=LR;', '  node [shape=box, style=rounded, fontname="Helvetica"];'];
+  for (const node of nodes) {
+    const cleanId = sanitizeId(node.id);
+    const cleanLabel = (node.label || node.fileName).replace(/"/g, '\\"');
+    lines.push(`  ${cleanId} [label="${cleanLabel}"];`);
+  }
+  for (const edge of edges) {
+    const fromId = sanitizeId(edge.from);
+    const toId = sanitizeId(edge.to);
+    lines.push(`  ${fromId} -> ${toId};`);
+  }
+  lines.push('}');
+  return lines.join('\n');
+}

package/scaffold/.agent/skills/total-recall/plugins/openwiki/src/lint.mjs

@@ -0,0 +1,153 @@
+/**
+ * OpenWiki Documentation & Link Hygiene Linter.
+ *
+ * Scans markdown hubs across workspace or all vaults, detecting:
+ * - Broken relative links and dead file targets
+ * - Missing or broken wikilinks
+ * - Missing title headers or empty pages
+ * - Orphaned hubs with zero inbound or outbound links
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { discoverAllHubs } from './mesh-vaults.mjs';
+
+export async function lintWikiHubs({
+  allVaults = false,
+  node = null,
+  cwd = process.cwd(),
+} = {}) {
+  const hubsResult = await discoverAllHubs({ allVaults, node, cwd });
+  const allHubs = hubsResult.hubs;
+
+  const existingFileSet = new Set(allHubs.map((h) => path.resolve(h.path)));
+  const baseNameMap = new Map();
+  for (const h of allHubs) {
+    baseNameMap.set(h.fileName.toLowerCase(), h.path);
+    baseNameMap.set(h.fileName.replace(/\.md$/, '').toLowerCase(), h.path);
+  }
+
+  const issues = [];
+  let totalLinks = 0;
+  let brokenLinksCount = 0;
+  const inboundCount = new Map(allHubs.map((h) => [h.path, 0]));
+  const outboundCount = new Map(allHubs.map((h) => [h.path, 0]));
+
+  for (const hub of allHubs) {
+    let content;
+    try {
+      content = fs.readFileSync(hub.path, 'utf8');
+    } catch {
+      issues.push({
+        type: 'unreadable_file',
+        file: hub.path,
+        message: 'Could not read markdown file',
+      });
+      continue;
+    }
+
+    if (!content.trim()) {
+      issues.push({
+        type: 'empty_hub',
+        file: hub.path,
+        message: 'Hub markdown file is completely empty',
+      });
+      continue;
+    }
+
+    // Check title header
+    if (!content.match(/^#\s+.+/m)) {
+      issues.push({
+        type: 'missing_title',
+        file: hub.path,
+        message: 'Missing level 1 header (# Title)',
+      });
+    }
+
+    const hubDir = path.dirname(hub.path);
+
+    // 1. Check relative markdown links: [Text](target.md)
+    const mdLinkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+    let match;
+    while ((match = mdLinkRegex.exec(content)) !== null) {
+      const linkText = match[1];
+      const rawUrl = match[2].trim().split('#')[0]; // exclude anchor
+      if (!rawUrl || rawUrl.startsWith('http://') || rawUrl.startsWith('https://') || rawUrl.startsWith('mailto:')) {
+        continue;
+      }
+
+      totalLinks++;
+      outboundCount.set(hub.path, (outboundCount.get(hub.path) || 0) + 1);
+
+      // Resolve relative to hub file directory
+      const resolvedTarget = path.resolve(hubDir, rawUrl);
+      if (fs.existsSync(resolvedTarget)) {
+        inboundCount.set(resolvedTarget, (inboundCount.get(resolvedTarget) || 0) + 1);
+      } else {
+        // Fallback: check if matches another hub by basename
+        const targetBase = path.basename(rawUrl).toLowerCase();
+        if (baseNameMap.has(targetBase)) {
+          const alternate = baseNameMap.get(targetBase);
+          inboundCount.set(alternate, (inboundCount.get(alternate) || 0) + 1);
+        } else {
+          brokenLinksCount++;
+          issues.push({
+            type: 'broken_link',
+            file: hub.path,
+            linkText,
+            target: rawUrl,
+            message: `Target file not found: ${rawUrl}`,
+          });
+        }
+      }
+    }
+
+    // 2. Check wikilinks: [[Target]]
+    const wikiRegex = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+    while ((match = wikiRegex.exec(content)) !== null) {
+      const targetName = match[1].trim().toLowerCase();
+      totalLinks++;
+      outboundCount.set(hub.path, (outboundCount.get(hub.path) || 0) + 1);
+
+      if (baseNameMap.has(targetName) || baseNameMap.has(`${targetName}.md`)) {
+        const targetPath = baseNameMap.get(targetName) || baseNameMap.get(`${targetName}.md`);
+        inboundCount.set(targetPath, (inboundCount.get(targetPath) || 0) + 1);
+      } else {
+        brokenLinksCount++;
+        issues.push({
+          type: 'broken_wikilink',
+          file: hub.path,
+          target: match[1].trim(),
+          message: `Wikilink target not found: [[${match[1].trim()}]]`,
+        });
+      }
+    }
+  }
+
+  // Detect orphaned pages
+  const orphans = [];
+  for (const hub of allHubs) {
+    const inbound = inboundCount.get(hub.path) || 0;
+    const outbound = outboundCount.get(hub.path) || 0;
+    if (inbound === 0 && outbound === 0 && allHubs.length > 1) {
+      orphans.push({
+        file: hub.path,
+        title: hub.title,
+      });
+    }
+  }
+
+  const score = totalLinks === 0
+    ? (issues.length === 0 ? 100 : Math.max(50, 100 - issues.length * 10))
+    : Math.max(0, Math.round(((totalLinks - brokenLinksCount) / totalLinks) * 100));
+
+  return {
+    ok: issues.length === 0,
+    healthScore: score,
+    totalHubs: allHubs.length,
+    totalLinks,
+    brokenLinksCount,
+    orphanCount: orphans.length,
+    issues,
+    orphans,
+  };
+}