total-recall-brain 3.5.0 → 3.6.0

package/frontend/src/pages/GraphPage.tsx

@@ -0,0 +1,62 @@
+import { useState, useEffect } from 'react'
+import { listMemory, listResearch, fetchChatThreads } from '../api'
+import type { MemoryNode, ResearchItem } from '../types'
+import type { ChatThread } from '../api'
+import Graph3D from '../components/Graph3D'
+
+export default function GraphPage({ activeBrainId }: { activeBrainId?: string }) {
+  const [memoryNodes, setMemoryNodes] = useState<MemoryNode[]>([])
+  const [researchItems, setResearchItems] = useState<ResearchItem[]>([])
+  const [threads, setThreads] = useState<ChatThread[]>([])
+  const [loading, setLoading] = useState(true)
+  const [selectedGroundingNodes, setSelectedGroundingNodes] = useState<string[]>([])
+
+  useEffect(() => {
+    async function loadData() {
+      setLoading(true)
+      try {
+        const [nodes, researchRes, threadsRes] = await Promise.all([
+          listMemory(activeBrainId),
+          listResearch(activeBrainId).catch(() => ({ items: [] })),
+          fetchChatThreads().catch(() => [])
+        ])
+        setMemoryNodes(nodes)
+        setResearchItems(researchRes.items || [])
+        setThreads(threadsRes)
+      } catch (err) {
+        console.error('Failed to load graph data', err)
+      } finally {
+        setLoading(false)
+      }
+    }
+    loadData()
+  }, [activeBrainId])
+
+  return (
+    <div className="page" style={{ padding: 0, height: 'calc(100vh - 1px)', display: 'flex', flexDirection: 'column', overflow: 'hidden' }}>
+      <div style={{ padding: '16px 24px', background: 'rgba(18, 18, 26, 0.45)', backdropFilter: 'blur(16px)', borderBottom: '1px solid var(--border)', flexShrink: 0 }}>
+        <h1 style={{ fontSize: '18px', fontWeight: 700 }}>Sovereign Memory Network</h1>
+        <p style={{ fontSize: '12px', color: 'var(--text-secondary)', marginTop: '2px' }}>Interactive 3D constellation layout of your knowledge vault, sessions, and active thoughts.</p>
+      </div>
+
+      <div style={{ flex: 1, position: 'relative', overflow: 'hidden' }}>
+        {loading ? (
+          <div className="skeleton" style={{ width: '100%', height: '100%' }} />
+        ) : (
+          <Graph3D
+            threads={threads}
+            memoryNodes={memoryNodes}
+            researchItems={researchItems}
+            onOpenThread={() => {}}
+            onGroundMemoryNode={(slug) => {
+              setSelectedGroundingNodes(prev =>
+                prev.includes(slug) ? prev.filter(s => s !== slug) : [...prev, slug]
+              )
+            }}
+            selectedGroundingNodes={selectedGroundingNodes}
+          />
+        )}
+      </div>
+    </div>
+  )
+}

package/frontend/src/pages/HealthPage.tsx

@@ -50,8 +50,13 @@ export default function HealthPage() {
       }
     }
     checkVersion()
+    const updateInterval = setInterval(checkVersion, 3600000)
 
-    return () => { active = false; clearInterval(interval) }
+    return () => { 
+      active = false; 
+      clearInterval(interval); 
+      clearInterval(updateInterval); 
+    }
   }, [])
 
   const handleStartUpdate = async () => {

package/frontend/src/pages/HelpPage.tsx

@@ -0,0 +1,141 @@
+import { useState, useEffect } from 'react'
+import { fetchHelpTopics, fetchHelpContent } from '../api'
+import type { HelpTopic } from '../api'
+
+function renderMarkdown(md: string) {
+  if (!md) return ''
+  let html = md
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+
+  // Headings
+  html = html.replace(/^### (.*$)/gim, '<h3>$1</h3>')
+  html = html.replace(/^## (.*$)/gim, '<h2>$1</h2>')
+  html = html.replace(/^# (.*$)/gim, '<h1>$1</h1>')
+
+  // Bold & Italic
+  html = html.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
+  html = html.replace(/\*(.*?)\*/g, '<em>$1</em>')
+
+  // Code blocks
+  html = html.replace(
+    /```([\s\S]*?)```/g,
+    '<pre style="background:rgba(0,0,0,0.3);padding:12px;border-radius:6px;border:1px solid rgba(255,255,255,0.06);font-family:monospace;overflow-x:auto;margin:12px 0;"><code>$1</code></pre>'
+  )
+  html = html.replace(
+    /`(.*?)`/g,
+    '<code style="background:rgba(255,255,255,0.06);padding:2px 6px;border-radius:4px;font-family:monospace;">$1</code>'
+  )
+
+  // Blockquotes
+  html = html.replace(
+    /^> (.*$)/gim,
+    '<blockquote style="border-left:4px solid var(--purple-primary);padding-left:14px;color:var(--text-secondary);margin:16px 0;font-style:italic;">$1</blockquote>'
+  )
+
+  // Lists
+  html = html.replace(/^- (.*$)/gim, '<li style="margin-left:20px;list-style-type:disc;">$1</li>')
+
+  // Linebreaks
+  html = html.replace(/\n/g, '<br />')
+
+  return <div dangerouslySetInnerHTML={{ __html: html }} className="markdown-preview-block" />
+}
+
+export default function HelpPage() {
+  const [topics, setTopics] = useState<HelpTopic[]>([])
+  const [selectedTopic, setSelectedTopic] = useState<string | null>(null)
+  const [content, setContent] = useState<string>('')
+  const [loading, setLoading] = useState<boolean>(false)
+  const [error, setError] = useState<string>('')
+
+  useEffect(() => {
+    async function loadTopics() {
+      try {
+        const data = await fetchHelpTopics()
+        setTopics(data.topics)
+        if (data.topics.length > 0) {
+          setSelectedTopic(data.topics[0].id)
+        }
+      } catch (err: unknown) {
+        setError((err as Error).message || 'Failed to load help topics')
+      }
+    }
+    loadTopics()
+  }, [])
+
+  useEffect(() => {
+    if (!selectedTopic) return
+    async function loadContent() {
+      setLoading(true)
+      try {
+        const data = await fetchHelpContent(selectedTopic!)
+        setContent(data.content)
+      } catch (err: unknown) {
+        setError((err as Error).message || 'Failed to load help details')
+      } finally {
+        setLoading(false)
+      }
+    }
+    loadContent()
+  }, [selectedTopic])
+
+  return (
+    <div className="page" style={{ padding: '24px', maxWidth: '1100px', margin: '0 auto' }}>
+      <div className="page-header" style={{ marginBottom: '24px' }}>
+        <h1>Sovereign Help System</h1>
+        <p>Offline interactive documentation, CLI command references, and SSSS VFS specifications.</p>
+      </div>
+
+      {error && <div className="badge badge-error" style={{ marginBottom: '16px' }}>⚠️ {error}</div>}
+
+      <div style={{ display: 'grid', gridTemplateColumns: '260px 1fr', gap: '30px', minHeight: '500px' }}>
+        {/* Sidebar */}
+        <div style={{ display: 'flex', flexDirection: 'column', gap: '10px' }}>
+          {topics.map((t) => (
+            <button
+              key={t.id}
+              onClick={() => setSelectedTopic(t.id)}
+              style={{
+                textAlign: 'left',
+                padding: '12px 16px',
+                borderRadius: '8px',
+                background: selectedTopic === t.id ? 'var(--purple-primary)' : 'rgba(255, 255, 255, 0.03)',
+                color: '#fff',
+                border: selectedTopic === t.id ? '1px solid var(--purple-border)' : '1px solid transparent',
+                cursor: 'pointer',
+                transition: 'all 0.2s',
+              }}
+            >
+              <div style={{ fontWeight: 'bold', fontSize: '14px' }}>{t.title}</div>
+              <div style={{ fontSize: '11px', color: selectedTopic === t.id ? 'rgba(255, 255, 255, 0.8)' : 'var(--text-secondary)', marginTop: '4px' }}>
+                {t.description}
+              </div>
+            </button>
+          ))}
+        </div>
+
+        {/* Content Pane */}
+        <div
+          style={{
+            background: 'var(--surface-bg)',
+            border: '1px solid var(--border)',
+            borderRadius: '12px',
+            padding: '30px',
+            overflowY: 'auto',
+            maxHeight: '75vh',
+          }}
+        >
+          {loading ? (
+            <div className="skeleton" style={{ height: '400px' }} />
+          ) : (
+            <div className="markdown-body" style={{ color: 'var(--text-primary)', lineHeight: '1.7', fontSize: '14px' }}>
+              {renderMarkdown(content)}
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "Sovereign AI system — database-free, SSSS-powered personal intelligence engine deployable to any POSIX host.",
   "type": "module",
   "bin": {
@@ -65,7 +65,8 @@
     "jsonwebtoken": "^9.0.3",
     "natural": "^8.1.1",
     "yaml": "^2.9.0",
-    "zod": "^4.4.3"
+    "zod": "^4.4.3",
+    "ws": "^8.18.0"
   },
   "devDependencies": {
     "skills": "^1.5.7",

package/src/cli/collab.mjs

@@ -0,0 +1,49 @@
+/**
+ * total-recall collab
+ *
+ * Start the collaboration backend server and frontend development portal.
+ *
+ * Usage:
+ *   npx total-recall collab
+ */
+
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const COLLAB_DIR = path.join(__dirname, '..', '..', 'collab');
+
+export default function collab(args) {
+  if (args.includes('--help') || args.includes('-h')) {
+    console.log(`
+  total-recall collab — Start collaboration workspace
+
+  Usage: total-recall collab [options]
+
+  Options:
+    --help           Show this help
+
+  Starts the Express/WebSockets backend server and Vite frontend portal.
+`);
+    return;
+  }
+
+  console.log(`  ⚡ Starting Total Recall collaboration workspace...`);
+  console.log(`  Press Ctrl+C to stop.\n`);
+
+  const child = spawn('npm', ['run', 'dev'], {
+    cwd: COLLAB_DIR,
+    stdio: 'inherit',
+    shell: true,
+  });
+
+  process.on('SIGINT', () => {
+    child.kill('SIGINT');
+    process.exit(0);
+  });
+
+  child.on('exit', (code) => {
+    process.exit(code ?? 0);
+  });
+}

package/src/cli/help.mjs

@@ -148,6 +148,6 @@ function printGeneralHelp() {
   \x1b[1mAll Commands Available:\x1b[0m
     \x1b[33minit\x1b[0m, \x1b[33msetup\x1b[0m, \x1b[33mstart\x1b[0m, \x1b[33mdeploy\x1b[0m, \x1b[33mdoctor\x1b[0m, \x1b[33mdream\x1b[0m, \x1b[33mlint\x1b[0m, \x1b[33mdaemon\x1b[0m, \x1b[33mbackup\x1b[0m, \x1b[33mrestore\x1b[0m, \x1b[33msync\x1b[0m, \x1b[33mstatus\x1b[0m,
     \x1b[33mgenerate-pat\x1b[0m, \x1b[33mhash-password\x1b[0m, \x1b[33mreset-password\x1b[0m, \x1b[33mcompile\x1b[0m, \x1b[33mrebuild\x1b[0m, \x1b[33msnapshot\x1b[0m, \x1b[33mmigrate\x1b[0m,
-    \x1b[33mimport\x1b[0m, \x1b[33mingest\x1b[0m, \x1b[33mconnect\x1b[0m, \x1b[33mrelay\x1b[0m, \x1b[33muninstall\x1b[0m, \x1b[33mchat\x1b[0m, \x1b[33mremember\x1b[0m, \x1b[33mrecall\x1b[0m, \x1b[33mresearch\x1b[0m, \x1b[33mmap\x1b[0m, \x1b[33mbrain\x1b[0m, \x1b[33mskill\x1b[0m
+    \x1b[33mimport\x1b[0m, \x1b[33mingest\x1b[0m, \x1b[33mconnect\x1b[0m, \x1b[33mrelay\x1b[0m, \x1b[33muninstall\x1b[0m, \x1b[33mchat\x1b[0m, \x1b[33mremember\x1b[0m, \x1b[33mrecall\x1b[0m, \x1b[33mresearch\x1b[0m, \x1b[33mmap\x1b[0m, \x1b[33mbrain\x1b[0m, \x1b[33mskill\x1b[0m, \x1b[33mcollab\x1b[0m
 `);
 }

package/src/core/dream.mjs

@@ -137,6 +137,47 @@ export function autoPruneStorage(brainDir, vaultDir, conflictsDir) {
   pruneDir(inboxDir, draftMaxAgeMs); // Keeps the local dev inbox fully clean of automatic research noise
   if (conflictsDir) pruneDir(conflictsDir, draftMaxAgeMs);
 
+  // Prune scheduler tasks queue: move completed/failed ones to archive, delete archives older than logMaxAgeMs
+  const tasksQueueDir = path.join(brainDir, 'scheduler', 'queue');
+  const tasksArchiveDir = path.join(tasksQueueDir, 'archive');
+  if (fs.existsSync(tasksQueueDir)) {
+    try {
+      if (!fs.existsSync(tasksArchiveDir)) {
+        fs.mkdirSync(tasksArchiveDir, { recursive: true });
+      }
+      const files = fs.readdirSync(tasksQueueDir);
+      for (const file of files) {
+        if (!file.endsWith('.md')) continue;
+        const fullPath = path.join(tasksQueueDir, file);
+        try {
+          const raw = fs.readFileSync(fullPath, 'utf8');
+          const { data } = matter(raw);
+          if (data.status === 'completed' || data.status === 'failed') {
+            fs.renameSync(fullPath, path.join(tasksArchiveDir, file));
+          }
+        } catch (e) {
+          // ignore parsing/permission errors for individual files
+        }
+      }
+
+      const archiveFiles = fs.readdirSync(tasksArchiveDir);
+      for (const file of archiveFiles) {
+        if (!file.endsWith('.md')) continue;
+        const fullPath = path.join(tasksArchiveDir, file);
+        try {
+          const stat = fs.statSync(fullPath);
+          if (now - stat.mtimeMs > logMaxAgeMs) {
+            fs.unlinkSync(fullPath);
+          }
+        } catch (e) {
+          // ignore
+        }
+      }
+    } catch (err) {
+      logger.warn('dream', `Failed to prune scheduler tasks: ${err.message}`);
+    }
+  }
+
   // Automatically prune transient planning files in every active/historical conversation folder,
   // while permanently preserving the conversations/threads themselves (.system_generated)
   try {

package/src/core/source-adapters.mjs

@@ -1,7 +1,15 @@
 import fs from 'fs';
 import path from 'path';
 import yaml from 'yaml';
-import { brainDir } from './config.mjs';
+import {
+  brainDir,
+  braveApiKey as defaultBraveKey,
+  braveSearchApiKey as defaultBraveSearchKey,
+  exaApiKey as defaultExaKey,
+  githubToken as defaultGithubToken,
+  serperApiKey as defaultSerperKey,
+  tavilyApiKey as defaultTavilyKey
+} from './config.mjs';
 
 /**
  * Research Source Adapters — Real Service Integrations
@@ -42,12 +50,12 @@ export function loadResearchConfig(configPath = DEFAULT_CONFIG_PATH) {
 
   return {
     // ── Paid search providers (fallback chain: Brave → Tavily → Exa → Serper) ──
-    braveApiKey:  process.env.BRAVE_SEARCH_API_KEY || process.env.BRAVE_API_KEY || secrets.brave_api_key || fileConfig.brave_api_key  || null,
-    tavilyApiKey: process.env.TAVILY_API_KEY                                    || secrets.tavily_api_key || fileConfig.tavily_api_key || null,
-    exaApiKey:    process.env.EXA_API_KEY                                       || secrets.exa_api_key    || fileConfig.exa_api_key    || null,
-    serperApiKey: process.env.SERPER_API_KEY                                    || secrets.serper_api_key || fileConfig.serper_api_key || null,
+    braveApiKey:  process.env.BRAVE_SEARCH_API_KEY || process.env.BRAVE_API_KEY || secrets.brave_api_key || fileConfig.brave_api_key  || defaultBraveKey || defaultBraveSearchKey || null,
+    tavilyApiKey: process.env.TAVILY_API_KEY                                    || secrets.tavily_api_key || fileConfig.tavily_api_key || defaultTavilyKey || null,
+    exaApiKey:    process.env.EXA_API_KEY                                       || secrets.exa_api_key    || fileConfig.exa_api_key    || defaultExaKey || null,
+    serperApiKey: process.env.SERPER_API_KEY                                    || secrets.serper_api_key || fileConfig.serper_api_key || defaultSerperKey || null,
     // GitHub: personal access token for higher rate limits (60→5000 req/hr)
-    githubToken: process.env.GITHUB_TOKEN || secrets.github_token || fileConfig.github_token || null,
+    githubToken: process.env.GITHUB_TOKEN || secrets.github_token || fileConfig.github_token || defaultGithubToken || null,
     fetchTimeoutMs: fileConfig.fetch_timeout_ms || 10000,
     maxResultsPerSource: fileConfig.max_results_per_source || 5,
     userAgent: fileConfig.user_agent || 'TotalRecall/1.0 (knowledge-acquisition; +https://github.com/gregiteen/total-recall)',

package/src/server/index.mjs

@@ -198,8 +198,12 @@ try {
   app.use('/api', apiRateLimiter());
   app.use(restRouter);
   logger.info('server', 'REST API mounted (/api/*, /v1/models, /.well-known/total-recall.json)');
+
+  const { collabRouter } = await import('./routes/collab.mjs');
+  app.use(collabRouter);
+  logger.info('server', 'Collaboration API routes mounted (/api/collab/*)');
 } catch (err) {
-  logger.error('server', `REST API failed to load: ${err.message}`);
+  logger.error('server', `REST API or Collab routes failed to load: ${err.message}`);
 }
 
 // ─── API Routes (/v1/chat/completions) ──────────────────────────────────────────
@@ -573,7 +577,23 @@ const HOST = nodeEnv === 'production' && publicBindRequested && serverSecurityCo
   ? '127.0.0.1'
   : configuredHost;
 
+import { WebSocketServer } from 'ws';
+const collabWss = new WebSocketServer({ noServer: true });
+
 const server = app.listen(PORT, HOST, () => {
+  server.on('upgrade', async (request, socket, head) => {
+    const url = new URL(request.url, `http://${request.headers.host || 'localhost'}`);
+    if (url.pathname === '/collab-ws') {
+      try {
+        const { handleCollabUpgrade } = await import('./routes/collab.mjs');
+        handleCollabUpgrade(request, socket, head, collabWss);
+      } catch (err) {
+        socket.write('HTTP/1.1 500 Internal Server Error\r\n\r\n');
+        socket.destroy();
+      }
+    }
+  });
+
   if (HOST !== configuredHost) {
     logger.error("server", `Refusing public bind '${configuredHost}' in production. Bound to ${HOST}.`);
   }

package/src/server/rest.mjs

@@ -944,12 +944,19 @@ router.get('/api/extension/status', requireAuth, requireScope('config:read'), as
   try {
     const extDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../extension');
     const available = fs.existsSync(path.join(extDir, 'manifest.json'));
+    let version = '0.0.0';
+    if (available) {
+      try {
+        const manifest = JSON.parse(fs.readFileSync(path.join(extDir, 'manifest.json'), 'utf8'));
+        version = manifest.version || '0.0.0';
+      } catch {}
+    }
 
     // Check if extension has ever connected by looking for the marker file
     const markerPath = path.join(BRAIN_DIR, 'config', '.extension-connected');
     const connected = fs.existsSync(markerPath);
 
-    res.json({ available, connected });
+    res.json({ available, connected, version });
   } catch (err) { serverError(res, err); }
 });
 
@@ -2067,4 +2074,45 @@ router.post('/api/update/run', requireAuth, async (req, res) => {
   }
 });
 
+router.get('/api/help', requireAuth, (req, res) => {
+  const { topic } = req.query;
+  const docsDir = path.join(ROOT, 'docs');
+  const referenceDir = path.join(docsDir, 'reference');
+
+  if (!topic) {
+    return res.json({
+      topics: [
+        { id: 'cli-reference', title: 'CLI Reference Guide', description: 'npx total-recall command catalog and flags' },
+        { id: 'ssss', title: 'SSSS Specifications', description: 'Structured Semantic Syntax System guide' },
+        { id: 'architecture', title: 'System Architecture', description: 'System topology and VFS structures' },
+        { id: 'collab', title: 'Collaboration Guide', description: 'Collaborative workspaces and team annotations' }
+      ]
+    });
+  }
+
+  try {
+    let filePath = '';
+    if (topic === 'cli-reference') {
+      filePath = path.join(referenceDir, 'cli-reference.md');
+    } else if (topic === 'ssss') {
+      filePath = path.join(docsDir, 'SSSS.md');
+    } else if (topic === 'architecture') {
+      filePath = path.join(docsDir, 'ARCHITECTURE.md');
+    } else if (topic === 'collab') {
+      filePath = path.join(referenceDir, 'collab.md');
+    } else {
+      return res.status(404).json({ error: 'Help topic not found' });
+    }
+
+    if (!fs.existsSync(filePath)) {
+      return res.status(404).json({ error: 'Help topic document not found on server' });
+    }
+
+    const content = fs.readFileSync(filePath, 'utf8');
+    res.json({ topic, content });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
 export { router as restRouter };

package/src/server/route-manifest.json

@@ -103,6 +103,10 @@
     "method": "GET",
     "path": "/api/graph"
   },
+  {
+    "method": "GET",
+    "path": "/api/help"
+  },
   {
     "method": "GET",
     "path": "/api/import/rules"