total-recall-brain 3.1.0

package/src/cli/import-rules.mjs

@@ -0,0 +1,102 @@
+/**
+ * src/cli/import-rules.mjs
+ *
+ * CLI handler for: npx total-recall import [--dir <path>] [--force] [--dry-run]
+ *
+ * Detects existing agent rule files and imports them into the vault.
+ */
+
+import path from 'node:path';
+import { detectAndImport } from '../core/import-rules.mjs';
+import { resolveAgentDir, resolveBrainDir } from './agent-dir.mjs';
+
+export async function run(argv = []) {
+  const args = argv.slice(3); // strip node, script, 'import'
+
+  const dirs = [];
+  let force  = false;
+  let dryRun = false;
+
+  for (let i = 0; i < args.length; i++) {
+    if ((args[i] === '--dir' || args[i] === '-d') && args[i + 1]) {
+      dirs.push(path.resolve(args[++i]));
+    } else if (args[i] === '--force' || args[i] === '-f') {
+      force = true;
+    } else if (args[i] === '--dry-run') {
+      dryRun = true;
+    } else if (args[i] === '--help' || args[i] === '-h') {
+      printHelp();
+      return;
+    }
+  }
+
+  // Default: search cwd
+  if (dirs.length === 0) dirs.push(process.cwd());
+
+  const brainDir = resolveBrainDir();
+  const vaultDir = path.join(brainDir, 'memory-vault');
+
+  console.log(`\n🔍 Scanning for rule files in: ${dirs.join(', ')}\n`);
+
+  const result = detectAndImport({ dirs, force, vaultDir, dryRun });
+
+  if (result.detected.length === 0) {
+    console.log('ℹ️  No rule files detected. Nothing to import.');
+    console.log('   (Looked for: AGENTS.md, CLAUDE.md, .cursorrules, .clauderules, etc.)\n');
+    return;
+  }
+
+  console.log(`📋 Detected ${result.detected.length} rule file(s):\n`);
+  for (const f of result.detected) {
+    const status = f.alreadyImported ? '  [already imported]' : '';
+    console.log(`   ${f.alreadyImported ? '⚪' : '✅'} ${f.filename.padEnd(30)} ${f.label}${status}`);
+    console.log(`      ${f.absolutePath}`);
+    console.log(`      ${(f.size / 1024).toFixed(1)}KB — preview: ${f.preview.slice(0, 80).replace(/\n/g, ' ')}…\n`);
+  }
+
+  if (dryRun) {
+    console.log(`🔎 Dry run — would import ${result.toImport.length} file(s). Run without --dry-run to proceed.\n`);
+    return;
+  }
+
+  if (result.imported.length > 0) {
+    console.log(`\n✅ Imported ${result.imported.length} file(s) into vault:\n`);
+    for (const n of result.imported) {
+      console.log(`   → ${n.slug}  (${(n.size / 1024).toFixed(1)}KB)`);
+    }
+  }
+
+  if (result.skipped.length > 0) {
+    console.log(`\n⚪ Skipped ${result.skipped.length} (already imported — use --force to re-import)`);
+  }
+
+  if (result.failed.length > 0) {
+    console.log(`\n❌ Failed ${result.failed.length}:`);
+    for (const f of result.failed) console.log(`   ${f.filename}: ${f.error}`);
+  }
+
+  if (result.imported.length > 0) {
+    console.log('\n💡 Run  npx total-recall compile  to rebuild INSTRUCTIONS.md with your imported rules.\n');
+  }
+}
+
+function printHelp() {
+  console.log(`
+npx total-recall import [options]
+
+Detects existing agent rule files (AGENTS.md, .cursorrules, CLAUDE.md, etc.)
+and imports them into your Total Recall memory vault so they are included in
+all future AI sessions.
+
+Options:
+  --dir <path>   Directory to search (default: current directory)
+  --force        Re-import files that were previously imported
+  --dry-run      Show what would be imported without writing anything
+  --help         Show this help
+
+Examples:
+  npx total-recall import
+  npx total-recall import --dir ~/my-project
+  npx total-recall import --force --dry-run
+`);
+}

package/src/cli/ingest/index.mjs

@@ -0,0 +1,318 @@
+/**
+ * Google Takeout Ingestion — CLI entry point
+ *
+ * Orchestrates the ingestion of Google Takeout export data into the
+ * Total Recall memory vault. Walks the Takeout directory, detects
+ * available data types, runs parsers, deduplicates, and writes nodes.
+ *
+ * Usage:
+ *   npx total-recall ingest google-takeout <path> [options]
+ *
+ * Options:
+ *   --dry-run              Preview what would be ingested without writing
+ *   --types <comma-list>   Only process specified data types
+ *   --brain <id>           Target brain ID (default: auto-detect)
+ *   --max-age <duration>   Only include entries newer than duration (e.g. 90d, 1y)
+ *   --help, -h             Show help
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { walkTakeoutDir } from './utils/takeout-walker.mjs';
+import { dedup } from './utils/dedup.mjs';
+import { parseSearchHistory } from './parsers/search-history.mjs';
+import { parseChromeBookmarks } from './parsers/chrome-bookmarks.mjs';
+import { parseGoogleKeep } from './parsers/google-keep.mjs';
+import { parseYoutubeHistory } from './parsers/youtube-history.mjs';
+import { writeNode, loadNodes } from '../../core/vault.mjs';
+import { resolveBrainDir, parseLayerFlag, getBothBrains, defaultLayerForCategory } from '../../cli/agent-dir.mjs';
+
+/**
+ * Parser registry: maps data type names to their directory detection
+ * signatures and parser functions.
+ */
+const PARSERS = {
+  'search-history':   { detect: 'My Activity/Search',                   parser: parseSearchHistory },
+  'chrome-bookmarks': { detect: 'Chrome/Bookmarks',                     parser: parseChromeBookmarks },
+  'google-keep':      { detect: 'Keep',                                 parser: parseGoogleKeep },
+  'youtube-history':  { detect: 'YouTube and YouTube Music/history',     parser: parseYoutubeHistory },
+};
+
+/**
+ * Parse a human-readable duration string into a Date threshold.
+ * Supports: 30d, 90d, 1y, 6m, etc.
+ *
+ * @param {string} duration
+ * @returns {Date}
+ */
+function parseDuration(duration) {
+  const match = duration.match(/^(\d+)\s*(d|m|y|w)$/i);
+  if (!match) throw new Error(`Invalid duration format: "${duration}". Use e.g. 30d, 6m, 1y`);
+
+  const value = parseInt(match[1], 10);
+  const unit = match[2].toLowerCase();
+  const now = new Date();
+
+  switch (unit) {
+    case 'd': now.setDate(now.getDate() - value); break;
+    case 'w': now.setDate(now.getDate() - (value * 7)); break;
+    case 'm': now.setMonth(now.getMonth() - value); break;
+    case 'y': now.setFullYear(now.getFullYear() - value); break;
+  }
+
+  return now;
+}
+
+/**
+ * Parse CLI arguments for the google-takeout ingest command.
+ *
+ * @param {string[]} args — CLI arguments after 'ingest google-takeout'.
+ * @returns {{ takeoutPath: string|null, dryRun: boolean, types: string[]|null, brain: string|null, maxAge: Date|null, help: boolean, layer: string }}
+ */
+function parseArgs(args) {
+  const { layer, remainingArgs } = parseLayerFlag(args);
+  const opts = { takeoutPath: null, dryRun: false, types: null, brain: null, maxAge: null, help: false, layer };
+
+  for (let i = 0; i < remainingArgs.length; i++) {
+    const arg = remainingArgs[i];
+    const val = remainingArgs[i + 1];
+
+    switch (arg) {
+      case '--dry-run':
+        opts.dryRun = true;
+        break;
+      case '--types':
+        if (val) { opts.types = val.split(',').map(t => t.trim()); i++; }
+        break;
+      case '--brain':
+        if (val) { opts.brain = val; i++; }
+        break;
+      case '--max-age':
+        if (val) { opts.maxAge = parseDuration(val); i++; }
+        break;
+      case '--help': case '-h':
+        opts.help = true;
+        break;
+      default:
+        if (!arg.startsWith('-') && !opts.takeoutPath) {
+          opts.takeoutPath = arg;
+        }
+        break;
+    }
+  }
+
+  return opts;
+}
+
+/**
+ * Print help text for the google-takeout ingest command.
+ */
+function printHelp() {
+  console.log(`
+  total-recall ingest google-takeout — Ingest Google Takeout data
+
+  Walks a Google Takeout export directory, detects available data types,
+  parses them into SSSS memory nodes, deduplicates against the existing
+  vault, and writes new nodes.
+
+  Usage: total-recall ingest google-takeout <path> [options]
+
+  Supported data types:
+    search-history      Google Search history (My Activity/Search/)
+    chrome-bookmarks    Chrome bookmarks (Chrome/Bookmarks.html or .json)
+    google-keep         Google Keep notes (Keep/*.json)
+    youtube-history     YouTube watch history (YouTube and YouTube Music/history/)
+
+  Options:
+    --dry-run              Preview what would be ingested without writing
+    --types <comma-list>   Only process specified types (e.g. "search-history,google-keep")
+    --brain <id>           Target brain ID
+    --max-age <duration>   Only include entries newer than duration (30d, 6m, 1y)
+    --global               Write to global brain
+    --project              Write to project brain
+    --help, -h             Show this help
+
+  Examples:
+    npx total-recall ingest google-takeout ~/Downloads/Takeout
+    npx total-recall ingest google-takeout ~/Downloads/Takeout --dry-run
+    npx total-recall ingest google-takeout ~/Downloads/Takeout --types search-history,google-keep
+    npx total-recall ingest google-takeout ~/Downloads/Takeout --max-age 90d
+`);
+}
+
+/**
+ * Main ingestion function for Google Takeout data.
+ *
+ * @param {string} takeoutPath — Path to the Takeout export directory.
+ * @param {object} options
+ * @param {boolean} [options.dryRun=false] — Preview mode; don't write nodes.
+ * @param {string[]|null} [options.types=null] — Filter to specific data types.
+ * @param {string|null} [options.brain=null] — Target brain ID.
+ * @param {Date|null} [options.maxAge=null] — Only include entries newer than this.
+ * @param {string} [options.layer='auto'] — Brain layer target.
+ * @returns {Promise<{ detected: number, parsed: number, deduplicated: number, written: number, stats: object }>}
+ */
+export async function ingestGoogleTakeout(takeoutPath, options = {}) {
+  const { dryRun = false, types = null, maxAge = null, layer = 'auto' } = options;
+
+  // Validate path exists
+  const resolvedPath = path.resolve(takeoutPath);
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(`Takeout directory not found: ${resolvedPath}`);
+  }
+
+  // Resolve brain target
+  let effectiveLayer = layer;
+  if (effectiveLayer === 'auto') {
+    effectiveLayer = defaultLayerForCategory('facts');
+    const project = getBothBrains().project;
+    if (effectiveLayer === 'project' && !project) {
+      effectiveLayer = 'global';
+    }
+  }
+
+  const brainDir = resolveBrainDir(effectiveLayer);
+  const vaultDir = path.join(brainDir, 'memory-vault');
+
+  // Walk directory to detect available data types
+  console.error('\n  🔍 Scanning Takeout directory...\n');
+  const detected = walkTakeoutDir(resolvedPath);
+
+  if (detected.length === 0) {
+    console.error('  ⚠️  No recognized Takeout data types found.\n');
+    return { detected: 0, parsed: 0, deduplicated: 0, written: 0, stats: {} };
+  }
+
+  // Filter by requested types
+  const toProcess = types
+    ? detected.filter(d => types.includes(d.type))
+    : detected;
+
+  if (toProcess.length === 0) {
+    console.error('  ⚠️  No matching data types found for the specified filter.\n');
+    return { detected: detected.length, parsed: 0, deduplicated: 0, written: 0, stats: {} };
+  }
+
+  console.error(`  📦 Detected ${detected.length} data type(s):`);
+  for (const d of detected) {
+    const included = toProcess.some(t => t.type === d.type);
+    const icon = included ? '🟢' : '⚪';
+    console.error(`     ${icon} ${d.type} → ${d.path}`);
+  }
+  console.error('');
+
+  // Load existing vault nodes for dedup
+  let existingNodes = [];
+  if (fs.existsSync(vaultDir)) {
+    existingNodes = loadNodes(vaultDir);
+  }
+
+  // Run parsers for each detected type
+  const stats = {};
+  let allNewNodes = [];
+
+  for (const source of toProcess) {
+    const parserEntry = PARSERS[source.type];
+    if (!parserEntry) {
+      stats[source.type] = { parsed: 0, error: 'No parser available' };
+      continue;
+    }
+
+    console.error(`  ⏳ Parsing ${source.type}...`);
+    try {
+      const parserOpts = maxAge ? { maxAge } : {};
+      const nodes = parserEntry.parser(source.path, parserOpts);
+      stats[source.type] = { parsed: nodes.length };
+      allNewNodes.push(...nodes);
+      console.error(`     ✅ ${nodes.length} entries parsed`);
+    } catch (err) {
+      stats[source.type] = { parsed: 0, error: err.message };
+      console.error(`     ❌ Error: ${err.message}`);
+    }
+  }
+
+  // Dedup against existing vault
+  console.error('\n  🔄 Deduplicating against existing vault...');
+  const { unique, duplicateCount } = dedup(allNewNodes, existingNodes);
+  console.error(`     Found ${duplicateCount} duplicate(s), ${unique.length} new node(s)\n`);
+
+  // Write nodes
+  let written = 0;
+  if (dryRun) {
+    console.error('  🏷️  DRY RUN — No nodes written.\n');
+  } else {
+    // Ensure vault directory exists
+    if (!fs.existsSync(vaultDir)) {
+      fs.mkdirSync(vaultDir, { recursive: true });
+    }
+
+    for (const node of unique) {
+      try {
+        writeNode(node, vaultDir);
+        written++;
+      } catch (err) {
+        console.error(`  ⚠️  Failed to write ${node.slug}: ${err.message}`);
+      }
+    }
+    console.error(`  ✅ Wrote ${written} node(s) to vault\n`);
+  }
+
+  // Print summary report
+  const layerLabel = effectiveLayer === 'project' ? '[project]' : '[global]';
+  console.error('  ═══════════════════════════════════════');
+  console.error('  📊 Ingestion Summary');
+  console.error('  ═══════════════════════════════════════');
+  console.error(`  Brain:        ${layerLabel} ${brainDir}`);
+  console.error(`  Detected:     ${detected.length} data type(s)`);
+  console.error(`  Processed:    ${toProcess.length} data type(s)`);
+  console.error(`  Parsed:       ${allNewNodes.length} total entries`);
+  console.error(`  Duplicates:   ${duplicateCount}`);
+  console.error(`  Written:      ${dryRun ? `${unique.length} (dry run)` : written}`);
+  console.error('  ───────────────────────────────────────');
+  for (const [type, info] of Object.entries(stats)) {
+    const status = info.error ? `❌ ${info.error}` : `${info.parsed} entries`;
+    console.error(`  ${type}: ${status}`);
+  }
+  console.error('  ═══════════════════════════════════════\n');
+
+  return {
+    detected: detected.length,
+    parsed: allNewNodes.length,
+    deduplicated: duplicateCount,
+    written: dryRun ? 0 : written,
+    stats,
+  };
+}
+
+/**
+ * CLI handler for `npx total-recall ingest google-takeout <path>`.
+ *
+ * @param {string[]} args — CLI arguments after 'ingest google-takeout'.
+ */
+export async function runGoogleTakeout(args) {
+  const opts = parseArgs(args);
+
+  if (opts.help) {
+    printHelp();
+    return;
+  }
+
+  if (!opts.takeoutPath) {
+    console.error('  ❌ Error: No Takeout directory path specified.\n');
+    printHelp();
+    process.exit(1);
+  }
+
+  try {
+    await ingestGoogleTakeout(opts.takeoutPath, {
+      dryRun: opts.dryRun,
+      types: opts.types,
+      brain: opts.brain,
+      maxAge: opts.maxAge,
+      layer: opts.layer,
+    });
+  } catch (err) {
+    console.error(`  ❌ Error: ${err.message}\n`);
+    process.exit(1);
+  }
+}

package/src/cli/ingest/parsers/chrome-bookmarks.mjs

@@ -0,0 +1,238 @@
+/**
+ * chrome-bookmarks.mjs — Parse Chrome bookmarks from Takeout export
+ *
+ * Reads Chrome bookmarks from the Takeout "Chrome/" directory.
+ * Supports both JSON format (Chrome/Bookmarks) and HTML format
+ * (Chrome/Bookmarks.html) exported by Google Takeout.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { collectFiles } from '../utils/takeout-walker.mjs';
+
+/**
+ * Extract bookmarks from Chrome's JSON bookmark format.
+ * The structure nests bookmarks under roots.bookmark_bar / roots.other / roots.synced.
+ *
+ * @param {object} jsonData — Parsed JSON bookmark file.
+ * @returns {{ name: string, url: string, folder: string }[]}
+ */
+function extractFromJson(jsonData) {
+  const bookmarks = [];
+
+  function walk(node, folderPath) {
+    if (!node) return;
+
+    if (node.type === 'url' && node.url) {
+      bookmarks.push({
+        name: node.name || 'Untitled',
+        url: node.url,
+        folder: folderPath || 'Other',
+      });
+    }
+
+    if (node.children && Array.isArray(node.children)) {
+      const currentFolder = node.name || folderPath;
+      for (const child of node.children) {
+        walk(child, currentFolder);
+      }
+    }
+  }
+
+  // Chrome bookmarks JSON has a "roots" key with bookmark_bar, other, synced
+  const roots = jsonData.roots || jsonData;
+  if (roots.bookmark_bar) walk(roots.bookmark_bar, 'Bookmarks Bar');
+  if (roots.other) walk(roots.other, 'Other Bookmarks');
+  if (roots.synced) walk(roots.synced, 'Mobile Bookmarks');
+
+  // Fallback: if none of the above, try treating the root as a node
+  if (bookmarks.length === 0) {
+    walk(roots, 'Bookmarks');
+  }
+
+  return bookmarks;
+}
+
+/**
+ * Extract bookmarks from Chrome's HTML export format.
+ * Parses <DT><A HREF="url">name</A> patterns and tracks folder hierarchy.
+ *
+ * @param {string} htmlContent — Raw HTML string.
+ * @returns {{ name: string, url: string, folder: string }[]}
+ */
+function extractFromHtml(htmlContent) {
+  const bookmarks = [];
+  const folderStack = ['Bookmarks'];
+
+  const lines = htmlContent.split('\n');
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    // Detect folder headers: <DT><H3 ...>Folder Name</H3>
+    const folderMatch = trimmed.match(/<H3[^>]*>([^<]+)<\/H3>/i);
+    if (folderMatch) {
+      folderStack.push(folderMatch[1]);
+      continue;
+    }
+
+    // Detect end of folder list
+    if (trimmed === '</DL><p>' || trimmed === '</DL>') {
+      if (folderStack.length > 1) folderStack.pop();
+      continue;
+    }
+
+    // Detect bookmark links: <DT><A HREF="url" ...>name</A>
+    const linkMatch = trimmed.match(/<A\s+HREF="([^"]+)"[^>]*>([^<]*)<\/A>/i);
+    if (linkMatch) {
+      bookmarks.push({
+        name: linkMatch[2] || 'Untitled',
+        url: linkMatch[1],
+        folder: folderStack[folderStack.length - 1] || 'Bookmarks',
+      });
+    }
+  }
+
+  return bookmarks;
+}
+
+/**
+ * Generate a slug-safe string from a bookmark name.
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+function slugify(name) {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 50);
+}
+
+/**
+ * Extract domain from a URL for tagging.
+ *
+ * @param {string} url
+ * @returns {string}
+ */
+function extractDomain(url) {
+  try {
+    return new URL(url).hostname.replace(/^www\./, '');
+  } catch {
+    return 'unknown';
+  }
+}
+
+/**
+ * Parse Chrome bookmarks from the given directory path.
+ *
+ * @param {string} dirPath — Path to "Chrome/" directory.
+ * @param {object} [options]
+ * @returns {object[]} Array of SSSS-compatible memory node objects.
+ */
+export function parseChromeBookmarks(dirPath, options = {}) {
+  let rawBookmarks = [];
+
+  // Try JSON first
+  const jsonFiles = collectFiles(dirPath, ['.json']);
+  for (const file of jsonFiles) {
+    if (path.basename(file).toLowerCase().includes('bookmark')) {
+      try {
+        const raw = fs.readFileSync(file, 'utf8');
+        const data = JSON.parse(raw);
+        rawBookmarks.push(...extractFromJson(data));
+      } catch {
+        // Skip unparseable files
+      }
+    }
+  }
+
+  // Try HTML if no JSON results
+  if (rawBookmarks.length === 0) {
+    const htmlFiles = collectFiles(dirPath, ['.html']);
+    for (const file of htmlFiles) {
+      if (path.basename(file).toLowerCase().includes('bookmark')) {
+        try {
+          const raw = fs.readFileSync(file, 'utf8');
+          rawBookmarks.push(...extractFromHtml(raw));
+        } catch {
+          // Skip unparseable files
+        }
+      }
+    }
+  }
+
+  // Convert to SSSS nodes
+  const now = new Date().toISOString();
+  const nodes = [];
+  const seenUrls = new Set();
+
+  for (const bm of rawBookmarks) {
+    if (!bm.url || seenUrls.has(bm.url)) continue;
+    seenUrls.add(bm.url);
+
+    // Skip internal Chrome URLs
+    if (bm.url.startsWith('chrome://') || bm.url.startsWith('chrome-extension://')) continue;
+
+    const suffix = crypto.randomBytes(3).toString('hex');
+    const slugBase = slugify(bm.name) || 'bookmark';
+    const slug = `bookmark-${slugBase}-${suffix}`;
+    const domain = extractDomain(bm.url);
+    const folderTag = bm.folder
+      ? bm.folder.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
+      : '';
+
+    const tags = ['google-takeout', 'chrome-bookmarks'];
+    if (folderTag) tags.push(folderTag);
+
+    nodes.push({
+      type: 'memory',
+      slug,
+      title: bm.name,
+      category: 'facts',
+      body: `Bookmark: ${bm.name}\nURL: ${bm.url}\nFolder: ${bm.folder}\nDomain: ${domain}`,
+      status: 'active',
+      confidence: 0.9,
+      importance: 2,
+      created: now,
+      updated: now,
+      last_accessed: now,
+      source: {
+        type: 'chrome-bookmarks',
+        session_id: `takeout-bookmark-${suffix}`,
+        evidence_count: 1,
+      },
+      supersedes: [],
+      superseded_by: null,
+      contradicts: [],
+      tags,
+      related: [],
+      routes_to_skills: [],
+      sentiment_polarity: 'descriptive',
+      sentiment_target: bm.name.slice(0, 40),
+      modality: 'should',
+      subject: 'user',
+      predicate: 'bookmarked',
+      object: bm.name.slice(0, 120),
+      decay: { half_life_days: 90, access_count: 0 },
+      schema_version: 2,
+      x_temporal_context: now,
+      x_citations: [{
+        source: 'chrome-bookmarks',
+        title: bm.name,
+        url: bm.url,
+        published: now,
+        relevance: 1.0,
+        accessed: now,
+      }],
+      x_browser_context: {
+        url: bm.url,
+        domain,
+        title: bm.name,
+      },
+    });
+  }
+
+  return nodes;
+}