total-recall-brain 3.1.0

package/src/cli/ingest/parsers/google-keep.mjs

@@ -0,0 +1,147 @@
+/**
+ * google-keep.mjs — Parse Google Keep notes from Takeout export
+ *
+ * Reads JSON files from the "Keep/" directory and converts each note
+ * into an SSSS-compatible memory node. Notes are categorized as 'lore'
+ * since they represent the user's own words and thoughts.
+ */
+
+import fs from 'node:fs';
+import crypto from 'node:crypto';
+import { collectFiles } from '../utils/takeout-walker.mjs';
+
+/**
+ * Generate a slug-safe string from a title.
+ *
+ * @param {string} title
+ * @returns {string}
+ */
+function slugify(title) {
+  return title
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 50);
+}
+
+/**
+ * Convert a Google Keep microsecond timestamp to ISO 8601.
+ *
+ * @param {number} usec — Timestamp in microseconds since epoch.
+ * @returns {string} ISO 8601 datetime string.
+ */
+function usecToIso(usec) {
+  if (!usec || typeof usec !== 'number') return new Date().toISOString();
+  return new Date(usec / 1000).toISOString();
+}
+
+/**
+ * Parse Google Keep notes from the given directory path.
+ *
+ * Each Keep JSON file represents a single note with fields:
+ * - title: Note title
+ * - textContent: Note body text
+ * - labels: Array of { name: string } label objects
+ * - color: Note color
+ * - createdTimestampUsec: Creation time in microseconds
+ * - userEditedTimestampUsec: Last edit time in microseconds
+ * - isTrashed / isArchived: Status flags
+ * - listContent: Array of { text, isChecked } for list notes
+ *
+ * @param {string} dirPath — Path to "Keep/" directory.
+ * @param {object} [options]
+ * @param {Date} [options.maxAge] — Only include notes newer than this date.
+ * @returns {object[]} Array of SSSS-compatible memory node objects.
+ */
+export function parseGoogleKeep(dirPath, options = {}) {
+  const jsonFiles = collectFiles(dirPath, ['.json']);
+  if (jsonFiles.length === 0) return [];
+
+  const nodes = [];
+
+  for (const file of jsonFiles) {
+    let note;
+    try {
+      const raw = fs.readFileSync(file, 'utf8');
+      note = JSON.parse(raw);
+    } catch {
+      continue; // Skip unparseable files
+    }
+
+    // Skip trashed and archived notes
+    if (note.isTrashed || note.isArchived) continue;
+
+    // Build body text from either textContent or listContent
+    let bodyText = note.textContent || '';
+    if (!bodyText && note.listContent && Array.isArray(note.listContent)) {
+      bodyText = note.listContent
+        .map(item => `${item.isChecked ? '☑' : '☐'} ${item.text || ''}`)
+        .join('\n');
+    }
+
+    // Skip empty notes
+    if (!bodyText.trim() && !note.title) continue;
+
+    const createdTs = usecToIso(note.createdTimestampUsec);
+    const updatedTs = usecToIso(note.userEditedTimestampUsec || note.createdTimestampUsec);
+
+    // Filter by max age
+    if (options.maxAge && new Date(createdTs) < options.maxAge) continue;
+
+    const title = note.title || bodyText.slice(0, 80).replace(/\n/g, ' ');
+    const suffix = crypto.randomBytes(3).toString('hex');
+    const slugBase = slugify(title) || 'keep-note';
+    const slug = `keep-${slugBase}-${suffix}`;
+
+    // Extract labels as tags
+    const labels = (note.labels || [])
+      .map(l => l.name)
+      .filter(Boolean)
+      .map(l => l.toLowerCase().replace(/[^a-z0-9]+/g, '-'));
+    const tags = ['google-takeout', 'google-keep', ...labels];
+
+    nodes.push({
+      type: 'memory',
+      slug,
+      title,
+      category: 'lore',
+      body: bodyText.trim(),
+      status: 'active',
+      confidence: 0.9,
+      importance: 3,
+      created: createdTs,
+      updated: updatedTs,
+      last_accessed: updatedTs,
+      source: {
+        type: 'google-keep',
+        session_id: `takeout-keep-${suffix}`,
+        evidence_count: 1,
+      },
+      supersedes: [],
+      superseded_by: null,
+      contradicts: [],
+      tags,
+      related: [],
+      routes_to_skills: [],
+      sentiment_polarity: 'descriptive',
+      sentiment_target: title.slice(0, 40),
+      modality: 'should',
+      subject: 'user',
+      predicate: 'noted',
+      object: title.slice(0, 120),
+      decay: { half_life_days: 120, access_count: 0 },
+      schema_version: 2,
+      x_temporal_context: createdTs,
+      x_citations: [{
+        source: 'google-keep',
+        title,
+        url: 'keep://note',
+        published: createdTs,
+        relevance: 1.0,
+        accessed: updatedTs,
+      }],
+    });
+  }
+
+  return nodes;
+}

package/src/cli/ingest/parsers/search-history.mjs

@@ -0,0 +1,142 @@
+/**
+ * search-history.mjs — Parse Google Search history from Takeout export
+ *
+ * Reads JSON files from "My Activity/Search/" and extracts search queries,
+ * grouping related searches into SSSS-compatible memory nodes.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { collectFiles } from '../utils/takeout-walker.mjs';
+
+/**
+ * Extract the raw search query from a Takeout search entry title.
+ * Takeout entries typically use "Searched for <query>" format.
+ *
+ * @param {string} title — The raw title string from the JSON entry.
+ * @returns {string} The extracted search query.
+ */
+function extractQuery(title) {
+  if (!title) return '';
+  return title.replace(/^Searched for\s+/i, '').trim();
+}
+
+/**
+ * Generate a slug-safe string from a query.
+ *
+ * @param {string} query
+ * @returns {string}
+ */
+function slugify(query) {
+  return query
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 50);
+}
+
+/**
+ * Parse Google Search history from the given directory path.
+ *
+ * Reads all JSON files under the search history directory. Each file
+ * may contain a top-level array of search entries or a single
+ * MyActivity.json with nested structure.
+ *
+ * Each entry typically has: { title, titleUrl, time }
+ *
+ * @param {string} dirPath — Path to "My Activity/Search/" directory.
+ * @param {object} [options]
+ * @param {Date} [options.maxAge] — Only include entries newer than this date.
+ * @returns {object[]} Array of SSSS-compatible memory node objects.
+ */
+export function parseSearchHistory(dirPath, options = {}) {
+  const jsonFiles = collectFiles(dirPath, ['.json']);
+  if (jsonFiles.length === 0) return [];
+
+  const allEntries = [];
+
+  for (const file of jsonFiles) {
+    try {
+      const raw = fs.readFileSync(file, 'utf8');
+      const data = JSON.parse(raw);
+
+      // Takeout exports search history as a top-level array
+      const entries = Array.isArray(data) ? data : [];
+      allEntries.push(...entries);
+    } catch {
+      // Skip unparseable files
+    }
+  }
+
+  // Filter by max age if specified
+  const filtered = options.maxAge
+    ? allEntries.filter(e => e.time && new Date(e.time) >= options.maxAge)
+    : allEntries;
+
+  // Sort by time, most recent first
+  filtered.sort((a, b) => {
+    const ta = a.time ? new Date(a.time).getTime() : 0;
+    const tb = b.time ? new Date(b.time).getTime() : 0;
+    return tb - ta;
+  });
+
+  // Convert each search entry into an SSSS node
+  const nodes = [];
+  const seenQueries = new Set();
+
+  for (const entry of filtered) {
+    const query = extractQuery(entry.title);
+    if (!query || seenQueries.has(query.toLowerCase())) continue;
+    seenQueries.add(query.toLowerCase());
+
+    const ts = entry.time ? new Date(entry.time).toISOString() : new Date().toISOString();
+    const suffix = crypto.randomBytes(3).toString('hex');
+    const slugBase = slugify(query) || 'search';
+    const slug = `search-${slugBase}-${suffix}`;
+
+    nodes.push({
+      type: 'memory',
+      slug,
+      title: query,
+      category: 'facts',
+      body: `Search query: ${query}${entry.titleUrl ? `\nURL: ${entry.titleUrl}` : ''}`,
+      status: 'active',
+      confidence: 0.7,
+      importance: 2,
+      created: ts,
+      updated: ts,
+      last_accessed: ts,
+      source: {
+        type: 'google-search-history',
+        session_id: `takeout-search-${suffix}`,
+        evidence_count: 1,
+      },
+      supersedes: [],
+      superseded_by: null,
+      contradicts: [],
+      tags: ['google-takeout', 'search-history'],
+      related: [],
+      routes_to_skills: [],
+      sentiment_polarity: 'descriptive',
+      sentiment_target: query.slice(0, 40),
+      modality: 'should',
+      subject: 'user',
+      predicate: 'searched_for',
+      object: query.slice(0, 120),
+      decay: { half_life_days: 60, access_count: 0 },
+      schema_version: 2,
+      x_temporal_context: ts,
+      x_citations: entry.titleUrl ? [{
+        source: 'google-search',
+        title: query,
+        url: entry.titleUrl,
+        published: ts,
+        relevance: 0.8,
+        accessed: ts,
+      }] : [],
+    });
+  }
+
+  return nodes;
+}

package/src/cli/ingest/parsers/youtube-history.mjs

@@ -0,0 +1,181 @@
+/**
+ * youtube-history.mjs — Parse YouTube watch history from Takeout export
+ *
+ * Reads JSON files from "YouTube and YouTube Music/history/" and converts
+ * video watch entries into SSSS-compatible memory nodes, grouping by channel.
+ */
+
+import fs from 'node:fs';
+import crypto from 'node:crypto';
+import { collectFiles } from '../utils/takeout-walker.mjs';
+
+/**
+ * Generate a slug-safe string from text.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function slugify(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 50);
+}
+
+/**
+ * Extract the video title from a Takeout entry.
+ * YouTube entries use "Watched <title>" format.
+ *
+ * @param {string} title
+ * @returns {string}
+ */
+function extractVideoTitle(title) {
+  if (!title) return '';
+  return title.replace(/^Watched\s+/i, '').trim();
+}
+
+/**
+ * Extract channel name from a Takeout entry's subtitles array.
+ *
+ * @param {object} entry — A YouTube history entry.
+ * @returns {string}
+ */
+function extractChannel(entry) {
+  if (entry.subtitles && Array.isArray(entry.subtitles)) {
+    for (const sub of entry.subtitles) {
+      if (sub.name) return sub.name;
+    }
+  }
+  return 'Unknown Channel';
+}
+
+/**
+ * Extract YouTube video URL from a Takeout entry.
+ *
+ * @param {object} entry
+ * @returns {string|null}
+ */
+function extractVideoUrl(entry) {
+  return entry.titleUrl || null;
+}
+
+/**
+ * Parse YouTube watch history from the given directory path.
+ *
+ * Each entry in the JSON has:
+ * - title: "Watched <video title>"
+ * - titleUrl: YouTube video URL
+ * - subtitles: [{ name: "Channel Name", url: "..." }]
+ * - time: ISO 8601 timestamp
+ * - products: ["YouTube"]
+ * - activityControls: [...]
+ *
+ * @param {string} dirPath — Path to "YouTube and YouTube Music/history/" directory.
+ * @param {object} [options]
+ * @param {Date} [options.maxAge] — Only include entries newer than this date.
+ * @returns {object[]} Array of SSSS-compatible memory node objects.
+ */
+export function parseYoutubeHistory(dirPath, options = {}) {
+  const jsonFiles = collectFiles(dirPath, ['.json']);
+  if (jsonFiles.length === 0) return [];
+
+  const allEntries = [];
+
+  for (const file of jsonFiles) {
+    try {
+      const raw = fs.readFileSync(file, 'utf8');
+      const data = JSON.parse(raw);
+      const entries = Array.isArray(data) ? data : [];
+      allEntries.push(...entries);
+    } catch {
+      // Skip unparseable files
+    }
+  }
+
+  // Filter out entries without titles (ads, etc.) and apply max age filter
+  const filtered = allEntries.filter(e => {
+    if (!e.title) return false;
+    // Skip ad-related entries
+    if (e.title === 'Visited YouTube' || e.title.startsWith('Searched for')) return false;
+    if (options.maxAge && e.time && new Date(e.time) < options.maxAge) return false;
+    return true;
+  });
+
+  // Sort by time, most recent first
+  filtered.sort((a, b) => {
+    const ta = a.time ? new Date(a.time).getTime() : 0;
+    const tb = b.time ? new Date(b.time).getTime() : 0;
+    return tb - ta;
+  });
+
+  // Convert each watch entry to an SSSS node
+  const nodes = [];
+  const seenUrls = new Set();
+
+  for (const entry of filtered) {
+    const videoTitle = extractVideoTitle(entry.title);
+    if (!videoTitle) continue;
+
+    const url = extractVideoUrl(entry);
+    // Deduplicate by URL within this batch
+    if (url && seenUrls.has(url)) continue;
+    if (url) seenUrls.add(url);
+
+    const channel = extractChannel(entry);
+    const ts = entry.time ? new Date(entry.time).toISOString() : new Date().toISOString();
+    const suffix = crypto.randomBytes(3).toString('hex');
+    const slugBase = slugify(videoTitle) || 'youtube';
+    const slug = `youtube-${slugBase}-${suffix}`;
+
+    const channelTag = channel
+      ? channel.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
+      : '';
+    const tags = ['google-takeout', 'youtube-history'];
+    if (channelTag) tags.push(channelTag);
+
+    nodes.push({
+      type: 'memory',
+      slug,
+      title: videoTitle,
+      category: 'facts',
+      body: `Video: ${videoTitle}\nChannel: ${channel}${url ? `\nURL: ${url}` : ''}\nWatched: ${ts}`,
+      status: 'active',
+      confidence: 0.7,
+      importance: 2,
+      created: ts,
+      updated: ts,
+      last_accessed: ts,
+      source: {
+        type: 'youtube-history',
+        session_id: `takeout-youtube-${suffix}`,
+        evidence_count: 1,
+      },
+      supersedes: [],
+      superseded_by: null,
+      contradicts: [],
+      tags,
+      related: [],
+      routes_to_skills: [],
+      sentiment_polarity: 'descriptive',
+      sentiment_target: videoTitle.slice(0, 40),
+      modality: 'should',
+      subject: 'user',
+      predicate: 'watched',
+      object: videoTitle.slice(0, 120),
+      decay: { half_life_days: 60, access_count: 0 },
+      schema_version: 2,
+      x_temporal_context: ts,
+      x_citations: url ? [{
+        source: 'youtube',
+        title: videoTitle,
+        url,
+        published: ts,
+        relevance: 0.8,
+        accessed: ts,
+      }] : [],
+    });
+  }
+
+  return nodes;
+}

package/src/cli/ingest/utils/dedup.mjs

@@ -0,0 +1,69 @@
+/**
+ * dedup.mjs — SHA-256 based deduplication for memory nodes
+ *
+ * Hashes each node's body content and compares against existing vault
+ * nodes to filter out duplicates before writing.
+ */
+
+import crypto from 'node:crypto';
+
+/**
+ * Compute a SHA-256 hex digest of a string.
+ *
+ * @param {string} content — The content to hash.
+ * @returns {string} Hex-encoded SHA-256 hash.
+ */
+export function sha256(content) {
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+/**
+ * Build a content fingerprint for a memory node.
+ * Uses the body plus key metadata fields so that nodes with
+ * identical content but different slugs are still detected as dupes.
+ *
+ * @param {object} node — An SSSS-compatible memory node object.
+ * @returns {string} Hex-encoded SHA-256 hash.
+ */
+function nodeFingerprint(node) {
+  const parts = [
+    node.body || '',
+    node.title || '',
+    node.source?.type || '',
+  ];
+  return sha256(parts.join('\n'));
+}
+
+/**
+ * Deduplicate new nodes against existing vault nodes.
+ *
+ * Hashes each new node's body + metadata and compares against hashes of
+ * existing nodes. Returns only nodes that have no matching fingerprint
+ * in the existing set.
+ *
+ * @param {object[]} newNodes — Candidate nodes to ingest.
+ * @param {object[]} existingNodes — Already-stored vault nodes.
+ * @returns {{ unique: object[], duplicateCount: number }}
+ */
+export function dedup(newNodes, existingNodes) {
+  // Build a set of existing fingerprints
+  const existingHashes = new Set();
+  for (const node of existingNodes) {
+    existingHashes.add(nodeFingerprint(node));
+  }
+
+  const unique = [];
+  let duplicateCount = 0;
+
+  for (const node of newNodes) {
+    const fp = nodeFingerprint(node);
+    if (existingHashes.has(fp)) {
+      duplicateCount++;
+    } else {
+      existingHashes.add(fp); // Prevent intra-batch dupes too
+      unique.push(node);
+    }
+  }
+
+  return { unique, duplicateCount };
+}

package/src/cli/ingest/utils/takeout-walker.mjs

@@ -0,0 +1,101 @@
+/**
+ * takeout-walker.mjs — Walk a Google Takeout directory and detect available data types
+ *
+ * Recursively checks for known Takeout subdirectory signatures and returns
+ * which data sources are present along with their resolved paths.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Known Takeout data type signatures.
+ * Each entry maps a type name to one or more directory path fragments
+ * (relative to the Takeout root) that indicate the data is present.
+ */
+const TAKEOUT_SIGNATURES = [
+  {
+    type: 'search-history',
+    /** Google Search history lives under "My Activity/Search/" */
+    paths: ['My Activity/Search'],
+  },
+  {
+    type: 'chrome-bookmarks',
+    /** Chrome bookmarks can be in "Chrome/Bookmarks.html" or "Chrome/Bookmarks.json" */
+    paths: ['Chrome'],
+  },
+  {
+    type: 'google-keep',
+    /** Google Keep notes are exported under "Keep/" */
+    paths: ['Keep'],
+  },
+  {
+    type: 'youtube-history',
+    /** YouTube watch history under "YouTube and YouTube Music/history/" */
+    paths: ['YouTube and YouTube Music/history'],
+  },
+];
+
+/**
+ * Walk a Google Takeout root directory and return which data types are present.
+ *
+ * The function checks for the existence of known subdirectory signatures
+ * inside `rootPath`. Some Takeout exports nest everything under a top-level
+ * "Takeout/" directory — we handle that transparently.
+ *
+ * @param {string} rootPath — Absolute path to the Takeout export root.
+ * @returns {{ type: string, path: string }[]} Array of detected data sources.
+ */
+export function walkTakeoutDir(rootPath) {
+  if (!fs.existsSync(rootPath)) {
+    throw new Error(`Takeout directory does not exist: ${rootPath}`);
+  }
+
+  const stat = fs.statSync(rootPath);
+  if (!stat.isDirectory()) {
+    throw new Error(`Not a directory: ${rootPath}`);
+  }
+
+  // Some exports nest everything under a "Takeout/" subdirectory.
+  const nestedTakeout = path.join(rootPath, 'Takeout');
+  const effectiveRoot = fs.existsSync(nestedTakeout) && fs.statSync(nestedTakeout).isDirectory()
+    ? nestedTakeout
+    : rootPath;
+
+  const detected = [];
+
+  for (const sig of TAKEOUT_SIGNATURES) {
+    for (const relPath of sig.paths) {
+      const fullPath = path.join(effectiveRoot, relPath);
+      if (fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) {
+        detected.push({ type: sig.type, path: fullPath });
+        break; // Only add each type once
+      }
+    }
+  }
+
+  return detected;
+}
+
+/**
+ * Recursively collect all files matching given extensions within a directory.
+ *
+ * @param {string} dir — Directory to search.
+ * @param {string[]} extensions — File extensions to include (e.g. ['.json', '.html']).
+ * @returns {string[]} Array of absolute file paths.
+ */
+export function collectFiles(dir, extensions) {
+  const results = [];
+  if (!fs.existsSync(dir)) return results;
+
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...collectFiles(fullPath, extensions));
+    } else if (extensions.some(ext => entry.name.endsWith(ext))) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}