zouroboros-memory 3.0.0

package/dist/import-pipeline.js

@@ -0,0 +1,324 @@
+#!/usr/bin/env bun
+/**
+ * zo-memory-system Import Pipeline v1.0
+ *
+ * Import facts from external sources into the memory system.
+ *
+ * Supported sources:
+ *   chatgpt  — ChatGPT JSON export (conversations format)
+ *   obsidian — Obsidian vault (markdown files with frontmatter)
+ *   markdown — Generic markdown files
+ *
+ * Usage:
+ *   bun import.ts --source chatgpt --path ~/export.json
+ *   bun import.ts --source obsidian --path ~/Vault --dry-run
+ *   bun import.ts --source markdown --path ~/notes/file.md
+ */
+import { Database } from "bun:sqlite";
+import { randomUUID } from "crypto";
+import { existsSync, readFileSync, readdirSync, statSync } from "fs";
+import { join, basename, extname } from "path";
+// --- Config ---
+const DB_PATH = process.env.ZO_MEMORY_DB || "/home/workspace/.zo/memory/shared-facts.db";
+const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
+const EMBEDDING_MODEL = process.env.ZO_EMBEDDING_MODEL || "nomic-embed-text";
+// --- Embedding helper ---
+async function getEmbedding(text) {
+    try {
+        const resp = await fetch(`${OLLAMA_URL}/api/embeddings`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ model: EMBEDDING_MODEL, prompt: text }),
+            signal: AbortSignal.timeout(30000),
+        });
+        if (!resp.ok)
+            return null;
+        const data = await resp.json();
+        return data.embedding || null;
+    }
+    catch {
+        return null;
+    }
+}
+// --- Store fact ---
+async function storeFact(db, fact) {
+    try {
+        const id = randomUUID();
+        const now = Date.now();
+        const nowSec = Math.floor(now / 1000);
+        const expiresAt = nowSec + 90 * 86400; // stable decay = 90 days
+        db.prepare(`
+      INSERT INTO facts (id, persona, entity, key, value, text, category, decay_class,
+                         importance, source, created_at, expires_at, last_accessed, confidence)
+      VALUES (?, 'shared', ?, ?, ?, ?, ?, 'stable', 1.0, ?, ?, ?, ?, 1.0)
+    `).run(id, fact.entity, fact.key, fact.value, fact.text, fact.category, fact.source, now, expiresAt, nowSec);
+        // Generate embedding
+        const embedding = await getEmbedding(fact.text);
+        if (embedding) {
+            db.prepare("INSERT INTO fact_embeddings (fact_id, embedding, model) VALUES (?, ?, ?)")
+                .run(id, Buffer.from(new Float32Array(embedding).buffer), EMBEDDING_MODEL);
+        }
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
+// --- Check for duplicate ---
+function isDuplicate(db, entity, key, value) {
+    const row = key
+        ? db.prepare("SELECT id FROM facts WHERE entity = ? AND key = ? AND value = ?").get(entity, key, value)
+        : db.prepare("SELECT id FROM facts WHERE entity = ? AND value = ?").get(entity, value);
+    return !!row;
+}
+// ==========================================================================
+// IMPORTERS
+// ==========================================================================
+/** Import from ChatGPT JSON export (conversations format) */
+function parseChatGPTExport(filePath) {
+    const data = JSON.parse(readFileSync(filePath, "utf-8"));
+    const facts = [];
+    // Handle array of conversations
+    const conversations = Array.isArray(data) ? data : [data];
+    for (const convo of conversations) {
+        const title = convo.title || "untitled";
+        const mapping = convo.mapping || {};
+        // Extract assistant messages
+        for (const [, node] of Object.entries(mapping)) {
+            if (!node?.message)
+                continue;
+            const msg = node.message;
+            if (msg.author?.role !== "assistant")
+                continue;
+            const content = msg.content?.parts?.join("\n") || "";
+            if (content.length < 50)
+                continue; // skip short responses
+            // Extract key decisions/facts from assistant responses
+            const lines = content.split("\n").filter((l) => l.trim().length > 20);
+            // Take meaningful lines (headers, key points, conclusions)
+            const keyLines = lines.filter((l) => l.startsWith("##") || l.startsWith("- ") || l.startsWith("* ") ||
+                l.includes(":") || /^\d+\./.test(l.trim())).slice(0, 5);
+            if (keyLines.length > 0) {
+                facts.push({
+                    entity: `chatgpt.${title.replace(/[^a-zA-Z0-9]/g, "-").slice(0, 40)}`,
+                    key: "summary",
+                    value: keyLines.join("; ").slice(0, 500),
+                    text: `ChatGPT conversation "${title}": ${keyLines.join("; ").slice(0, 300)}`,
+                    category: "reference",
+                    source: `chatgpt:${basename(filePath)}`,
+                });
+            }
+        }
+    }
+    return facts;
+}
+/** Import from Obsidian vault (markdown files with YAML frontmatter) */
+function parseObsidianVault(dirPath) {
+    const facts = [];
+    function walkDir(dir) {
+        for (const entry of readdirSync(dir)) {
+            const fullPath = join(dir, entry);
+            const stat = statSync(fullPath);
+            if (stat.isDirectory()) {
+                if (!entry.startsWith("."))
+                    walkDir(fullPath);
+                continue;
+            }
+            if (extname(entry) !== ".md")
+                continue;
+            const content = readFileSync(fullPath, "utf-8");
+            const name = basename(entry, ".md");
+            // Parse YAML frontmatter
+            let frontmatter = {};
+            const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
+            if (fmMatch) {
+                // Simple YAML parser for key: value pairs
+                for (const line of fmMatch[1].split("\n")) {
+                    const kv = line.match(/^(\w+):\s*(.+)$/);
+                    if (kv)
+                        frontmatter[kv[1]] = kv[2].trim();
+                }
+            }
+            // Get body content (after frontmatter)
+            const body = fmMatch ? content.slice(fmMatch[0].length).trim() : content.trim();
+            if (body.length < 30)
+                return; // skip nearly empty files
+            // Extract entity from path or frontmatter
+            const entity = frontmatter.entity ||
+                `obsidian.${name.replace(/[^a-zA-Z0-9]/g, "-").slice(0, 40)}`;
+            const category = frontmatter.category || "reference";
+            // Extract key sections (headings and their first paragraph)
+            const sections = body.split(/^##\s+/m).filter(s => s.trim().length > 20);
+            if (sections.length > 1) {
+                // Multiple sections — create a fact per section
+                for (const section of sections.slice(0, 8)) {
+                    const lines = section.split("\n");
+                    const heading = lines[0]?.trim() || "untitled";
+                    const sectionBody = lines.slice(1).join(" ").trim().slice(0, 400);
+                    if (sectionBody.length < 20)
+                        continue;
+                    facts.push({
+                        entity,
+                        key: heading.replace(/[^a-zA-Z0-9\s-]/g, "").slice(0, 60),
+                        value: sectionBody.slice(0, 300),
+                        text: `${name} — ${heading}: ${sectionBody}`,
+                        category,
+                        source: `obsidian:${fullPath}`,
+                    });
+                }
+            }
+            else {
+                // Single document — one fact
+                facts.push({
+                    entity,
+                    key: null,
+                    value: body.slice(0, 500),
+                    text: `${name}: ${body.slice(0, 400)}`,
+                    category,
+                    source: `obsidian:${fullPath}`,
+                });
+            }
+        }
+    }
+    walkDir(dirPath);
+    return facts;
+}
+/** Import from generic markdown file */
+function parseMarkdownFile(filePath) {
+    const content = readFileSync(filePath, "utf-8");
+    const name = basename(filePath, ".md");
+    const facts = [];
+    const entity = `markdown.${name.replace(/[^a-zA-Z0-9]/g, "-").slice(0, 40)}`;
+    // Split by headings
+    const sections = content.split(/^##?\s+/m).filter(s => s.trim().length > 20);
+    if (sections.length > 1) {
+        for (const section of sections.slice(0, 10)) {
+            const lines = section.split("\n");
+            const heading = lines[0]?.trim() || "untitled";
+            const body = lines.slice(1).join(" ").trim().slice(0, 400);
+            if (body.length < 20)
+                continue;
+            facts.push({
+                entity,
+                key: heading.replace(/[^a-zA-Z0-9\s-]/g, "").slice(0, 60),
+                value: body.slice(0, 300),
+                text: `${name} — ${heading}: ${body}`,
+                category: "reference",
+                source: `markdown:${filePath}`,
+            });
+        }
+    }
+    else {
+        // Whole document as one fact
+        facts.push({
+            entity,
+            key: null,
+            value: content.slice(0, 500),
+            text: `${name}: ${content.slice(0, 400)}`,
+            category: "reference",
+            source: `markdown:${filePath}`,
+        });
+    }
+    return facts;
+}
+// ==========================================================================
+// MAIN
+// ==========================================================================
+async function main() {
+    const args = process.argv.slice(2);
+    const flags = {};
+    const boolFlags = new Set();
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--dry-run") {
+            boolFlags.add("dry-run");
+        }
+        else if (args[i].startsWith("--")) {
+            flags[args[i].slice(2)] = args[i + 1] || "";
+            i++;
+        }
+    }
+    const source = flags.source;
+    const path = flags.path;
+    const dryRun = boolFlags.has("dry-run");
+    if (!source || !path) {
+        console.error("Usage: bun import.ts --source <chatgpt|obsidian|markdown> --path <file|dir> [--dry-run]");
+        process.exit(1);
+    }
+    if (!existsSync(path)) {
+        console.error(`Path not found: ${path}`);
+        process.exit(1);
+    }
+    console.log(`Importing from ${source}: ${path}${dryRun ? " (dry run)" : ""}\n`);
+    // Parse source
+    let facts;
+    switch (source) {
+        case "chatgpt":
+            facts = parseChatGPTExport(path);
+            break;
+        case "obsidian":
+            facts = parseObsidianVault(path);
+            break;
+        case "markdown":
+            facts = parseMarkdownFile(path);
+            break;
+        default:
+            console.error(`Unknown source: ${source}. Supported: chatgpt, obsidian, markdown`);
+            process.exit(1);
+    }
+    console.log(`Parsed ${facts.length} facts from ${source}.\n`);
+    if (facts.length === 0) {
+        console.log("No facts to import.");
+        return;
+    }
+    const result = {
+        source,
+        factsFound: facts.length,
+        factsStored: 0,
+        factsSkipped: 0,
+        errors: [],
+    };
+    if (dryRun) {
+        console.log("=== DRY RUN — Preview ===\n");
+        for (const fact of facts.slice(0, 20)) {
+            console.log(`  ${fact.entity}.${fact.key || "_"} = ${fact.value.slice(0, 80)}`);
+            console.log(`    category: ${fact.category} | source: ${fact.source}`);
+            console.log();
+        }
+        if (facts.length > 20) {
+            console.log(`  ... and ${facts.length - 20} more\n`);
+        }
+        console.log(`Total: ${facts.length} facts would be imported.`);
+        return;
+    }
+    // Store facts
+    const db = new Database(DB_PATH);
+    db.exec("PRAGMA journal_mode = WAL");
+    for (const fact of facts) {
+        // Check for duplicates
+        if (isDuplicate(db, fact.entity, fact.key, fact.value)) {
+            result.factsSkipped++;
+            continue;
+        }
+        const stored = await storeFact(db, fact);
+        if (stored) {
+            result.factsStored++;
+            process.stdout.write(`  Stored: ${fact.entity}.${fact.key || "_"} (${result.factsStored}/${result.factsFound})\r`);
+        }
+        else {
+            result.errors.push(`Failed to store: ${fact.entity}.${fact.key || "_"}`);
+        }
+    }
+    db.close();
+    console.log(`\n\n=== Import Complete ===`);
+    console.log(`  Source: ${result.source}`);
+    console.log(`  Found: ${result.factsFound}`);
+    console.log(`  Stored: ${result.factsStored}`);
+    console.log(`  Skipped (duplicates): ${result.factsSkipped}`);
+    if (result.errors.length > 0) {
+        console.log(`  Errors: ${result.errors.length}`);
+        for (const err of result.errors.slice(0, 5)) {
+            console.log(`    - ${err}`);
+        }
+    }
+}
+main().catch(console.error);

package/dist/index.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Zouroboros Memory System
+ *
+ * Hybrid SQLite + Vector memory with episodic, procedural, and cognitive capabilities.
+ *
+ * @module zouroboros-memory
+ */
+export declare const VERSION = "3.0.0";
+export { initDatabase, getDatabase, closeDatabase, isInitialized, runMigrations, getDbStats, } from './database.js';
+export { generateEmbedding, generateHypotheticalAnswer, generateHyDEExpansion, blendEmbeddings, cosineSimilarity, serializeEmbedding, deserializeEmbedding, checkOllamaHealth, listAvailableModels, } from './embeddings.js';
+export { storeFact, searchFacts, searchFactsVector, searchFactsHybrid, getFact, deleteFact, touchFact, cleanupExpiredFacts, } from './facts.js';
+export { createEpisode, searchEpisodes, getEntityEpisodes, updateEpisodeOutcome, getEpisodeStats, } from './episodes.js';
+export { buildEntityGraph, getRelatedEntities, searchFactsGraphBoosted, extractQueryEntities, invalidateGraphCache, } from './graph.js';
+export { getProfile, updateTraits, updatePreferences, recordInteraction, getRecentInteractions, getProfileSummary, listProfiles, deleteProfile, ensureProfileSchema, } from './profiles.js';
+export { extractFromText, autoCapture, bufferForCapture, startAutoCapture, stopAutoCapture, getCaptureBufferSize, } from './capture.js';
+export type { CaptureResult, CaptureOptions } from './capture.js';
+export { handleMessage, startMcpServer } from './mcp-server.js';
+export { estimateTokens, estimateFactTokens, getBudget, updateBudget, resetBudget, initBudget, planCompression, createCheckpoint, loadCheckpoint, listCheckpoints, retrievalWithBudget, } from './context-budget.js';
+export type { ContextBudget, BudgetCheckpoint, CompressedFact, BudgetMetrics, CompressionPlan, RetrievalBudgetResult, } from './context-budget.js';
+export { compressEpisodes, getCompressedEpisode, listCompressedEpisodes, shouldSummarize, } from './episode-summarizer.js';
+export type { CompressedEpisode, SummarizationResult, EpisodeForCompression, ShouldSummarizeResult, } from './episode-summarizer.js';
+export { recordSearchOperation, recordCaptureOperation, recordGateDecision, collectMetrics, printReport, } from './metrics.js';
+export type { MemoryMetrics, CaptureStats, SearchMetrics, OperationStats, GateMetrics, } from './metrics.js';
+export type { HopResult, MultiHopResult, } from './multi-hop.js';
+export { isContradiction, findEntityConflicts, detectNewConflict, resolveConflict, resolveAllPending, trackProvenance, getProvenance, getFactHistory, } from './conflict-resolver.js';
+export type { ConflictType, ResolutionStrategy, ConflictRecord, ProvenanceRecord, } from './conflict-resolver.js';
+export { listPools, createPool, addToPool, removeFromPool, setInheritance, getAccessiblePersonas, searchCrossPersona, } from './cross-persona.js';
+export type { SharedPool, PersonaNode, } from './cross-persona.js';
+export { getAncestors, getDescendants, detectCycles, inferRelations, exportDot, KNOWN_RELATIONS, } from './graph-traversal.js';
+export type { MemoryEntry, MemorySearchResult, EpisodicMemory, TemporalQuery, CognitiveProfile, GraphNode, GraphEdge, } from 'zouroboros-core';
+type MemoryConfig = import('zouroboros-core').MemoryConfig;
+/**
+ * Initialize the memory system
+ */
+export declare function init(config: MemoryConfig): void;
+/**
+ * Shutdown the memory system
+ */
+export declare function shutdown(): void;
+/**
+ * Get memory system statistics
+ */
+export declare function getStats(config: MemoryConfig): {
+    database: {
+        facts: number;
+        episodes: number;
+        procedures: number;
+        openLoops: number;
+        embeddings: number;
+    };
+    episodes: {
+        total: number;
+        byOutcome: Record<string, number>;
+    };
+};

package/dist/index.js

@@ -0,0 +1,62 @@
+/**
+ * Zouroboros Memory System
+ *
+ * Hybrid SQLite + Vector memory with episodic, procedural, and cognitive capabilities.
+ *
+ * @module zouroboros-memory
+ */
+import { initDatabase as _initDb, closeDatabase as _closeDb, runMigrations as _runMigrations, getDbStats as _getDbStats } from './database.js';
+import { getEpisodeStats as _getEpisodeStats } from './episodes.js';
+import { ensureProfileSchema as _ensureProfileSchema } from './profiles.js';
+export const VERSION = '3.0.0';
+// Database
+export { initDatabase, getDatabase, closeDatabase, isInitialized, runMigrations, getDbStats, } from './database.js';
+// Embeddings
+export { generateEmbedding, generateHypotheticalAnswer, generateHyDEExpansion, blendEmbeddings, cosineSimilarity, serializeEmbedding, deserializeEmbedding, checkOllamaHealth, listAvailableModels, } from './embeddings.js';
+// Facts
+export { storeFact, searchFacts, searchFactsVector, searchFactsHybrid, getFact, deleteFact, touchFact, cleanupExpiredFacts, } from './facts.js';
+// Episodes
+export { createEpisode, searchEpisodes, getEntityEpisodes, updateEpisodeOutcome, getEpisodeStats, } from './episodes.js';
+// Graph
+export { buildEntityGraph, getRelatedEntities, searchFactsGraphBoosted, extractQueryEntities, invalidateGraphCache, } from './graph.js';
+// Cognitive Profiles
+export { getProfile, updateTraits, updatePreferences, recordInteraction, getRecentInteractions, getProfileSummary, listProfiles, deleteProfile, ensureProfileSchema, } from './profiles.js';
+// Auto-capture
+export { extractFromText, autoCapture, bufferForCapture, startAutoCapture, stopAutoCapture, getCaptureBufferSize, } from './capture.js';
+// MCP Server
+export { handleMessage, startMcpServer } from './mcp-server.js';
+// v4 Enhancements — Context Budget (MEM-001)
+export { estimateTokens, estimateFactTokens, getBudget, updateBudget, resetBudget, initBudget, planCompression, createCheckpoint, loadCheckpoint, listCheckpoints, retrievalWithBudget, } from './context-budget.js';
+// v4 Enhancements — Episode Summarizer (MEM-002)
+export { compressEpisodes, getCompressedEpisode, listCompressedEpisodes, shouldSummarize, } from './episode-summarizer.js';
+// v4 Enhancements — Metrics Dashboard (MEM-101)
+export { recordSearchOperation, recordCaptureOperation, recordGateDecision, collectMetrics, printReport, } from './metrics.js';
+// v4 Enhancements — Conflict Resolver (MEM-103)
+export { isContradiction, findEntityConflicts, detectNewConflict, resolveConflict, resolveAllPending, trackProvenance, getProvenance, getFactHistory, } from './conflict-resolver.js';
+// v4 Enhancements — Cross-Persona Memory (MEM-104)
+export { listPools, createPool, addToPool, removeFromPool, setInheritance, getAccessiblePersonas, searchCrossPersona, } from './cross-persona.js';
+// v4 Enhancements — Graph Traversal (MEM-105)
+export { getAncestors, getDescendants, detectCycles, inferRelations, exportDot, KNOWN_RELATIONS, } from './graph-traversal.js';
+/**
+ * Initialize the memory system
+ */
+export function init(config) {
+    _initDb(config);
+    _runMigrations(config);
+    _ensureProfileSchema();
+}
+/**
+ * Shutdown the memory system
+ */
+export function shutdown() {
+    _closeDb();
+}
+/**
+ * Get memory system statistics
+ */
+export function getStats(config) {
+    return {
+        database: _getDbStats(config),
+        episodes: _getEpisodeStats(),
+    };
+}

package/dist/mcp-server.d.ts

@@ -0,0 +1,31 @@
+/**
+ * MCP (Model Context Protocol) server for Zouroboros Memory
+ *
+ * Exposes memory operations as MCP tools accessible by external
+ * AI agents and clients via stdio transport.
+ *
+ * Usage: bun run packages/memory/src/mcp-server.ts [--db-path <path>]
+ */
+import type { MemoryConfig } from 'zouroboros-core';
+interface McpRequest {
+    jsonrpc: '2.0';
+    id: number | string;
+    method: string;
+    params?: Record<string, unknown>;
+}
+interface McpResponse {
+    jsonrpc: '2.0';
+    id: number | string;
+    result?: unknown;
+    error?: {
+        code: number;
+        message: string;
+        data?: unknown;
+    };
+}
+export declare function handleMessage(message: McpRequest, config: MemoryConfig): Promise<McpResponse>;
+/**
+ * Start the MCP server on stdio.
+ */
+export declare function startMcpServer(config: MemoryConfig): Promise<void>;
+export {};