stellar-memory 0.5.0

package/dist/storage/database.js

@@ -0,0 +1,265 @@
+import { DatabaseSync } from 'node:sqlite';
+import { loadVecExtension, VEC_DDL } from './vec.js';
+// Singleton instance
+let _db = null;
+const DDL = `
+-- Core memories table
+CREATE TABLE IF NOT EXISTS memories (
+  id TEXT PRIMARY KEY,
+  project TEXT NOT NULL DEFAULT 'default',
+  content TEXT NOT NULL,
+  summary TEXT NOT NULL,
+  type TEXT NOT NULL DEFAULT 'observation',
+  tags TEXT DEFAULT '[]',
+  distance REAL NOT NULL DEFAULT 5.0,
+  importance REAL NOT NULL DEFAULT 0.5,
+  velocity REAL NOT NULL DEFAULT 0.0,
+  impact REAL NOT NULL DEFAULT 0.5,
+  access_count INTEGER NOT NULL DEFAULT 0,
+  last_accessed_at TEXT,
+  metadata TEXT DEFAULT '{}',
+  source TEXT DEFAULT 'manual',
+  source_path TEXT,
+  source_hash TEXT,
+  content_hash TEXT,
+  valid_from TEXT,
+  valid_until TEXT,
+  superseded_by TEXT,
+  consolidated_into TEXT,
+  quality_score REAL DEFAULT 0.5,
+  is_universal INTEGER DEFAULT 0,
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+  deleted_at TEXT
+);
+
+-- Data sources table: tracks directories registered for scanning
+CREATE TABLE IF NOT EXISTS data_sources (
+  id TEXT PRIMARY KEY,
+  path TEXT NOT NULL,
+  type TEXT NOT NULL DEFAULT 'local',
+  status TEXT NOT NULL DEFAULT 'active',
+  last_scanned_at TEXT,
+  file_count INTEGER DEFAULT 0,
+  total_size INTEGER DEFAULT 0,
+  config TEXT DEFAULT '{}',
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Index for deduplication lookup
+CREATE INDEX IF NOT EXISTS idx_memories_source_path
+  ON memories(source_path)
+  WHERE source_path IS NOT NULL;
+
+-- Index for content-hash deduplication
+CREATE INDEX IF NOT EXISTS idx_memories_content_hash
+  ON memories(project, content_hash)
+  WHERE content_hash IS NOT NULL;
+
+-- Sun state — one row per project, acts as the "star" context
+CREATE TABLE IF NOT EXISTS sun_state (
+  project TEXT PRIMARY KEY,
+  content TEXT NOT NULL DEFAULT '',
+  current_work TEXT DEFAULT '',
+  recent_decisions TEXT DEFAULT '[]',
+  next_steps TEXT DEFAULT '[]',
+  active_errors TEXT DEFAULT '[]',
+  project_context TEXT DEFAULT '',
+  token_count INTEGER NOT NULL DEFAULT 0,
+  last_commit_at TEXT,
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Audit log of orbital position changes
+CREATE TABLE IF NOT EXISTS orbit_log (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  memory_id TEXT NOT NULL,
+  project TEXT NOT NULL,
+  old_distance REAL NOT NULL,
+  new_distance REAL NOT NULL,
+  old_importance REAL NOT NULL,
+  new_importance REAL NOT NULL,
+  trigger TEXT NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- FTS5 virtual table for full-text search
+CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+  content, summary, tags,
+  content='memories', content_rowid='rowid'
+);
+
+-- Keep FTS index in sync with the memories table
+CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+  INSERT INTO memories_fts(rowid, content, summary, tags)
+  VALUES (new.rowid, new.content, new.summary, new.tags);
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+  INSERT INTO memories_fts(memories_fts, rowid, content, summary, tags)
+  VALUES ('delete', old.rowid, old.content, old.summary, old.tags);
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+  INSERT INTO memories_fts(memories_fts, rowid, content, summary, tags)
+  VALUES ('delete', old.rowid, old.content, old.summary, old.tags);
+  INSERT INTO memories_fts(rowid, content, summary, tags)
+  VALUES (new.rowid, new.content, new.summary, new.tags);
+END;
+
+-- Indexes for common access patterns
+CREATE INDEX IF NOT EXISTS idx_memories_project
+  ON memories(project);
+
+CREATE INDEX IF NOT EXISTS idx_memories_project_distance
+  ON memories(project, distance);
+
+CREATE INDEX IF NOT EXISTS idx_memories_project_importance
+  ON memories(project, importance DESC);
+
+CREATE INDEX IF NOT EXISTS idx_memories_deleted
+  ON memories(deleted_at);
+
+CREATE INDEX IF NOT EXISTS idx_memories_project_created
+  ON memories(project, created_at);
+
+-- Knowledge Graph: Constellation edges
+CREATE TABLE IF NOT EXISTS constellation_edges (
+  id TEXT PRIMARY KEY,
+  source_id TEXT NOT NULL REFERENCES memories(id),
+  target_id TEXT NOT NULL REFERENCES memories(id),
+  relation TEXT NOT NULL DEFAULT 'related_to',
+  weight REAL NOT NULL DEFAULT 0.5,
+  project TEXT NOT NULL DEFAULT 'default',
+  metadata TEXT DEFAULT '{}',
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  UNIQUE(source_id, target_id, relation)
+);
+CREATE INDEX IF NOT EXISTS idx_constellation_source ON constellation_edges(source_id);
+CREATE INDEX IF NOT EXISTS idx_constellation_target ON constellation_edges(target_id);
+CREATE INDEX IF NOT EXISTS idx_constellation_project ON constellation_edges(project);
+
+-- Conflict Detection
+CREATE TABLE IF NOT EXISTS memory_conflicts (
+  id TEXT PRIMARY KEY,
+  memory_id TEXT NOT NULL REFERENCES memories(id),
+  conflicting_memory_id TEXT NOT NULL REFERENCES memories(id),
+  severity TEXT NOT NULL DEFAULT 'medium',
+  description TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'open',
+  resolution TEXT,
+  project TEXT NOT NULL DEFAULT 'default',
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  resolved_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_conflicts_project ON memory_conflicts(project);
+CREATE INDEX IF NOT EXISTS idx_conflicts_status ON memory_conflicts(status);
+
+-- Observation Log
+CREATE TABLE IF NOT EXISTS observation_log (
+  id TEXT PRIMARY KEY,
+  content TEXT NOT NULL,
+  extracted_memories TEXT DEFAULT '[]',
+  source TEXT NOT NULL DEFAULT 'conversation',
+  project TEXT NOT NULL DEFAULT 'default',
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE INDEX IF NOT EXISTS idx_observations_project ON observation_log(project);
+`;
+// Migrate existing databases that lack newer columns.
+// Each entry: [column_name, column_definition]
+const MIGRATIONS = [
+    ['source', "TEXT DEFAULT 'manual'"],
+    ['source_path', 'TEXT'],
+    ['source_hash', 'TEXT'],
+    ['content_hash', 'TEXT'],
+    ['valid_from', 'TEXT'],
+    ['valid_until', 'TEXT'],
+    ['superseded_by', 'TEXT'],
+    ['consolidated_into', 'TEXT'],
+    ['quality_score', 'REAL DEFAULT 0.5'],
+    ['is_universal', 'INTEGER DEFAULT 0'],
+];
+function migrateMemoriesTable(db) {
+    const cols = db.prepare('PRAGMA table_info(memories)').all();
+    const existing = new Set(cols.map((c) => c.name));
+    for (const [col, def] of MIGRATIONS) {
+        if (!existing.has(col)) {
+            db.exec(`ALTER TABLE memories ADD COLUMN ${col} ${def};`);
+        }
+    }
+}
+export function initDatabase(dbPath) {
+    // allowExtension is required for sqlite-vec to load its native module.
+    const db = new DatabaseSync(dbPath, { allowExtension: true });
+    // Enable WAL mode for better concurrent read performance
+    db.exec('PRAGMA journal_mode = WAL;');
+    db.exec('PRAGMA foreign_keys = ON;');
+    db.exec('PRAGMA synchronous = NORMAL;');
+    // Load sqlite-vec extension (must be before DDL that references vec0)
+    try {
+        loadVecExtension(db);
+    }
+    catch (err) {
+        // Non-fatal: vec extension may be unavailable in some test environments.
+        // Vector search will fall back to FTS5-only mode.
+        const msg = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[stellar-memory] sqlite-vec not available: ${msg}\n`);
+    }
+    // Create all tables, triggers, and indexes
+    db.exec(DDL);
+    // Migrate existing tables that predate newer columns
+    try {
+        migrateMemoriesTable(db);
+    }
+    catch { /* ignore migration errors */ }
+    // Create vector tables (separated so they run after the extension loads)
+    try {
+        db.exec(VEC_DDL);
+    }
+    catch {
+        // vec0 tables require the extension; skip silently if it wasn't loaded.
+    }
+    _db = db;
+    return db;
+}
+export function getDatabase() {
+    if (!_db) {
+        throw new Error('Database not initialized. Call initDatabase(dbPath) before using getDatabase().');
+    }
+    return _db;
+}
+// ---------------------------------------------------------------------------
+// Transaction helper
+// ---------------------------------------------------------------------------
+/**
+ * Run `fn` inside a SQLite transaction.
+ * Commits on success, rolls back and re-throws on any error.
+ */
+export function withTransaction(fn) {
+    const db = getDatabase();
+    db.exec('BEGIN');
+    try {
+        const result = fn();
+        db.exec('COMMIT');
+        return result;
+    }
+    catch (err) {
+        db.exec('ROLLBACK');
+        throw err;
+    }
+}
+// Allow tests to reset the singleton (e.g., open an in-memory DB)
+export function resetDatabase() {
+    if (_db) {
+        try {
+            _db.close();
+        }
+        catch {
+            // Ignore close errors during reset
+        }
+    }
+    _db = null;
+}
+//# sourceMappingURL=database.js.map

package/dist/storage/database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/storage/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAErD,qBAAqB;AACrB,IAAI,GAAG,GAAwB,IAAI,CAAC;AAEpC,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoKX,CAAC;AAEF,sDAAsD;AACtD,+CAA+C;AAC/C,MAAM,UAAU,GAA4B;IAC1C,CAAC,QAAQ,EAAE,uBAAuB,CAAC;IACnC,CAAC,aAAa,EAAE,MAAM,CAAC;IACvB,CAAC,aAAa,EAAE,MAAM,CAAC;IACvB,CAAC,cAAc,EAAE,MAAM,CAAC;IACxB,CAAC,YAAY,EAAE,MAAM,CAAC;IACtB,CAAC,aAAa,EAAE,MAAM,CAAC;IACvB,CAAC,eAAe,EAAE,MAAM,CAAC;IACzB,CAAC,mBAAmB,EAAE,MAAM,CAAC;IAC7B,CAAC,eAAe,EAAE,kBAAkB,CAAC;IACrC,CAAC,cAAc,EAAE,mBAAmB,CAAC;CACtC,CAAC;AAEF,SAAS,oBAAoB,CAAC,EAAgB;IAC5C,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC,GAAG,EAA6B,CAAC;IACxF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,EAAE,CAAC,IAAI,CAAC,mCAAmC,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,uEAAuE;IACvE,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9D,yDAAyD;IACzD,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACtC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACrC,EAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAExC,sEAAsE;IACtE,IAAI,CAAC;QACH,gBAAgB,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,yEAAyE;QACzE,kDAAkD;QAClD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,GAAG,IAAI,CAAC,CAAC;IAC9E,CAAC;IAED,2CAA2C;IAC3C,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEb,qDAAqD;IACrD,IAAI,CAAC;QAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,6BAA6B,CAAC,CAAC;IAEzE,yEAAyE;IACzE,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;IAC1E,CAAC;IAED,GAAG,GAAG,EAAE,CAAC;IACT,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAI,EAAW;IAC5C,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC;QACpB,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,aAAa;IAC3B,IAAI,GAAG,EAAE,CAAC;QACR,IAAI,CAAC;YACH,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;IACH,CAAC;IACD,GAAG,GAAG,IAAI,CAAC;AACb,CAAC"}

package/dist/storage/queries.d.ts

@@ -0,0 +1,85 @@
+import type { Memory, OrbitZone, OrbitChange, SunState, ConstellationEdge, MemoryConflict, MemoryAnalytics, ObservationEntry } from '../engine/types.js';
+import type { DataSource } from '../scanner/types.js';
+export declare function insertMemory(memory: Partial<Memory>): Memory;
+export declare function getMemoryById(id: string): Memory | null;
+export declare function getMemoryByIds(ids: string[]): Memory[];
+export declare function getMemoriesByProject(project: string, includeDeleted?: boolean): Memory[];
+/**
+ * Get memories created within the last `hoursAgo` hours for a project.
+ * Used by auto-commit on shutdown to summarize the current session.
+ */
+export declare function getRecentMemories(project: string, hoursAgo?: number): Memory[];
+export declare function getMemoriesInZone(project: string, zone: OrbitZone): Memory[];
+export declare function updateMemoryOrbit(id: string, distance: number, importance: number, velocity: number): void;
+export declare function updateMemoryAccess(id: string): void;
+export declare function softDeleteMemory(id: string): void;
+export declare function searchMemories(project: string, query: string, limit?: number): Memory[];
+export declare function searchMemoriesInRange(project: string, query: string, minDistance: number, maxDistance: number, limit: number): Memory[];
+export declare function getNearestMemories(project: string, limit: number): Memory[];
+export declare function getSunState(project: string): SunState | null;
+export declare function upsertSunState(state: Partial<SunState> & {
+    project: string;
+}): void;
+export declare function insertOrbitLog(change: OrbitChange): void;
+export declare function cleanupOrbitLog(retentionDays?: number): number;
+/**
+ * Check whether a memory already exists for the given source_path + source_hash.
+ * Returns true if an identical (path, hash) pair is already stored and not deleted.
+ */
+export declare function memoryExistsForSource(sourcePath: string, sourceHash: string): boolean;
+/**
+ * Find a memory by source_path (regardless of hash). Used to update stale entries.
+ */
+export declare function getMemoryBySourcePath(sourcePath: string): Memory | null;
+/**
+ * Find a non-deleted memory in the given project that has the same content hash.
+ * Used by createMemory() for content-level deduplication.
+ */
+export declare function getMemoryByContentHash(project: string, contentHash: string): Memory | null;
+export declare function insertDataSource(ds: Omit<DataSource, 'created_at' | 'updated_at'>): DataSource;
+export declare function updateDataSource(id: string, patch: Partial<Pick<DataSource, 'status' | 'last_scanned_at' | 'file_count' | 'total_size' | 'config'>>): void;
+export declare function getAllDataSources(): DataSource[];
+export declare function getDataSourceByPath(path: string): DataSource | null;
+export declare function createEdge(edge: ConstellationEdge): void;
+export declare function getEdges(memoryId: string, project: string): ConstellationEdge[];
+export declare function getConstellation(memoryId: string, project: string, depth?: number): {
+    nodes: Memory[];
+    edges: ConstellationEdge[];
+};
+/**
+ * Get all constellation neighbors for a batch of memory IDs.
+ * Returns a map from memory ID → set of neighbor IDs.
+ */
+export declare function getEdgesForBatch(memoryIds: string[], project: string): Map<string, Set<string>>;
+export declare function deleteEdge(id: string): void;
+export declare function createConflict(conflict: MemoryConflict): void;
+export declare function getConflicts(project: string, status?: string): MemoryConflict[];
+export declare function getConflictsForMemory(memoryId: string): MemoryConflict[];
+export declare function resolveConflict(id: string, resolution: string): void;
+export declare function createObservation(entry: ObservationEntry): void;
+export declare function getObservations(project: string, limit?: number): ObservationEntry[];
+export declare function getMemoriesAtTime(project: string, timestamp: string): Memory[];
+export declare function supersedMemory(memoryId: string, newMemoryId: string): void;
+export declare function getSupersessionChain(memoryId: string): Memory[];
+export declare function consolidateMemories(sourceIds: string[], targetId: string): void;
+export declare function getConsolidationHistory(memoryId: string): Memory[];
+export declare function getUniversalMemories(limit?: number): Memory[];
+export declare function setUniversal(memoryId: string, isUniversal: boolean): void;
+export declare function listProjects(): Array<{
+    project: string;
+    count: number;
+}>;
+export declare function updateQualityScore(memoryId: string, score: number): void;
+export declare function getMemoriesByQuality(project: string, minScore?: number, maxScore?: number): Memory[];
+export declare function getTopTags(project: string, limit?: number): Array<{
+    tag: string;
+    count: number;
+}>;
+export declare function getActivityTimeline(project: string, days?: number): Array<{
+    date: string;
+    created: number;
+    accessed: number;
+}>;
+export declare function getRecallSuccessRate(project: string): number;
+export declare function getAnalytics(project: string): MemoryAnalytics;
+//# sourceMappingURL=queries.d.ts.map