squish-memory 1.0.2 → 1.2.0

package/db/migrations/tier-conversion.ts

@@ -0,0 +1,24 @@
+/**
+ * Tier conversion migration
+ * Converts deprecated 'warm' tier memories to 'cold'
+ */
+
+import type { Database } from 'better-sqlite3';
+import { logger } from '../../core/logger.js';
+
+export async function runTierConversion(sqlite: Database): Promise<void> {
+  const warmMemories = sqlite.prepare(
+    "SELECT COUNT(*) as count FROM memories WHERE tier = 'warm'"
+  ).get() as { count: number } | undefined;
+
+  if (warmMemories && warmMemories.count > 0) {
+    logger.info(`Migration: Converting ${warmMemories.count} warm-tiered memories to cold`);
+    try {
+      sqlite.exec("UPDATE memories SET tier = 'cold' WHERE tier = 'warm'");
+      logger.info('Migration: Tier conversion complete (warm -> cold)');
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      logger.warn(`Migration: Could not convert warm tier: ${msg}`);
+    }
+  }
+}

package/db/neon.ts

@@ -0,0 +1,22 @@
+import { config } from '../config.js';
+import { neon } from '@neondatabase/serverless';
+import { drizzle } from 'drizzle-orm/neon-http';
+
+/**
+ * Neon client wrapper that returns a Drizzle HTTP client.
+ * Neon HTTP driver is faster for single non-interactive transactions.
+ */
+export async function createNeonClient() {
+  if (!config.neonProjectId || !config.neonServiceKey) {
+    throw new Error('Neon configuration missing (NEON_PROJECT_ID or NEON_SERVICE_KEY)');
+  }
+
+  // Neon HTTP driver - use DATABASE_URL with Neon connection string
+  const connectionString = process.env.DATABASE_URL;
+  if (!connectionString) {
+    throw new Error('DATABASE_URL required for Neon connection');
+  }
+
+  const sql = neon(connectionString);
+  return drizzle(sql);
+}

package/db/schema/beliefs.ts

@@ -0,0 +1,50 @@
+/**
+ * Beliefs table schema for migrations
+ * Column definitions for ALTER TABLE migrations
+ */
+
+import type { TableSchema } from '../generator.js';
+
+export const beliefsSchema: TableSchema = {
+  name: 'beliefs',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    project_id: { type: 'TEXT' },
+    belief_type: { type: 'TEXT' },
+    statement: { type: 'TEXT' },
+    normalized_key: { type: 'TEXT' },
+    confidence: { type: 'REAL', default: '0.5' },
+    belief_decay_rate: { type: 'INTEGER', default: '30' },
+    last_confirmed_at: { type: 'INTEGER' },
+    source_count: { type: 'INTEGER', default: '1' },
+    status: { type: 'TEXT', default: "'active'" },
+    reason: { type: 'TEXT' },
+    context: { type: 'TEXT' },
+    evidence_summary: { type: 'TEXT' },
+    metadata: { type: 'TEXT' },
+    created_at: { type: 'INTEGER' },
+    updated_at: { type: 'INTEGER' },
+  },
+};
+
+export const beliefMemorySourcesSchema: TableSchema = {
+  name: 'belief_memory_sources',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    belief_id: { type: 'TEXT' },
+    memory_id: { type: 'TEXT' },
+    created_at: { type: 'INTEGER' },
+  },
+};
+
+export const beliefEdgesSchema: TableSchema = {
+  name: 'belief_edges',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    from_belief_id: { type: 'TEXT' },
+    to_belief_id: { type: 'TEXT' },
+    edge_type: { type: 'TEXT' },
+    metadata: { type: 'TEXT' },
+    created_at: { type: 'INTEGER' },
+  },
+};

package/db/schema/generator.ts

@@ -0,0 +1,159 @@
+/**
+ * Migration Generator - Creates migrations from schema definitions
+ * 
+ * This utility generates ALTER TABLE statements from schema column definitions,
+ * eliminating the need for inline column arrays in migration files.
+ */
+
+import type { Database } from 'better-sqlite3';
+import { logger } from '../../core/logger.js';
+
+export interface ColumnDefinition {
+  type: string;
+  primary?: boolean;
+  references?: string;
+  default?: string;
+  notNull?: boolean;
+}
+
+export interface IndexDefinition {
+  name: string;
+  columns: string[];
+  using?: string;
+  unique?: boolean;
+}
+
+export interface TableSchema {
+  name: string;
+  columns: Record<string, ColumnDefinition>;
+  indexes?: IndexDefinition[];
+}
+
+/**
+ * Get existing columns for a table
+ */
+function getExistingColumns(sqlite: Database, tableName: string): Set<string> {
+  try {
+    const tableInfo = sqlite.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name: string }>;
+    return new Set(tableInfo.map(col => col.name));
+  } catch {
+    return new Set();
+  }
+}
+
+/**
+ * Get existing indexes for a table
+ */
+function getExistingIndexes(sqlite: Database, tableName: string): Set<string> {
+  try {
+    const indexes = sqlite.prepare(
+      "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name=?"
+    ).all(tableName) as Array<{ name: string }>;
+    return new Set(indexes.map(idx => idx.name));
+  } catch {
+    return new Set();
+  }
+}
+
+/**
+ * Generate and run column migrations for a table schema
+ */
+export async function migrateTable(
+  sqlite: Database,
+  schema: TableSchema
+): Promise<void> {
+  const existingColumns = getExistingColumns(sqlite, schema.name);
+
+  for (const [columnName, definition] of Object.entries(schema.columns)) {
+    if (!existingColumns.has(columnName)) {
+      const sql = generateAlterTable(schema.name, columnName, definition);
+      try {
+        sqlite.exec(sql);
+        logger.info(`Migration: Added column ${columnName} to ${schema.name}`);
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        if (msg.includes('duplicate column name')) {
+          logger.debug(`Migration skipped for ${schema.name}.${columnName}: already exists`);
+        } else {
+          throw new Error(`Migration failed for ${schema.name}.${columnName}: ${msg}`);
+        }
+      }
+    }
+  }
+
+  // Handle indexes
+  if (schema.indexes) {
+    await migrateIndexes(sqlite, schema.name, schema.indexes);
+  }
+}
+
+/**
+ * Generate and run index migrations
+ */
+export async function migrateIndexes(
+  sqlite: Database,
+  tableName: string,
+  indexes: IndexDefinition[]
+): Promise<void> {
+  const existingIndexes = getExistingIndexes(sqlite, tableName);
+
+  for (const indexDef of indexes) {
+    if (!existingIndexes.has(indexDef.name)) {
+      const sql = generateCreateIndex(tableName, indexDef);
+      try {
+        sqlite.exec(sql);
+        logger.info(`Migration: Added index ${indexDef.name}`);
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        logger.warn(`Index migration failed for ${indexDef.name}: ${msg}`);
+      }
+    }
+  }
+}
+
+/**
+ * Generate ALTER TABLE ADD COLUMN statement
+ */
+function generateAlterTable(
+  tableName: string,
+  columnName: string,
+  definition: ColumnDefinition
+): string {
+  let sql = `ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${definition.type}`;
+
+  if (definition.default) {
+    sql += ` DEFAULT ${definition.default}`;
+  }
+
+  return sql;
+}
+
+/**
+ * Generate CREATE INDEX statement
+ */
+function generateCreateIndex(
+  tableName: string,
+  indexDef: IndexDefinition
+): string {
+  const columns = indexDef.columns.join(', ');
+  const unique = indexDef.unique ? 'UNIQUE ' : '';
+  return `CREATE ${unique}INDEX IF NOT EXISTS ${indexDef.name} ON ${tableName}(${columns})`;
+}
+
+/**
+ * Run all table migrations from schema list
+ */
+export async function runAllSchemaMigrations(
+  sqlite: Database,
+  schemas: TableSchema[]
+): Promise<void> {
+  for (const schema of schemas) {
+    const tableCheck = sqlite.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name=?"
+    ).get(schema.name) as { name: string } | undefined;
+
+    if (tableCheck) {
+      await migrateTable(sqlite, schema);
+    }
+  }
+}

package/db/schema/index.ts

@@ -0,0 +1,58 @@
+/**
+ * Schema Index - Table schema definitions
+ * 
+ * DEPRECATED: This module was previously used to aggregate table schemas.
+ * Table definitions are now in db/bootstrap.ts for first-time setup
+ * and db/schema/*.ts for migration column definitions.
+ * 
+ * These exports remain for backward compatibility but return empty arrays.
+ */
+
+import type { TableSchema } from './generator.js';
+
+// Re-export migration schemas for compatibility
+export { memoriesSchema } from './memories.js';
+export { learningsSchema } from './learnings.js';
+export { beliefsSchema, beliefMemorySourcesSchema, beliefEdgesSchema } from './beliefs.js';
+
+/**
+ * Table schemas for migration utilities
+ * @deprecated Use db/schema/memories.ts, db/schema/learnings.ts, db/schema/beliefs.ts directly
+ */
+export const tableSchemas: TableSchema[] = [];
+
+/**
+ * Generate complete SQLite schema SQL
+ * @deprecated Table definitions moved to db/bootstrap.ts
+ */
+export function generateSqliteSchema(): string {
+  console.warn('DEPRECATED: generateSqliteSchema() - table definitions are in db/bootstrap.ts');
+  return '';
+}
+
+/**
+ * Generate complete PostgreSQL schema SQL
+ * @deprecated Table definitions moved to db/bootstrap.ts
+ */
+export function generatePostgresSchema(): string {
+  console.warn('DEPRECATED: generatePostgresSchema() - table definitions are in db/bootstrap.ts');
+  return '';
+}
+
+/**
+ * Get all SQLite table definitions in order
+ * @deprecated Table definitions moved to db/bootstrap.ts
+ */
+export function getAllSqliteTables(): string[] {
+  console.warn('DEPRECATED: getAllSqliteTables() - table definitions are in db/bootstrap.ts');
+  return [];
+}
+
+/**
+ * Get all PostgreSQL table definitions in order
+ * @deprecated Table definitions moved to db/bootstrap.ts
+ */
+export function getAllPostgresTables(): string[] {
+  console.warn('DEPRECATED: getAllPostgresTables() - table definitions are in db/bootstrap.ts');
+  return [];
+}

package/db/schema/learnings.ts

@@ -0,0 +1,32 @@
+/**
+ * Learnings table schema for migrations
+ * Column definitions for ALTER TABLE migrations
+ */
+
+import type { TableSchema } from '../generator.js';
+
+export const learningsSchema: TableSchema = {
+  name: 'learnings',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    project_id: { type: 'TEXT' },
+    type: { type: 'TEXT' },
+    content: { type: 'TEXT' },
+    summary: { type: 'TEXT' },
+    source: { type: 'TEXT' },
+    confidence: { type: 'INTEGER', default: '50' },
+    confidence_level: { type: 'TEXT', default: "'speculative'" },
+    tags: { type: 'TEXT' },
+    metadata: { type: 'TEXT' },
+    embedding: { type: 'BLOB' },
+    folder_path: { type: 'TEXT' },
+    project_path: { type: 'TEXT' },
+    is_private: { type: 'INTEGER', default: '0' },
+    has_secrets: { type: 'INTEGER', default: '0' },
+    relevance_score: { type: 'INTEGER', default: '50' },
+    memory_id: { type: 'TEXT' },
+    is_imported: { type: 'INTEGER', default: '0' },
+    created_at: { type: 'INTEGER' },
+    updated_at: { type: 'INTEGER' },
+  },
+};

package/db/schema/memories.ts

@@ -0,0 +1,83 @@
+/**
+ * Memories table schema for migrations
+ * Column definitions for ALTER TABLE migrations
+ */
+
+import type { TableSchema } from '../generator.js';
+
+export const memoriesSchema: TableSchema = {
+  name: 'memories',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    project_id: { type: 'TEXT' },
+    user_id: { type: 'TEXT' },
+    type: { type: 'TEXT' },
+    content: { type: 'TEXT' },
+    summary: { type: 'TEXT' },
+    embedding_json: { type: 'TEXT' },
+    embedding: { type: 'BLOB' },
+    source: { type: 'TEXT' },
+    confidence: { type: 'INTEGER', default: '50' },
+    confidence_level: { type: 'TEXT', default: "'speculative'" },
+    tags: { type: 'TEXT' },
+    metadata: { type: 'TEXT' },
+    is_private: { type: 'INTEGER', default: '0' },
+    has_secrets: { type: 'INTEGER', default: '0' },
+    relevance_score: { type: 'INTEGER', default: '50' },
+    is_active: { type: 'INTEGER', default: '1' },
+    expires_at: { type: 'INTEGER' },
+    access_count: { type: 'INTEGER', default: '0' },
+    last_accessed_at: { type: 'INTEGER' },
+    is_merged: { type: 'INTEGER', default: '0' },
+    merged_into_id: { type: 'TEXT' },
+    merged_at: { type: 'INTEGER' },
+    is_canonical: { type: 'INTEGER', default: '0' },
+    merge_source_ids: { type: 'TEXT' },
+    is_mergeable: { type: 'INTEGER', default: '1' },
+    merge_version: { type: 'INTEGER', default: '1' },
+    namespace_id: { type: 'TEXT' },
+    namespace_path: { type: 'TEXT' },
+    has_l0_abstract: { type: 'INTEGER', default: '0' },
+    has_l1_overview: { type: 'INTEGER', default: '0' },
+    last_layer_update: { type: 'INTEGER' },
+    importance_score: { type: 'INTEGER', default: '50' },
+    importance_decay_rate: { type: 'INTEGER', default: '30' },
+    last_importance_recalc: { type: 'INTEGER' },
+    retrieval_priority: { type: 'INTEGER', default: '50' },
+    tokens_estimate: { type: 'INTEGER', default: '0' },
+    consolidated_into: { type: 'TEXT' },
+    consolidated_at: { type: 'INTEGER' },
+    is_consolidated: { type: 'INTEGER', default: '0' },
+    sector: { type: 'TEXT', default: "'episodic'" },
+    tier: { type: 'TEXT', default: "'hot'" },
+    status: { type: 'TEXT', default: "'active'" },
+    encrypted_content: { type: 'TEXT' },
+    encryption_nonce: { type: 'TEXT' },
+    is_encrypted: { type: 'INTEGER', default: '0' },
+    context_status: { type: 'TEXT', default: "'out-of-context'" },
+    decay_rate: { type: 'INTEGER', default: '30' },
+    coactivation_score: { type: 'INTEGER', default: '0' },
+    last_decay_at: { type: 'INTEGER' },
+    agent_id: { type: 'TEXT' },
+    agent_role: { type: 'TEXT' },
+    visibility_scope: { type: 'TEXT', default: "'private'" },
+    is_protected: { type: 'INTEGER', default: '0' },
+    is_pinned: { type: 'INTEGER', default: '0' },
+    is_immutable: { type: 'INTEGER', default: '0' },
+    write_scope: { type: 'TEXT' },
+    read_scope: { type: 'TEXT' },
+    triggered_by: { type: 'TEXT' },
+    capture_reason: { type: 'TEXT' },
+    last_used_at: { type: 'INTEGER' },
+    usage_count: { type: 'INTEGER', default: '0' },
+    valid_from: { type: 'INTEGER' },
+    valid_to: { type: 'INTEGER' },
+    recorded_at: { type: 'INTEGER' },
+    superseded_by: { type: 'TEXT' },
+    version: { type: 'INTEGER', default: '1' },
+    place_id: { type: 'TEXT' },
+    place_sort_order: { type: 'INTEGER', default: '0' },
+    created_at: { type: 'INTEGER' },
+    updated_at: { type: 'INTEGER' },
+  },
+};

package/db/schema/projects.ts

@@ -0,0 +1,33 @@
+import type { TableSchema } from '../generator.js';
+
+export const projectsSchema: TableSchema = {
+  name: 'projects',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    name: { type: 'TEXT' },
+    path: { type: 'TEXT' },
+    description: { type: 'TEXT' },
+    metadata: { type: 'TEXT' },
+    created_at: { type: 'INTEGER' },
+    updated_at: { type: 'INTEGER' },
+  },
+  indexes: [
+    {
+      name: 'projects_path_idx',
+      columns: ['path'],
+    },
+  ],
+};
+
+export const usersSchema: TableSchema = {
+  name: 'users',
+  columns: {
+    id: { type: 'TEXT', primary: true },
+    external_id: { type: 'TEXT' },
+    name: { type: 'TEXT' },
+    email: { type: 'TEXT' },
+    preferences: { type: 'TEXT' },
+    created_at: { type: 'INTEGER' },
+    updated_at: { type: 'INTEGER' },
+  },
+};

package/db/schema.ts

@@ -0,0 +1,13 @@
+import { config } from '../config.js';
+
+export type SchemaModule = typeof import('./drizzle/schema.js') | typeof import('./drizzle/schema-sqlite.js');
+
+let cachedSchema: SchemaModule | null = null;
+
+export async function getSchema(): Promise<SchemaModule> {
+  if (cachedSchema) return cachedSchema;
+  cachedSchema = config.isTeamMode
+    ? await import('./drizzle/schema.js')
+    : await import('./drizzle/schema-sqlite.js');
+  return cachedSchema;
+}

package/db/supabase.ts

@@ -0,0 +1,27 @@
+import { config } from '../config.js';
+import { drizzle } from 'drizzle-orm/node-postgres';
+
+/**
+ * Supabase client wrapper that returns a Drizzle PostgreSQL client.
+ * Supabase uses the standard Postgres wire protocol, so we can reuse the same
+ * drizzle-node-postgres driver as the regular Postgres backend.
+ */
+export async function createSupabaseClient() {
+  if (!config.supabaseUrl || !config.supabaseKey) {
+    throw new Error('Supabase configuration missing (SUPABASE_URL or SUPABASE_SERVICE_KEY)');
+  }
+
+  if (!process.env.DATABASE_URL) {
+    throw new Error('DATABASE_URL is required for Supabase connection');
+  }
+
+  const { Pool } = await import('pg');
+  const pool = new Pool({
+    connectionString: process.env.DATABASE_URL,
+    max: 20,
+  });
+
+  // Dynamically import the schema module to avoid circular dependencies.
+  const schemaModule = await import('./drizzle/schema.js');
+  return drizzle(pool, { schema: schemaModule });
+}

package/dist/config.d.ts

@@ -1,23 +1,47 @@
 export declare function getDataDir(): string;
 export declare const config: {
+    mode: "local" | "team" | "remote";
+    isLocalMode: boolean;
     isTeamMode: boolean;
+    isRemoteMode: boolean;
+    teamBackend: "postgres" | "supabase" | "neon";
+    remoteBackend: "supabase" | "neon";
     isManagedMode: boolean;
+    managedMode: boolean;
+    managedApiUrl: string;
+    managedApiKey: string;
     redisEnabled: boolean;
     dataDir: string;
     mcpServerPort: number;
-    mcpServerEnabled: boolean;
-    embeddingsProvider: "local" | "openai" | "ollama" | "google" | "none" | "auto";
+    embeddingsProvider: "openai" | "ollama" | "lmstudio" | "transformers" | "local" | "none" | "google" | "auto";
     openAiApiKey: string;
-    openAiApiUrl: any;
-    openAiEmbeddingModel: any;
+    openAiApiUrl: string;
+    openAiEmbeddingModel: string;
     googleCloudApiKey: string;
     googleCloudProject: string;
     googleCloudLocation: string;
-    googleEmbeddingModel: any;
-    ollamaUrl: any;
-    ollamaEmbeddingModel: any;
+    googleEmbeddingModel: string;
+    ollamaUrl: string;
+    ollamaEmbeddingModel: string;
+    lmStudioUrl: string;
+    lmStudioEmbeddingModel: string;
+    transformersLocalModel: string;
+    supabaseUrl: string;
+    supabaseKey: string;
+    neonProjectId: string;
+    neonServiceKey: string;
+    clientEncryptionEnabled: boolean;
+    encryptionPassphrase: string;
     lifecycleEnabled: boolean;
     lifecycleInterval: number;
+    decayThreshold: number;
+    sectorDecayIntervals: {
+        episodic: number;
+        semantic: number;
+        procedural: number;
+        autobiographical: number;
+        working: number;
+    };
     summarizationEnabled: boolean;
     incrementalThreshold: number;
     rollingWindowSize: number;
@@ -26,13 +50,10 @@ export declare const config: {
     governanceEnabled: boolean;
     consolidationEnabled: boolean;
     consolidationSimilarityThreshold: number;
-    qmdEnabled: boolean;
-    qmdCollectionsPath: string;
-    qmdFallbackMode: "qmd-only" | "cloud-first" | "hybrid" | "local-only";
-    qmdCollectionMapping: any;
-    managedMode: boolean;
-    managedApiUrl: string;
-    managedApiKey: string;
+    obsidianEnabled: boolean;
+    obsidianVaultPath: string;
+    externalMemoryEnabled: boolean;
+    externalMemoryPath: string;
     sessionAutoLoadEnabled: boolean;
     sessionAutoLoadRecentCount: number;
     sessionAutoLoadImportanceThreshold: number;
@@ -42,6 +63,13 @@ export declare const config: {
     feedbackTrackingEnabled: boolean;
     feedbackEchoBonus: number;
     feedbackFizzlePenalty: number;
+    scoringWeights: {
+        recency: number;
+        relevance: number;
+        importance: number;
+        vectorSim: number;
+        graphBoost: number;
+    };
     schedulerMode: "cron" | "interval" | "heartbeat";
     cronEnabled: boolean;
     heartbeatInterval: number;
@@ -54,6 +82,25 @@ export declare const config: {
     openAiTimeoutMs: number;
     ollamaTimeoutMs: number;
     googleTimeoutMs: number;
+    llmEnabled: boolean;
+    llmApiKey: string;
+    llmProvider: "openai" | "ollama" | "lmstudio" | "local" | "anthropic";
+    llmExtractionModel: string;
+    llmReasoningModel: string;
+    llmEndpoint: string;
+    llm: {
+        enabled: boolean;
+        provider: "openai" | "ollama" | "lmstudio" | "local" | "anthropic";
+        endpoint: string;
+    };
+    graphAutoBuild: boolean;
+    graphExtractionMethod: "auto" | "llm" | "regex";
+    graphMaxContentLength: number;
+    graph: {
+        autoBuild: boolean;
+        extractionMethod: "auto" | "llm" | "regex";
+        maxContentLength: number;
+    };
 };
 export default config;
 //# sourceMappingURL=config.d.ts.map