squish-memory 1.0.2 → 1.2.0

package/core/integrations/obsidian-vault.ts

@@ -0,0 +1,197 @@
+/** Obsidian Vault Integration
+ * 
+ * Appends hot memories to Obsidian daily notes
+ * Format: YYYY-MM-DD.md with memory blocks
+ */
+
+import { existsSync, mkdirSync, appendFileSync, readFileSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+import { logger } from '../logger.js';
+
+export interface ObsidianMemoryInput {
+  content: string;
+  id: string;
+  type: string;
+  tags: string[];
+  reasoning?: string;
+  memoryContext?: string;
+  examples?: string;
+  exceptions?: string;
+  source?: string;
+}
+
+/**
+ * Append a memory to the daily note in Obsidian vault
+ * Creates YYYY-MM-DD.md file if it doesn't exist
+ */
+export async function appendToObsidianVault(
+  memory: ObsidianMemoryInput,
+  vaultPath: string
+): Promise<void> {
+  const today = new Date();
+  const year = today.getFullYear();
+  const month = String(today.getMonth() + 1).padStart(2, '0');
+  const day = String(today.getDate()).padStart(2, '0');
+  const dateStr = `${year}-${month}-${day}`;
+  
+  const dailyNotePath = join(vaultPath, `${dateStr}.md`);
+  
+  // Ensure vault directory exists
+  if (!existsSync(vaultPath)) {
+    mkdirSync(vaultPath, { recursive: true });
+    logger.info(`[Obsidian] Created vault directory: ${vaultPath}`);
+  }
+  
+  // Build memory block content
+  const memoryBlock = formatMemoryAsBlock(memory);
+  
+  // Check if daily note exists, if not create with header
+  if (!existsSync(dailyNotePath)) {
+    const header = `# ${dateStr}\n\n## Squish Memories\n\n`;
+    appendFileSync(dailyNotePath, header, 'utf-8');
+    logger.info(`[Obsidian] Created daily note: ${dailyNotePath}`);
+  }
+  
+  // Append memory block
+  appendFileSync(dailyNotePath, memoryBlock + '\n\n', 'utf-8');
+  logger.info(`[Obsidian] Appended memory to ${dateStr}.md`);
+}
+
+/**
+ * Format memory as Obsidian-compatible block
+ */
+function formatMemoryAsBlock(memory: ObsidianMemoryInput): string {
+  const lines: string[] = [];
+  
+  // Memory content as checkbox (can be toggled in Obsidian)
+  lines.push(`- [ ] "${memory.content}"`);
+  
+  // Add ID reference for linking
+  lines.push(`  - **id:** \`${memory.id}\``);
+  
+  // Type and tags
+  if (memory.type) {
+    lines.push(`  - **type:** ${memory.type}`);
+  }
+  if (memory.tags && memory.tags.length > 0) {
+    const tagList = memory.tags.map(t => `#${t.replace(/\s+/g, '-')}`).join(' ');
+    lines.push(`  - **tags:** ${tagList}`);
+  }
+  
+  // Rich context fields
+  if (memory.source) {
+    lines.push(`  - **source:** ${memory.source}`);
+  }
+  if (memory.reasoning) {
+    lines.push(`  - **reasoning:** ${memory.reasoning}`);
+  }
+  if (memory.memoryContext) {
+    lines.push(`  - **context:** ${memory.memoryContext}`);
+  }
+  if (memory.examples) {
+    lines.push(`  - **examples:** ${memory.examples}`);
+  }
+  if (memory.exceptions) {
+    lines.push(`  - **exceptions:** ${memory.exceptions}`);
+  }
+  
+  return lines.join('\n');
+}
+
+/**
+ * Check if Obsidian vault is connected and accessible
+ */
+export function isObsidianConnected(vaultPath: string): boolean {
+  if (!vaultPath || vaultPath === '') {
+    return false;
+  }
+  
+  // Check if path exists or can be created
+  const parentDir = vaultPath.includes('/') 
+    ? vaultPath.split('/').slice(0, -1).join('/')
+    : vaultPath.split('\\').slice(0, -1).join('\\');
+    
+  return existsSync(parentDir) || true; // Assume creatable if parent exists
+}
+
+/**
+ * Read existing memories from Obsidian vault (for import feature)
+ * Scans vault for .md files and extracts memory blocks
+ */
+export async function readFromVault(vaultPath: string): Promise<ObsidianMemoryInput[]> {
+  if (!existsSync(vaultPath)) {
+    logger.warn(`[Obsidian] Vault path does not exist: ${vaultPath}`);
+    return [];
+  }
+
+  const memories: ObsidianMemoryInput[] = [];
+  
+  // Recursively find all .md files
+  const mdFiles = await findMarkdownFiles(vaultPath);
+  
+  for (const filePath of mdFiles) {
+    try {
+      const content = readFileSync(filePath, 'utf-8');
+      const filename = filePath.split(/[/\\]/).pop() || '';
+      
+      // Extract frontmatter if present
+      const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
+      let tags: string[] = [];
+      
+      if (frontmatterMatch) {
+        const tagMatch = frontmatterMatch[1].match(/tags:\s*\[([^\]]*)\]/);
+        if (tagMatch) {
+          tags = tagMatch[1].split(',').map((t: string) => t.trim()).filter(Boolean);
+        }
+      }
+      
+      // Extract content after frontmatter
+      const body = frontmatterMatch ? content.slice(frontmatterMatch[0].length).trim() : content;
+      
+      // Skip empty files or very short ones
+      if (body.length < 10) continue;
+      
+      memories.push({
+        id: randomUUID(),
+        content: body,
+        tags,
+        type: 'observation'
+      });
+    } catch (error) {
+      logger.warn(`[Obsidian] Failed to read file ${filePath}: ${error}`);
+    }
+  }
+  
+  logger.info(`[Obsidian] Found ${memories.length} memories in vault`);
+  return memories;
+}
+
+/**
+ * Recursively find all markdown files in a directory
+ */
+async function findMarkdownFiles(dir: string): Promise<string[]> {
+  const files: string[] = [];
+  
+  try {
+    const entries = readdirSync(dir, { withFileTypes: true });
+    
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      
+      if (entry.isDirectory()) {
+        // Skip hidden directories
+        if (!entry.name.startsWith('.')) {
+          const subFiles = await findMarkdownFiles(fullPath);
+          files.push(...subFiles);
+        }
+      } else if (entry.isFile() && entry.name.endsWith('.md')) {
+        files.push(fullPath);
+      }
+    }
+  } catch (error) {
+    logger.warn(`[Obsidian] Failed to read directory ${dir}: ${error}`);
+  }
+  
+  return files;
+}

package/core/layers/generator.ts

@@ -0,0 +1,115 @@
+/** Layer Extractor - Creates L0/L1 truncated layers for token-efficient hierarchical retrieval */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+
+export interface LayerGenerationConfig {
+  l0MaxLength?: number;     // Default: 50 chars (truncated preview)
+  l1MaxLength?: number;     // Default: 300 chars (first 2 sentences truncated)
+  autoGenerate?: boolean;    // Default: true for content > 500 chars
+}
+
+export interface LayerGenerationResult {
+  l0?: string;  // Ultra-short truncated preview
+  l1?: string;  // Short truncated preview (first 2 sentences)
+  l2?: string;  // Full content (reference)
+  tokenCounts: { l0: number; l1: number; l2: number };
+}
+
+const DEFAULT_CONFIG: LayerGenerationConfig = {
+  l0MaxLength: 50,
+  l1MaxLength: 300,
+  autoGenerate: true,
+};
+
+/**
+ * Extract L0/L1/L2 layers for content via truncation
+ */
+export async function extractLayers(
+  content: string,
+  config: LayerGenerationConfig = {}
+): Promise<LayerGenerationResult> {
+  const l0MaxLength = config.l0MaxLength ?? DEFAULT_CONFIG.l0MaxLength;
+  const l1MaxLength = config.l1MaxLength ?? DEFAULT_CONFIG.l1MaxLength;
+
+  // Extract L0 and L1 in parallel
+  const [l0, l1] = await Promise.all([
+    extractL0Truncation(content, l0MaxLength!),
+    extractL1Truncation(content, l1MaxLength!),
+  ]);
+
+  const l2Tokens = countTokens(content);
+  const l0Tokens = countTokens(l0 || '');
+  const l1Tokens = countTokens(l1 || '');
+
+  return {
+    l0: l0 ?? '',
+    l1: l1 ?? '',
+    l2: content,
+    tokenCounts: {
+      l0: l0Tokens,
+      l1: l1Tokens,
+      l2: l2Tokens,
+    },
+  };
+}
+
+/**
+ * Extract L0: Ultra-short truncated preview (first sentence or chars)
+ */
+async function extractL0Truncation(
+  content: string,
+  maxLength: number
+): Promise<string> {
+  // Simple truncation for L0 (first sentence or chars)
+  const trimmed = content.trim();
+  const sentences = trimmed.split(/[.!?]/).filter(s => s.trim().length > 0);
+
+  // Take first non-empty sentence and truncate if needed
+  let result = sentences.length > 0 ? sentences[0] : trimmed.substring(0, maxLength);
+
+  if (result.length > maxLength) {
+    result = result.substring(0, maxLength - 3) + '...';
+  }
+
+  return result.trim();
+}
+
+/**
+ * Extract L1: Short truncated preview (first 2 meaningful sentences)
+ */
+async function extractL1Truncation(
+  content: string,
+  maxLength: number
+): Promise<string> {
+  // Simple truncation for L1 (first 2 meaningful sentences)
+  const trimmed = content.trim();
+  const sentences = trimmed.split(/[.!?]/).filter(s => s.trim().length > 10);
+
+  // Take first meaningful sentences and truncate if needed
+  let result = sentences.length > 0
+    ? sentences.slice(0, 2).join(' ').trim()
+    : trimmed.substring(0, maxLength);
+
+  if (result.length > maxLength) {
+    result = result.substring(0, maxLength - 3) + '...';
+  }
+
+  return result.trim();
+}
+
+/**
+ * Count tokens (approximate, 4 chars = 1 token for English)
+ */
+function countTokens(text: string): number {
+  // Simple approximation: 4 chars per token
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Check if content should have layers auto-extracted
+ */
+export function shouldAutoGenerate(contentLength: number): boolean {
+  const threshold = 500;
+  return contentLength > threshold;
+}

package/core/lib/db-client.ts

@@ -0,0 +1,168 @@
+/**
+ * Database Client & Schema Loading Abstraction
+ *
+ * Provides a unified interface for accessing both the database client and schema
+ * in a single call, eliminating repetitive patterns of:
+ *   `const db = createDatabaseClient(await getDb()); const schema = await getSchema();`
+ *
+ * Features:
+ * - Single async call to get both db and schema
+ * - Consistent error handling with clear messages
+ * - Preserves schema caching
+ * - Provides raw connection for special cases
+ * - Helper `withDbClient` for functional programming patterns
+ *
+ * @example
+ * ```typescript
+ * // Old pattern (repetitive)
+ * const db = createDatabaseClient(await getDb());
+ * const schema = await getSchema();
+ *
+ * // New pattern (unified)
+ * const { db, schema } = await getDbClient();
+ *
+ * // With helper
+ * const result = await withDbClient(async (client) => {
+ *   return await client.db.select().from(client.schema.memories);
+ * });
+ * ```
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { createDatabaseClient } from '../storage/database.js';
+import type { DatabaseClient } from '../storage/database.js';
+import type { SchemaModule } from '../../db/schema.js';
+
+/**
+ * Unified database client interface combining:
+ * - `db`: Wrapped database client with select/insert/update/delete methods
+ * - `schema`: Drizzle schema module with table definitions
+ * - `raw`: Raw underlying connection for special cases (mcp-server, index, etc.)
+ */
+export interface DbClient {
+  /**
+   * Wrapped database client with unified API
+   * Provides select(), insert(), update(), delete() methods
+   */
+  db: DatabaseClient;
+
+  /**
+   * Schema module containing table definitions
+   * Access tables via schema.tableName (e.g., schema.memories, schema.users)
+   */
+  schema: SchemaModule;
+
+  /**
+   * Raw underlying database connection
+   * Use for special cases that require direct access to the native driver
+   * (e.g., raw SQL queries, specific driver features)
+   */
+  raw: any;
+}
+
+/**
+ * Internal cached schema to preserve getSchema() caching behavior
+ */
+let cachedSchema: SchemaModule | null = null;
+
+/**
+ * Get a unified database client with both db and schema.
+ *
+ * This function:
+ * 1. Calls getDb() to initialize/retrieve the database connection
+ * 2. Calls getSchema() to get the appropriate schema (with caching)
+ * 3. Wraps the db with createDatabaseClient() for unified API
+ * 4. Returns all three as a single DbClient object
+ *
+ * Error handling:
+ * - Wraps any database initialization errors with a clear message
+ * - Preserves the original error as `cause` property
+ *
+ * Schema caching:
+ * - Uses the same caching mechanism as getSchema()
+ * - Multiple calls to getDbClient() will return the same schema reference
+ *
+ * @returns Promise<DbClient> Unified client with db, schema, and raw
+ * @throws {Error} When database initialization fails, with cause attached
+ *
+ * @example
+ * ```typescript
+ * const { db, schema, raw } = await getDbClient();
+ *
+ * // Use db for queries
+ * const memories = await db.select().from(schema.memories).limit(10);
+ *
+ * // Use raw for special cases
+ * const client = (raw as any).$client;
+ * if (client && typeof client.exec === 'function') {
+ *   client.exec('VACUUM');
+ * }
+ * ```
+ */
+export async function getDbClient(): Promise<DbClient> {
+  try {
+    // Get raw database connection
+    const rawDb = await getDb();
+
+    // Get schema (with caching)
+    if (!cachedSchema) {
+      cachedSchema = await getSchema();
+    }
+
+    // Create wrapped database client
+    const db = createDatabaseClient(rawDb);
+
+    return {
+      db,
+      schema: cachedSchema,
+      raw: rawDb,
+    };
+  } catch (error) {
+    const message =
+      error instanceof Error
+        ? `Database initialization failed: ${error.message}`
+        : 'Database initialization failed';
+
+    // Wrap error with clear message while preserving cause
+    const wrappedError = new Error(message);
+    // @ts-ignore - TypeScript doesn't allow 'cause' on Error, but it's valid
+    wrappedError.cause = error;
+
+    throw wrappedError;
+  }
+}
+
+/**
+ * Execute an operation with a unified database client.
+ *
+ * Convenience wrapper that:
+ * 1. Calls getDbClient() to obtain client
+ * 2. Executes the provided operation
+ * 3. Ensures proper cleanup if needed (no-op currently, for future extension)
+ *
+ * @param operation - Async function that receives the DbClient and returns a result
+ * @returns Promise<T> The result returned by the operation
+ * @throws {Error} When database initialization or operation fails
+ *
+ * @example
+ * ```typescript
+ * const count = await withDbClient(async (client) => {
+ *   const result = await client.db
+ *     .select({ count: sql`count(*)` })
+ *     .from(client.schema.memories);
+ *   return result[0].count;
+ * });
+ * ```
+ */
+export async function withDbClient<T>(
+  operation: (client: DbClient) => Promise<T>
+): Promise<T> {
+  const client = await getDbClient();
+  try {
+    return await operation(client);
+  } finally {
+    // Future: Add cleanup logic if needed (connection pooling, etc.)
+    // Currently getDb() handles connection pooling and reuse
+  }
+}

package/core/lib/parse-embedding.ts

@@ -0,0 +1,59 @@
+/**
+ * Parse embedding from various storage formats.
+ * Converted from 4 locations: hybrid-search.ts, consolidation.ts, memories.ts, entity-deduplicator.ts
+ * 
+ * @param embeddingData - Input in Float32Array, Buffer, Uint8Array, JSON string, or Array format
+ * @returns Array of numbers or null if parsing fails
+ */
+export function parseEmbedding(embeddingData: unknown): number[] | null {
+  if (!embeddingData) return null;
+
+  // Handle Array directly (most common case)
+  if (Array.isArray(embeddingData)) {
+    return embeddingData;
+  }
+
+  // Handle Float32Array directly
+  if (embeddingData instanceof Float32Array) {
+    return Array.from(embeddingData);
+  }
+
+  // Handle Uint8Array and Buffer - try JSON first, then binary Float32
+  if (embeddingData instanceof Uint8Array || Buffer.isBuffer(embeddingData)) {
+    // Try to parse as JSON string first
+    try {
+      const json = JSON.parse(embeddingData.toString());
+      if (Array.isArray(json)) {
+        return json;
+      }
+    } catch {
+      // Not JSON, try binary Float32
+    }
+
+    // Try to parse as binary Float32Array
+    try {
+      const buffer = embeddingData.buffer;
+      const arrayBuffer = buffer instanceof ArrayBuffer
+        ? buffer
+        : (buffer as unknown as ArrayBuffer);
+      const floatArray = new Float32Array(arrayBuffer);
+      return Array.from(floatArray);
+    } catch {
+      return null;
+    }
+  }
+
+  // Handle JSON string
+  if (typeof embeddingData === 'string') {
+    try {
+      const parsed = JSON.parse(embeddingData);
+      if (Array.isArray(parsed)) {
+        return parsed;
+      }
+    } catch {
+      return null;
+    }
+  }
+
+  return null;
+}

package/core/lib/schemas.ts

@@ -0,0 +1,102 @@
+/**
+ * Shared Zod schemas for input validation across Squish
+ * Provides consistent validation for common input types
+ */
+
+import { z } from 'zod';
+
+/**
+ * Limit schema: integer between 1 and 100, default 20
+ */
+export const limitSchema = z.number().int().min(1).max(100).default(20);
+
+/**
+ * Offset schema: integer >= 0, default 0
+ */
+export const offsetSchema = z.number().int().min(0).default(0);
+
+/**
+ * Project ID/path schema: non-empty string
+ */
+export const projectIdSchema = z.string().min(1);
+
+/**
+ * Memory ID schema: valid UUID
+ */
+export const memoryIdSchema = z.string().uuid();
+
+/**
+ * Pagination schema combining limit and offset
+ */
+export const paginationSchema = z.object({
+  limit: limitSchema,
+  offset: offsetSchema,
+});
+
+/**
+ * Search query schema: non-empty string
+ */
+export const searchQuerySchema = z.string().min(1).trim();
+
+/**
+ * Memory type schema
+ */
+export const memoryTypeSchema = z.enum(['observation', 'fact', 'decision', 'context', 'preference', 'note']);
+
+/**
+ * Association type schema
+ */
+export const associationTypeSchema = z.enum(['relates_to', 'supports', 'contradicts', 'supersedes', 'duplicate']);
+
+/**
+ * Weight schema: number between 0 and 1
+ */
+export const weightSchema = z.number().min(0).max(1);
+
+/**
+ * Learning type schema
+ */
+export const learningTypeSchema = z.enum(['success', 'failure', 'fix', 'observation']);
+
+/**
+ * Observation type schema
+ */
+export const observationTypeSchema = z.enum(['tool_use', 'file_change', 'error', 'pattern', 'insight']);
+
+/**
+ * Confidence level schema
+ */
+export const confidenceLevelSchema = z.enum(['certain', 'speculative', 'outdated']);
+
+/**
+ * Search input schema (common for search operations)
+ */
+export const searchInputSchema = z.object({
+  query: searchQuerySchema,
+  limit: limitSchema.optional(),
+  offset: offsetSchema.optional(),
+  project: projectIdSchema.optional(),
+  type: memoryTypeSchema.optional(),
+});
+
+/**
+ * Memory recall schema
+ */
+export const recallInputSchema = z.object({
+  memoryId: memoryIdSchema,
+  limit: limitSchema.optional(),
+  project: projectIdSchema.optional(),
+});
+
+/**
+ * Tag operation schema
+ */
+export const tagOperationSchema = z.object({
+  action: z.enum(['add', 'remove']),
+  tag: z.string().min(1),
+  search: z.string().optional(),
+  olderThan: z.string().optional(),
+  type: memoryTypeSchema.optional(),
+  limit: limitSchema.optional(),
+  project: projectIdSchema.optional(),
+});

package/core/lib/types.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared type definitions for Squish Memory system
+ * Consolidated from core/memory/memories.ts and core/memory/normalization.ts
+ */
+
+// Valid memory type values
+export type MemoryType = 'observation' | 'fact' | 'decision' | 'context' | 'preference' | 'note' | 'task';
+
+/**
+ * Confidence level for memory records
+ * - certain: High confidence, verified information
+ * - speculative: Low confidence, unverified or uncertain
+ * - outdated: Information that may no longer be accurate
+ */
+export type ConfidenceLevel = 'certain' | 'speculative' | 'outdated';
+
+/**
+ * Unified MemoryRecord interface
+ * Used across memories.ts and normalization.ts
+ *
+ * Note: type is string to avoid circular dependency with memories.ts
+ * Use MemoryType for type checking when needed
+ */
+export interface MemoryRecord {
+  id: string;
+  projectId?: string | null;
+  type: string; // Stored as string in DB (use MemoryType for valid values)
+  content: string;
+  summary?: string | null;
+  tags: string[];
+  metadata?: Record<string, unknown> | null;
+  createdAt?: string | null;
+  validFrom?: string | null;
+  validTo?: string | null;
+  recordedAt?: string | null;
+  similarity?: number;
+  importance?: number;
+  confidenceLevel?: ConfidenceLevel | null;
+}
+
+/**
+ * Result of a safety check
+ * Used to determine if a merge action can proceed
+ */
+export interface SafetyCheckResult {
+  passed: boolean;
+  warnings: string[];
+  blockers: string[]; // Hard failures
+}