squish-memory 1.0.2 → 1.2.0

package/core/scheduler/job-runner.ts

@@ -0,0 +1,197 @@
+/** Job Runner - Maintenance job implementations */
+
+import { logger } from '../logger.js';
+import { type JobExecutionContext } from './cron-scheduler.js';
+import { runDeduplicationJob } from '../consolidation.js';
+import { runLifecycleMaintenance } from '../lifecycle.js';
+import { pruneWeakAssociations } from '../associations.js';
+import { pruneOldSummaries } from '../summarization.js';
+import { getDb } from '../../db/index.js';
+import { memories, memoryFeedback } from '../../db/drizzle/schema-sqlite.js';
+import { eq, and, gt, lt } from 'drizzle-orm';
+
+export async function runNightlyJob(context: JobExecutionContext): Promise<{
+  recordsProcessed: number;
+  summary: Record<string, unknown>;
+}> {
+  const summary: Record<string, unknown> = {};
+  let recordsProcessed = 0;
+
+  logger.info('[NightlyJob] Starting nightly maintenance');
+
+  if (context.config.applyDecay !== false || context.config.updateTiers !== false) {
+    try {
+      const lifecycleResult = await runLifecycleMaintenance();
+      summary.decayApplied = lifecycleResult?.decayed || 0;
+      // Simplified: hot/cold only (warm removed)
+      summary.tiersUpdated = (lifecycleResult?.tierChanges?.hot || 0) +
+                            (lifecycleResult?.tierChanges?.cold || 0);
+      recordsProcessed += summary.decayApplied as number;
+      recordsProcessed += summary.tiersUpdated as number;
+      logger.info(`[NightlyJob] Lifecycle: ${summary.decayApplied} decayed, ${summary.tiersUpdated} tier updates`);
+    } catch (error) {
+      logger.error('[NightlyJob] Lifecycle maintenance failed:', error);
+      summary.lifecycleError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (context.config.mergeDuplicates !== false) {
+    try {
+      const dedupResult = await runDeduplicationJob();
+      summary.duplicatesMerged = dedupResult?.mergedCount || 0;
+      recordsProcessed += summary.duplicatesMerged as number;
+      logger.info(`[NightlyJob] Merged ${summary.duplicatesMerged} duplicate memories`);
+    } catch (error) {
+      logger.error('[NightlyJob] Deduplication failed:', error);
+      summary.dedupError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (context.config.boostAccessed !== false) {
+    try {
+      const boostResult = await boostFrequentlyAccessed();
+      summary.memoriesBoosted = boostResult;
+      recordsProcessed += boostResult;
+      logger.info(`[NightlyJob] Boosted ${summary.memoriesBoosted} frequently accessed memories`);
+    } catch (error) {
+      logger.error('[NightlyJob] Boost failed:', error);
+      summary.boostError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  logger.info(`[NightlyJob] Completed: ${recordsProcessed} records processed`);
+
+  return { recordsProcessed, summary };
+}
+
+export async function runWeeklyJob(context: JobExecutionContext): Promise<{
+  recordsProcessed: number;
+  summary: Record<string, unknown>;
+}> {
+  const summary: Record<string, unknown> = {};
+  let recordsProcessed = 0;
+
+  logger.info('[WeeklyJob] Starting weekly maintenance');
+
+  if (context.config.archiveStale !== false) {
+    try {
+      const archiveResult = await archiveStaleMemories(90);
+      summary.memoriesArchived = archiveResult;
+      recordsProcessed += archiveResult;
+      logger.info(`[WeeklyJob] Archived ${summary.memoriesArchived} stale memories`);
+    } catch (error) {
+      logger.error('[WeeklyJob] Archive failed:', error);
+      summary.archiveError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (context.config.pruneAssociations !== false) {
+    try {
+      const pruneResult = await pruneWeakAssociations();
+      summary.associationsPruned = typeof pruneResult === 'number' ? pruneResult : 0;
+      recordsProcessed += summary.associationsPruned as number;
+      logger.info(`[WeeklyJob] Pruned ${summary.associationsPruned} weak associations`);
+    } catch (error) {
+      logger.error('[WeeklyJob] Association pruning failed:', error);
+      summary.pruneError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (context.config.pruneSummaries !== false) {
+    try {
+      const summaryPruneResult = await pruneOldSummaries(30);
+      summary.summariesPruned = typeof summaryPruneResult === 'number' ? summaryPruneResult : 0;
+      recordsProcessed += summary.summariesPruned as number;
+      logger.info(`[WeeklyJob] Pruned ${summary.summariesPruned} old summaries`);
+    } catch (error) {
+      logger.error('[WeeklyJob] Summary pruning failed:', error);
+      summary.summaryPruneError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (context.config.cleanupFeedback !== false) {
+    try {
+      const feedbackCleanupResult = await cleanupOldFeedbackRecords(30);
+      summary.feedbackRecordsCleaned = feedbackCleanupResult;
+      recordsProcessed += feedbackCleanupResult;
+      logger.info(`[WeeklyJob] Cleaned ${summary.feedbackRecordsCleaned} old feedback records`);
+    } catch (error) {
+      logger.error('[WeeklyJob] Feedback cleanup failed:', error);
+      summary.feedbackError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  logger.info(`[WeeklyJob] Completed: ${recordsProcessed} records processed`);
+
+  return { recordsProcessed, summary };
+}
+
+async function boostFrequentlyAccessed(): Promise<number> {
+  const db = await getDb();
+  if (!db) return 0;
+
+  const sqliteDb = db as any;
+  const frequentlyAccessed = await sqliteDb
+    .select()
+    .from(memories)
+    .where(gt(memories.accessCount, 3));
+
+  let boosted = 0;
+  for (const memory of frequentlyAccessed) {
+    const currentPriority = memory.retrievalPriority ?? 50;
+    const newPriority = Math.min(100, currentPriority + 5);
+
+    await sqliteDb
+      .update(memories)
+      .set({ retrievalPriority: newPriority })
+      .where(eq(memories.id, memory.id));
+
+    boosted++;
+  }
+
+  return boosted;
+}
+
+async function archiveStaleMemories(daysOld: number): Promise<number> {
+  const db = await getDb();
+  if (!db) return 0;
+
+  const staleThreshold = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);
+  const sqliteDb = db as any;
+
+  const staleMemories = await sqliteDb
+    .select()
+    .from(memories)
+    .where(and(
+      lt(memories.lastAccessedAt, staleThreshold),
+      lt(memories.importanceScore, 30),
+      eq(memories.isProtected, false),
+      eq(memories.isPinned, false)
+    ));
+
+  let archived = 0;
+  for (const memory of staleMemories) {
+    await sqliteDb
+      .update(memories)
+      .set({ tier: 'cold' })
+      .where(eq(memories.id, memory.id));
+
+    archived++;
+  }
+
+  return archived;
+}
+
+async function cleanupOldFeedbackRecords(daysOld: number): Promise<number> {
+  const db = await getDb();
+  if (!db) return 0;
+
+  const oldThreshold = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);
+  const sqliteDb = db as any;
+
+  await sqliteDb
+    .delete(memoryFeedback)
+    .where(lt(memoryFeedback.createdAt, oldThreshold));
+
+  return 0;
+}

package/core/search/conversations.ts

@@ -0,0 +1,166 @@
+import { and, desc, eq, gte, lte } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { config } from '../../config.js';
+import { requireProject } from '../../core/projects.js';
+import { deserializeMetadata } from '../../core/memory/serialization.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { normalizeTimestamp, clampLimit } from '../lib/utils.js';
+
+export interface ConversationRecord {
+  id: string;
+  projectId?: string | null;
+  sessionId: string;
+  title?: string | null;
+  summary?: string | null;
+  messageCount?: number | null;
+  tokenCount?: number | null;
+  startedAt?: string | null;
+  endedAt?: string | null;
+}
+
+export interface ConversationSearchInput {
+  query: string;
+  limit?: number;
+  role?: 'user' | 'assistant';
+}
+
+export interface RecentConversationsInput {
+  n?: number;
+  before?: string;
+  after?: string;
+  project?: string;
+}
+
+export async function searchConversations(input: ConversationSearchInput): Promise<ConversationRecord[]> {
+  const limit = clampLimit(input.limit, 5, 1, 50);
+  if (config.isTeamMode) {
+    return await searchConversationsPostgres(input, limit);
+  }
+  return await searchConversationsSqlite(input, limit);
+}
+
+export async function getRecentConversations(input: RecentConversationsInput): Promise<ConversationRecord[]> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+  const limit = clampLimit(input.n, 3, 1, 50);
+  const whereParts = [] as any[];
+
+   if (input.project) {
+     const project = await requireProject(input.project);
+     whereParts.push(eq(schema.conversations.projectId, project.id));
+   }
+
+  if (input.before) {
+    whereParts.push(lte(schema.conversations.startedAt, new Date(input.before)));
+  }
+
+  if (input.after) {
+    whereParts.push(gte(schema.conversations.startedAt, new Date(input.after)));
+  }
+
+  const query = whereParts.length
+    ? db.select().from(schema.conversations).where(and(...whereParts)).orderBy(desc(schema.conversations.startedAt)).limit(limit)
+    : db.select().from(schema.conversations).orderBy(desc(schema.conversations.startedAt)).limit(limit);
+
+  const rows = await query;
+  return rows.map((row: any) => normalizeConversation(row));
+}
+
+async function searchConversationsSqlite(input: ConversationSearchInput, limit: number): Promise<ConversationRecord[]> {
+  const db = createDatabaseClient(await getDb());
+  const sqlite = db.$client as any;
+  const params: Array<string | number> = [input.query];
+  let where = 'messages_fts MATCH ?';
+
+  if (input.role) {
+    where += ' AND m.role = ?';
+    params.push(input.role);
+  }
+
+  params.push(limit);
+
+  const statement = sqlite.prepare(`
+    SELECT DISTINCT
+      c.id as id,
+      c.project_id as projectId,
+      c.session_id as sessionId,
+      c.title as title,
+      c.summary as summary,
+      c.message_count as messageCount,
+      c.token_count as tokenCount,
+      c.started_at as startedAt,
+      c.ended_at as endedAt,
+      c.metadata as metadata
+    FROM conversations c
+    JOIN messages m ON m.conversation_id = c.id
+    JOIN messages_fts ON messages_fts.rowid = m.rowid
+    WHERE ${where}
+    ORDER BY c.started_at DESC
+    LIMIT ?
+  `);
+
+  const rows = statement.all(...params) as Array<{
+    id: string;
+    projectId: string | null;
+    sessionId: string;
+    title: string | null;
+    summary: string | null;
+    messageCount: number | null;
+    tokenCount: number | null;
+    startedAt: string | null;
+    endedAt: string | null;
+    metadata: string | null;
+  }>;
+  return rows.map((row: any) => normalizeConversation(row));
+}
+
+async function searchConversationsPostgres(input: ConversationSearchInput, limit: number): Promise<ConversationRecord[]> {
+  const db = createDatabaseClient(await getDb());
+  const values: Array<string | number> = [`%${input.query}%`];
+  const whereParts: string[] = [`m.content ILIKE $1`];
+
+  if (input.role) {
+    values.push(input.role);
+    whereParts.push(`m.role = $${values.length}`);
+  }
+
+  values.push(limit);
+
+  const rows = await (db.$client as any).query(
+    `SELECT DISTINCT ON (c.id)
+      c.id as "id",
+      c.project_id as "projectId",
+      c.session_id as "sessionId",
+      c.title as "title",
+      c.summary as "summary",
+      c.message_count as "messageCount",
+      c.token_count as "tokenCount",
+      c.started_at as "startedAt",
+      c.ended_at as "endedAt",
+      c.metadata as "metadata"
+    FROM conversations c
+    JOIN messages m ON m.conversation_id = c.id
+    WHERE ${whereParts.join(' AND ')}
+    ORDER BY c.id, c.started_at DESC
+    LIMIT $${values.length}`,
+    values
+  );
+
+  return rows.rows.map((row: any) => normalizeConversation(row));
+}
+
+function normalizeConversation(row: any): ConversationRecord {
+  const metadata = deserializeMetadata(row.metadata ?? null);
+  return {
+    id: row.id,
+    projectId: row.projectId ?? row.project_id ?? null,
+    sessionId: row.sessionId ?? row.session_id,
+    title: row.title ?? null,
+    summary: row.summary ?? null,
+    messageCount: row.messageCount ?? row.message_count ?? null,
+    tokenCount: row.tokenCount ?? row.token_count ?? null,
+    startedAt: normalizeTimestamp(row.startedAt ?? row.started_at),
+    endedAt: normalizeTimestamp(row.endedAt ?? row.ended_at),
+  };
+}

package/core/search/entities.ts

@@ -0,0 +1,46 @@
+import { desc, eq } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { config } from '../../config.js';
+import { requireProject } from '../../core/projects.js';
+import { deserializeMetadata } from '../../core/memory/serialization.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { normalizeTimestamp } from '../lib/utils.js';
+
+export interface EntityRecord {
+  id: string;
+  projectId?: string | null;
+  name: string;
+  type: string;
+  description?: string | null;
+  properties?: Record<string, unknown> | null;
+  createdAt?: string | null;
+  updatedAt?: string | null;
+}
+
+export async function getEntitiesForProject(projectPath: string, limit: number): Promise<EntityRecord[]> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+  const project = await requireProject(projectPath);
+
+  const rows = await db.select().from(schema.entities)
+    .where(eq(schema.entities.projectId, project.id))
+    .orderBy(desc(schema.entities.createdAt))
+    .limit(limit);
+
+  return rows.map((row: any) => normalizeEntity(row));
+}
+
+function normalizeEntity(row: any): EntityRecord {
+  const properties = deserializeMetadata(row.properties ?? null);
+  return {
+    id: row.id,
+    projectId: row.projectId ?? row.project_id ?? null,
+    name: row.name,
+    type: row.type,
+    description: row.description ?? null,
+    properties,
+    createdAt: normalizeTimestamp(row.createdAt ?? row.created_at),
+    updatedAt: normalizeTimestamp(row.updatedAt ?? row.updated_at),
+  };
+}

package/core/search/folder-context.ts

@@ -0,0 +1,154 @@
+/**
+ * Folder Context Generation
+ * Generates and maintains per-folder CLAUDE.md files with session summaries
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+import { getRecentConversations } from './conversations.js';
+import { logger } from '../../core/logger.js';
+
+interface FolderContextData {
+  projectPath: string;
+  projectName: string;
+  gitRemote?: string;
+  gitBranch?: string;
+  recentConversations: Array<{ content: string; timestamp: string }>;
+  keyObservations: Array<{ summary: string; timestamp: string; type: string }>;
+  lastUpdated: string;
+}
+
+function formatDate(dateStr: string): string {
+  return new Date(dateStr).toLocaleDateString('en-US', {
+    year: 'numeric',
+    month: '2-digit',
+    day: '2-digit'
+  });
+}
+
+function generateClaudeMdContent(data: FolderContextData): string {
+  const lines: string[] = [
+    `# Squish Context - ${data.projectName}`,
+    '',
+    '<squish-context>',
+    '',
+    `**Project**: ${data.projectPath}`
+  ];
+
+  if (data.gitRemote) {
+    const gitInfo = data.gitBranch ? `${data.gitRemote} @ ${data.gitBranch}` : data.gitRemote;
+    lines.push(`**Git**: ${gitInfo}`);
+  }
+
+  lines.push(`**Last Updated**: ${data.lastUpdated}`, '');
+
+  if (data.recentConversations.length > 0) {
+    lines.push('## Recent Activity');
+    for (const conv of data.recentConversations) {
+      const date = formatDate(conv.timestamp);
+      const preview = conv.content.substring(0, 80).replace(/\n/g, ' ');
+      lines.push(`- [${date}] ${preview}...`);
+    }
+    lines.push('');
+  }
+
+  if (data.keyObservations.length > 0) {
+    lines.push('## Key Observations');
+    for (const obs of data.keyObservations) {
+      lines.push(`- ${obs.summary} (${obs.type})`);
+    }
+    lines.push('');
+  }
+
+  lines.push(
+    '## Previous Context',
+    '*(Managed by Squish - do not edit this section)*',
+    '',
+    '</squish-context>',
+    '',
+    '<!-- You can add custom notes and project context here -->',
+    ''
+  );
+
+  return lines.join('\n');
+}
+
+async function collectFolderContextData(projectPath: string): Promise<FolderContextData> {
+  const recentConversations = await getRecentConversations({ n: 5, project: projectPath })
+    .catch(() => []);
+
+  return {
+    projectPath,
+    projectName: path.basename(projectPath),
+    recentConversations: (recentConversations || []).map((conv: any) => ({
+      content: conv?.summary || '',
+      timestamp: conv?.startedAt?.toISOString() || new Date().toISOString()
+    })),
+    keyObservations: [],
+    lastUpdated: new Date().toISOString()
+  };
+}
+
+async function preserveUserContent(claudeMdPath: string): Promise<string> {
+  try {
+    const existingContent = await fs.readFile(claudeMdPath, 'utf-8');
+    const match = existingContent.match(/<\/squish-context>([\s\S]*)/);
+    return match ? match[1] : '';
+  } catch {
+    return '';
+  }
+}
+
+export async function generateAndInjectFolderContext(projectPath: string): Promise<void> {
+  const claudeMdPath = path.join(projectPath, 'CLAUDE.md');
+  const contextData = await collectFolderContextData(projectPath);
+  const newContent = generateClaudeMdContent(contextData);
+  const userContent = await preserveUserContent(claudeMdPath);
+
+  await fs.writeFile(claudeMdPath, newContent + userContent, 'utf-8');
+  logger.info(`Updated folder context: ${claudeMdPath}`);
+}
+
+export async function readFolderContext(projectPath: string): Promise<FolderContextData | null> {
+  try {
+    const claudeMdPath = path.join(projectPath, 'CLAUDE.md');
+    const content = await fs.readFile(claudeMdPath, 'utf-8');
+
+    const contextMatch = content.match(/<squish-context>([\s\S]*?)<\/squish-context>/);
+    if (!contextMatch) return null;
+
+    const contextContent = contextMatch[1];
+
+    const projectPathMatch = contextContent.match(/\*\*Project\*\*:\s*(.+)/);
+    const gitMatch = contextContent.match(/\*\*Git\*\*:\s*(.+)/);
+    const updatedMatch = contextContent.match(/\*\*Last Updated\*\*:\s*(.+)/);
+
+    return {
+      projectPath: projectPathMatch?.[1] || projectPath,
+      projectName: path.basename(projectPath),
+      gitRemote: gitMatch?.[1].split(' @')[0],
+      gitBranch: gitMatch?.[1].split('@ ')[1],
+      recentConversations: [],
+      keyObservations: [],
+      lastUpdated: updatedMatch?.[1] || new Date().toISOString()
+    };
+  } catch {
+    return null;
+  }
+}
+
+export async function injectFolderContextIntoSession(projectPath: string): Promise<string> {
+  try {
+    const claudeMdPath = path.join(projectPath, 'CLAUDE.md');
+    const content = await fs.readFile(claudeMdPath, 'utf-8');
+    const match = content.match(/<squish-context>([\s\S]*?)<\/squish-context>/);
+
+    if (match) {
+      return `\n<squish-folder-context>\n${match[1]}\n</squish-folder-context>\n`;
+    }
+  } catch {
+    // File doesn't exist or can't be read
+  }
+
+  return '';
+}

package/core/search/graph-boost.ts

@@ -0,0 +1,22 @@
+import { getDb } from '../../db/index.js';
+import { memoryAssociations } from '../../db/drizzle/schema.js';
+import { inArray } from 'drizzle-orm';
+
+/**
+ * Compute a simple graph boost for a set of memory IDs.
+ * For each memory, sum (weight * coactivationCount) of outgoing associations.
+ * Returns a map from memoryId to boost value (default 0).
+ */
+export async function computeGraphBoost(memoryIds: string[]): Promise<Record<string, number>> {
+  const boost: Record<string, number> = {};
+  if (memoryIds.length === 0) return boost;
+  const db = await getDb();
+  // @ts-ignore - drizzle overload
+  const rows = await db.select().from(memoryAssociations).where(inArray(memoryAssociations.fromMemoryId, memoryIds));
+  for (const row of rows) {
+    const from = row.fromMemoryId as string;
+    const val = (row.weight ?? 0) * (row.coactivationCount ?? 0);
+    boost[from] = (boost[from] ?? 0) + val;
+  }
+  return boost;
+}

package/core/search/index.ts

@@ -0,0 +1,4 @@
+// Search feature - Search and discovery
+export * from './conversations.js';
+export * from './entities.js';
+export * from './folder-context.js';

package/core/search/qmd-wrapper.ts

@@ -0,0 +1,84 @@
+import { getQMDClient, type QMDSearchResult } from '../embeddings/qmd-client.js';
+
+export type { QMDSearchResult };
+
+export interface QMDStatus {
+  indexHealth: string;
+  documentCount: number;
+  collections?: Array<{
+    name: string;
+    path: string;
+    documentCount: number;
+  }>;
+}
+
+export async function isAvailable(): Promise<boolean> {
+  const client = await getQMDClient();
+  return client.isAvailable();
+}
+
+export async function search(
+  query: string,
+  options: { collection?: string; limit?: number; minScore?: number } = {}
+): Promise<QMDSearchResult[]> {
+  const client = await getQMDClient();
+  if (!(await client.isAvailable())) return [];
+  return client.search({ query, ...options });
+}
+
+export async function vsearch(
+  query: string,
+  options: { collection?: string; limit?: number; minScore?: number } = {}
+): Promise<QMDSearchResult[]> {
+  const client = await getQMDClient();
+  if (!(await client.isAvailable())) return [];
+  return client.vsearch({ query, ...options });
+}
+
+export async function query(
+  queryText: string,
+  options: { collection?: string; limit?: number; minScore?: number } = {}
+): Promise<QMDSearchResult[]> {
+  const client = await getQMDClient();
+  if (!(await client.isAvailable())) return [];
+  return client.query({ query: queryText, ...options });
+}
+
+export async function get(
+  pathOrDocid: string,
+  options: { full?: boolean; maxBytes?: number } = {}
+): Promise<string> {
+  const client = await getQMDClient();
+  if (!(await client.isAvailable())) return '';
+  return client.get({ pathOrDocid, ...options });
+}
+
+export async function status(): Promise<QMDStatus | null> {
+  const client = await getQMDClient();
+  if (!(await client.isAvailable())) return null;
+  const result = await client.status();
+  if (!result) return null;
+
+  return {
+    indexHealth: result.indexHealth,
+    documentCount: result.collections.reduce((sum, collection) => sum + collection.documentCount, 0),
+    collections: result.collections,
+  };
+}
+
+export async function embed(): Promise<number[] | null> {
+  return null;
+}
+
+export async function ensureWikiDir(projectPath: string): Promise<string> {
+  const { join } = await import('path');
+  const { mkdirSync, existsSync } = await import('fs');
+  const wikiDir = join(projectPath, '.squish', 'wiki');
+  if (!existsSync(wikiDir)) mkdirSync(wikiDir, { recursive: true });
+  return wikiDir;
+}
+
+export async function close(): Promise<void> {
+  const client = await getQMDClient();
+  await client.disconnect();
+}