pi-hermes-memory 0.7.22 → 0.8.0

package/src/store/sqlite-memory-store.ts

@@ -74,6 +74,12 @@ export interface SqliteMemoryRemoveOptions {
   project?: string | null;
 }
 
+export interface MarkdownMemoryReconcileResult {
+  inserted: number;
+  existing: number;
+  removed: number;
+}
+
 export interface ParsedMarkdownMemoryEntry extends SqliteMemorySyncInput {}
 
 function today(): string {
@@ -179,13 +185,18 @@ function escapeLikePattern(text: string): string {
   return text.replace(/[\\%_]/g, '\\$&');
 }
 
-function parseMetadataComment(raw: string): { text: string; created: string; lastReferenced: string } {
-  const match = raw.match(/^(.*?)\s*<!--\s*created=([^,]+),\s*last=([^>]+)\s*-->\s*$/);
+function parseMetadataComment(raw: string): { text: string; created: string; lastReferenced: string; project: string | null } {
+  const match = raw.match(/^(.*?)\s*<!--\s*created=([^,]+),\s*last=([^,>]+)(?:,\s*project64=([A-Za-z0-9_-]+))?\s*-->\s*$/);
   if (match) {
+    let project: string | null = null;
+    if (match[4]) {
+      try { project = Buffer.from(match[4], 'base64url').toString('utf-8').trim() || null; } catch {}
+    }
     return {
       text: match[1].trim(),
       created: match[2].trim(),
       lastReferenced: match[3].trim(),
+      project,
     };
   }
 
@@ -194,6 +205,7 @@ function parseMetadataComment(raw: string): { text: string; created: string; las
     text: raw.trim(),
     created: fallback,
     lastReferenced: fallback,
+    project: null,
   };
 }
 
@@ -251,7 +263,6 @@ export function formatFailureMemoryContent(
   if (options.failureReason) parts.push(`Failed: ${options.failureReason}`);
   if (options.toolState) parts.push(`Tool state: ${options.toolState}`);
   if (options.correctedTo) parts.push(`Corrected to: ${options.correctedTo}`);
-  if (options.project) parts.push(`Project: ${options.project}`);
   return parts.join(' — ');
 }
 
@@ -265,7 +276,8 @@ export function parseMarkdownMemoryEntry(
   target: 'memory' | 'user' | 'failure',
   project: string | null = null,
 ): ParsedMarkdownMemoryEntry {
-  const { text, created, lastReferenced } = parseMetadataComment(rawEntry);
+  const metadata = parseMetadataComment(rawEntry);
+  const { text, created, lastReferenced } = metadata;
   const parsedProject = normalizeNullable(project);
 
   if (target !== 'failure') {
@@ -403,6 +415,111 @@ export function syncMemoryEntry(
   };
 }
 
+/**
+ * Make one exact Markdown target/project scope authoritative in SQLite.
+ * Upserts and orphan deletion are committed together when transactions are
+ * supported by the active SQLite driver.
+ */
+export function reconcileMarkdownMemoryScope(
+  dbManager: DatabaseManager,
+  rawEntries: string[],
+  target: 'memory' | 'user' | 'failure',
+  project: string | null = null,
+): MarkdownMemoryReconcileResult {
+  const db = dbManager.getDb();
+  const normalizedProject = normalizeNullable(project);
+
+  const reconcile = (): MarkdownMemoryReconcileResult => {
+    let inserted = 0;
+    let existing = 0;
+    const desiredIdentities = new Set<string>();
+
+    for (const rawEntry of rawEntries) {
+      const parsed = parseMarkdownMemoryEntry(rawEntry, target, normalizedProject);
+      desiredIdentities.add(JSON.stringify([
+        normalizeCategory(parsed.category),
+        parsed.content.trim(),
+      ]));
+      const result = syncMemoryEntry(dbManager, parsed);
+      if (result.action === 'inserted') inserted++;
+      else existing++;
+    }
+
+    const params: unknown[] = [];
+    const conditions = buildScopeConditions(params, target, normalizedProject);
+    const scopedRows = db.prepare(`
+      SELECT id, content, category
+      FROM memories
+      WHERE ${conditions.join(' AND ')}
+      ORDER BY id ASC
+    `).all(...params) as Array<{ id: number; content: string; category: MemoryCategory | null }>;
+    const retainedIdentities = new Set<string>();
+    const orphanIds: number[] = [];
+    for (const row of scopedRows) {
+      const identity = JSON.stringify([normalizeCategory(row.category), row.content.trim()]);
+      if (!desiredIdentities.has(identity) || retainedIdentities.has(identity)) {
+        orphanIds.push(row.id);
+      } else {
+        retainedIdentities.add(identity);
+      }
+    }
+
+    let removed = 0;
+    if (orphanIds.length > 0) {
+      const placeholders = orphanIds.map(() => '?').join(', ');
+      removed = db.prepare(`DELETE FROM memories WHERE id IN (${placeholders})`).run(...orphanIds).changes;
+    }
+
+    return { inserted, existing, removed };
+  };
+
+  const transactional = db.transaction?.(reconcile);
+  return transactional ? transactional() : reconcile();
+}
+
+function failureProject(rawEntry: string): string | null {
+  return parseMetadataComment(rawEntry).project;
+}
+
+export function reconcileMarkdownFailureScopes(
+  dbManager: DatabaseManager,
+  rawEntries: string[],
+): MarkdownMemoryReconcileResult {
+  const entriesByProject = new Map<string | null, string[]>();
+  for (const rawEntry of rawEntries) {
+    const project = failureProject(rawEntry);
+    const entries = entriesByProject.get(project) ?? [];
+    entries.push(rawEntry);
+    entriesByProject.set(project, entries);
+  }
+
+  const mirroredProjects = dbManager.getDb().prepare(`
+    SELECT DISTINCT project
+    FROM memories
+    WHERE target = 'failure'
+  `).all() as Array<{ project: string | null }>;
+  const projects = new Set<string | null>([
+    null,
+    ...entriesByProject.keys(),
+    ...mirroredProjects.map(({ project }) => normalizeNullable(project)),
+  ]);
+  const total: MarkdownMemoryReconcileResult = { inserted: 0, existing: 0, removed: 0 };
+
+  for (const project of projects) {
+    const result = reconcileMarkdownMemoryScope(
+      dbManager,
+      entriesByProject.get(project) ?? [],
+      'failure',
+      project,
+    );
+    total.inserted += result.inserted;
+    total.existing += result.existing;
+    total.removed += result.removed;
+  }
+
+  return total;
+}
+
 /**
  * Best-effort substring replacement for SQLite-backed memory sync.
  * Updates all matches in the scoped slice to recover from prior duplicate rows.

package/src/tools/memory-tool.ts

@@ -11,6 +11,8 @@ import { MemoryStore } from "../store/memory-store.js";
 import { DatabaseManager } from "../store/db.js";
 import {
   formatFailureMemoryContent,
+  reconcileMarkdownFailureScopes,
+  reconcileMarkdownMemoryScope,
   removeExactSyncedMemories,
   removeSyncedMemories,
   replaceSyncedMemories,
@@ -185,6 +187,31 @@ async function syncEvictionsFromSqlite(
   }
 }
 
+async function reconcileStoreScope(
+  entries: string[],
+  rawTarget: "memory" | "user" | "project" | "failure",
+  dbManager: DatabaseManager | null,
+  projectName?: string | null,
+): Promise<string | null | undefined> {
+  if (!dbManager) return undefined;
+  try {
+    if (rawTarget === "failure") {
+      reconcileMarkdownFailureScopes(dbManager, entries);
+      return null;
+    }
+    const target = sqliteTargetFor(rawTarget);
+    reconcileMarkdownMemoryScope(
+      dbManager,
+      entries,
+      target,
+      sqliteProjectFor(rawTarget, projectName) ?? null,
+    );
+    return null;
+  } catch (err) {
+    return `Saved to Markdown, but SQLite search reconciliation failed: ${err instanceof Error ? err.message : String(err)}`;
+  }
+}
+
 export function registerMemoryTool(
   pi: ExtensionAPI,
   store: MemoryStore,
@@ -192,6 +219,16 @@ export function registerMemoryTool(
   dbManager: DatabaseManager | null = null,
   projectName?: string | null,
 ): void {
+  const reconciledStores = new WeakSet<MemoryStore>();
+  if (typeof store.setMutationObserver === "function") {
+    store.setMutationObserver((target, entries) => reconcileStoreScope(entries, target, dbManager, projectName));
+    reconciledStores.add(store);
+  }
+  if (projectStore && typeof projectStore.setMutationObserver === "function") {
+    projectStore.setMutationObserver((_target, entries) => reconcileStoreScope(entries, "project", dbManager, projectName));
+    reconciledStores.add(projectStore);
+  }
+
   pi.registerTool({
     name: "memory",
     label: "Memory",
@@ -244,6 +281,7 @@ export function registerMemoryTool(
 
       let result: MemoryResult;
       let syncWarning: string | null = null;
+      const syncHandled = reconciledStores.has(store_);
       switch (action) {
         case "add":
           if (!content) {
@@ -267,12 +305,12 @@ export function registerMemoryTool(
               category: memoryCategory,
               failureReason: failure_reason,
             });
-            if (result.success) {
+            if (result.success && !syncHandled) {
               syncWarning = await syncAddToSqlite(rawTarget, content, memoryCategory, failure_reason, dbManager, projectName);
             }
           } else {
-            result = await store_.add(target, content);
-            if (result.success) {
+            result = await store_.add(target, content, signal);
+            if (result.success && !syncHandled) {
               await syncEvictionsFromSqlite(rawTarget, result.evicted_entries, dbManager, projectName);
               syncWarning = await syncAddToSqlite(rawTarget, content, undefined, undefined, dbManager, projectName);
             }
@@ -309,9 +347,7 @@ export function registerMemoryTool(
             };
           }
           result = await store_.replace(target, old_text, content);
-          if (result.success) {
-            syncWarning = await syncReplaceToSqlite(rawTarget, old_text, content, dbManager, projectName);
-          }
+          if (result.success && !syncHandled) syncWarning = await syncReplaceToSqlite(rawTarget, old_text, content, dbManager, projectName);
           break;
 
         case "remove":
@@ -330,9 +366,7 @@ export function registerMemoryTool(
             };
           }
           result = await store_.remove(target, old_text);
-          if (result.success) {
-            syncWarning = await syncRemoveFromSqlite(rawTarget, old_text, dbManager, projectName);
-          }
+          if (result.success && !syncHandled) syncWarning = await syncRemoveFromSqlite(rawTarget, old_text, dbManager, projectName);
           break;
 
         default:
@@ -342,6 +376,11 @@ export function registerMemoryTool(
           };
       }
 
+      if (result.success && !syncHandled && typeof store_.getRawEntriesForSync === "function") {
+        const reconciliationWarning = await reconcileStoreScope(store_.getRawEntriesForSync(target), rawTarget, dbManager, projectName);
+        if (reconciliationWarning !== undefined) syncWarning = reconciliationWarning;
+      }
+
       if (syncWarning && result.success) {
         result = appendSyncWarning(result, syncWarning);
       }

package/src/tools/session-search-tool.ts

@@ -14,6 +14,10 @@ interface SearchResult {
   count?: number;
   message?: string;
   output?: string;
+  outputChars?: number;
+  outputTruncated?: boolean;
+  snippetChars?: number;
+  truncatedCount?: number;
   ranges?: SessionAnchorRange[];
 }
 
@@ -22,6 +26,26 @@ interface SessionSearchToolOptions {
 }
 
 const DEFAULT_SESSIONS_DIR = path.join(AGENT_ROOT, 'sessions');
+const DEFAULT_LEGACY_SNIPPET_CHARS = 1_200;
+const MAX_LEGACY_SNIPPET_CHARS = 4_000;
+const MAX_LEGACY_OUTPUT_CHARS = 50 * 1024;
+
+function truncateLegacySnippet(text: string, maxChars: number): { text: string; truncated: boolean } {
+  if (text.length <= maxChars) return { text, truncated: false };
+  return {
+    text: `${text.slice(0, maxChars)}\n... (truncated, ${text.length} chars total — refine the query or increase snippetChars)`,
+    truncated: true,
+  };
+}
+
+function capLegacyOutput(text: string): { text: string; truncated: boolean } {
+  if (text.length <= MAX_LEGACY_OUTPUT_CHARS) return { text, truncated: false };
+  const suffix = `\n... (output truncated, ${text.length} chars total — refine the query or lower the result limit)`;
+  return {
+    text: `${text.slice(0, MAX_LEGACY_OUTPUT_CHARS - suffix.length)}${suffix}`,
+    truncated: true,
+  };
+}
 
 export function registerSessionSearchTool(
   pi: ExtensionAPI,
@@ -127,7 +151,7 @@ Examples:
 - "Find the PR where we fixed the test hang"
 - "What approach did we take for the database migration?"
 
-Returns conversation snippets with session dates and project context.`,
+Returns bounded conversation snippets with session dates and project context. Large messages are truncated with their original character count.`,
     promptSnippet: 'Search past conversations for relevant context',
     promptGuidelines: [
       'Use session_search when the user asks about previous discussions or past work.',
@@ -137,13 +161,27 @@ Returns conversation snippets with session dates and project context.`,
       query: Type.String({ description: 'Search query. Use natural language or specific terms.' }),
       project: Type.Optional(Type.String({ description: 'Filter by project name (optional).' })),
       role: Type.Optional(StringEnum(['user', 'assistant'] as const, { description: 'Filter by message role (optional).' })),
-      limit: Type.Optional(Type.Number({ description: 'Maximum results to return (default: 10, max: 20).' })),
+      limit: Type.Optional(Type.Number({
+        description: 'Maximum results to return (default: 10, min: 1, max: 20).',
+        minimum: 1,
+        maximum: 20,
+      })),
+      snippetChars: Type.Optional(Type.Number({
+        description: `Maximum characters per result snippet (default: ${DEFAULT_LEGACY_SNIPPET_CHARS}, max: ${MAX_LEGACY_SNIPPET_CHARS}).`,
+        minimum: 100,
+        maximum: MAX_LEGACY_SNIPPET_CHARS,
+      })),
     }),
-    execute: async (_id: string, args: { query: string; project?: string; role?: string; limit?: number }) => {
+    execute: async (_id: string, args: { query: string; project?: string; role?: string; limit?: number; snippetChars?: number }) => {
       const query = args.query;
       const project = args.project;
       const role = args.role;
-      const limit = Math.min(args.limit || 10, 20);
+      const requestedLimit = Number.isFinite(args.limit) ? Math.floor(args.limit!) : 10;
+      const limit = Math.min(Math.max(requestedLimit, 1), 20);
+      const requestedSnippetChars = Number.isFinite(args.snippetChars)
+        ? Math.floor(args.snippetChars!)
+        : DEFAULT_LEGACY_SNIPPET_CHARS;
+      const snippetChars = Math.min(Math.max(requestedSnippetChars, 100), MAX_LEGACY_SNIPPET_CHARS);
 
       if (!query || query.trim().length === 0) {
         const result: SearchResult = { success: false, message: 'query is required' };
@@ -159,11 +197,19 @@ Returns conversation snippets with session dates and project context.`,
       const results = searchSessions(dbManager, query, { project, role, limit });
 
       if (results.length === 0) {
-        const result: SearchResult = { success: true, count: 0, message: `No results found for "${query}". Try a different search term or broader query.` };
-        return { content: [{ type: 'text' as const, text: result.message! }], details: result };
+        const output = capLegacyOutput('No results found. Try a different search term or broader query.');
+        const result: SearchResult = {
+          success: true,
+          count: 0,
+          message: output.text,
+          outputChars: output.text.length,
+          outputTruncated: output.truncated,
+        };
+        return { content: [{ type: 'text' as const, text: output.text }], details: result };
       }
 
-      let output = `Found ${results.length} results for "${query}":\n\n`;
+      const blocks: string[] = [`Found ${results.length} results for "${query}":`];
+      let truncatedCount = 0;
 
       for (const r of results) {
         const date = new Date(r.timestamp).toLocaleDateString('en-US', {
@@ -172,13 +218,25 @@ Returns conversation snippets with session dates and project context.`,
           day: 'numeric',
         });
 
-        output += `---\n`;
-        output += `📅 ${date} | 📁 ${r.project} | ${r.role === 'user' ? '👤 User' : '🤖 Assistant'}\n`;
-        output += `${r.snippet}\n\n`;
+        const snippet = truncateLegacySnippet(r.snippet, snippetChars);
+        if (snippet.truncated) truncatedCount += 1;
+        blocks.push([
+          '---',
+          `📅 ${date} | 📁 ${r.project} | ${r.role === 'user' ? '👤 User' : '🤖 Assistant'}`,
+          snippet.text,
+        ].join('\n'));
       }
 
-      const finalResult: SearchResult = { success: true, count: results.length, output: output.trim() };
-      return { content: [{ type: 'text' as const, text: output.trim() }], details: finalResult };
+      const output = capLegacyOutput(blocks.join('\n\n').trim());
+      const finalResult: SearchResult = {
+        success: true,
+        count: results.length,
+        truncatedCount,
+        snippetChars,
+        outputChars: output.text.length,
+        outputTruncated: output.truncated,
+      };
+      return { content: [{ type: 'text' as const, text: output.text }], details: finalResult };
     },
   });
 }

package/src/types.ts

@@ -10,6 +10,8 @@ export type SessionSearchVariant = "legacy" | "anchors";
 
 export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh";
 
+export type ReviewTransport = "direct" | "subprocess";
+
 export interface SessionSearchConfig {
   /** Session search implementation variant. Default: legacy */
   variant: SessionSearchVariant;
@@ -34,6 +36,8 @@ export interface MemoryConfig {
   reviewRecentMessages?: number;
   /** Enable background learning loop. Default: true */
   reviewEnabled: boolean;
+  /** How background review invokes the LLM. Default: direct */
+  reviewTransport?: ReviewTransport;
   /** Flush memories before compaction. Default: true */
   flushOnCompact: boolean;
   /** Flush memories on session shutdown. Default: true */
@@ -52,6 +56,8 @@ export interface MemoryConfig {
   llmModelOverride?: string;
   /** Override thinking level used for child pi -p subprocess LLM calls. Default: unset */
   llmThinkingOverride?: ThinkingLevel;
+  /** Extra extension entry paths required by child Pi processes, such as provider auth adapters. */
+  childExtensionPaths?: string[];
   /** Strategy when memory is full. Default: auto-consolidate */
   memoryOverflowStrategy?: MemoryOverflowStrategy;
   /** Legacy alias for memoryOverflowStrategy. Default: true */