pi-hermes-memory 0.7.23 → 0.8.0

package/src/handlers/session-flush.ts

@@ -2,29 +2,73 @@
  * Session flush — gives the agent one turn to save memories before context is lost.
  * Ported from hermes-agent/run_agent.py (flush_memories).
  * See PLAN.md → "Hermes Source File Reference Map" for source lines.
+ *
+ * Default transport: in-process direct completion (same mechanism as
+ * background review — see review-memory-ops.ts). Falls back to a `pi -p`
+ * subprocess only if direct mode fails or reviewTransport forces subprocess.
  */
 
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "../store/memory-store.js";
-import { FLUSH_PROMPT } from "../constants.js";
+import { DatabaseManager } from "../store/db.js";
+import { DIRECT_FLUSH_SYSTEM_PROMPT, ENTRY_DELIMITER, FLUSH_PROMPT } from "../constants.js";
 import type { MemoryConfig } from "../types.js";
 import { collectMessageParts } from "./message-parts.js";
 import { execChildPrompt } from "./pi-child-process.js";
+import { runDirectMemoryCompletion, usesDirectTransport } from "./review-memory-ops.js";
+
+function buildDirectFlushUserPrompt(
+  store: MemoryStore,
+  projectStore: MemoryStore | null,
+  parts: string[],
+): string {
+  const sections = [
+    "--- Current Memory ---",
+    store.getMemoryEntries().join(ENTRY_DELIMITER) || "(empty)",
+    "",
+    "--- Current User Profile ---",
+    store.getUserEntries().join(ENTRY_DELIMITER) || "(empty)",
+  ];
+
+  if (projectStore) {
+    sections.push(
+      "",
+      "--- Current Project Memory ---",
+      projectStore.getMemoryEntries().join(ENTRY_DELIMITER) || "(empty)",
+    );
+  }
+
+  sections.push(
+    "",
+    "--- Conversation ---",
+    parts.join("\n\n"),
+  );
+
+  return sections.join("\n");
+}
 
 export function setupSessionFlush(
   pi: ExtensionAPI,
   store: MemoryStore,
   projectStore: MemoryStore | null,
   config: MemoryConfig,
+  dbManager: DatabaseManager | null = null,
+  projectName?: string | null,
+  deps: { runDirectMemoryCompletion?: typeof runDirectMemoryCompletion } = {},
 ): void {
   let userTurnCount = 0;
+  const runDirect = deps.runDirectMemoryCompletion ?? runDirectMemoryCompletion;
 
   pi.on("message_end", async (event, _ctx) => {
     if (event.message.role === "user") userTurnCount++;
   });
 
-  /** Shared flush logic — builds conversation snapshot and spawns pi -p */
-  async function flush(ctx: any, signal?: AbortSignal, timeoutMs = 30000): Promise<void> {
+  /** Shared flush logic — builds conversation snapshot and saves memories */
+  async function flush(
+    ctx: Pick<ExtensionContext, "sessionManager" | "model" | "modelRegistry">,
+    signal?: AbortSignal,
+    timeoutMs = 30000,
+  ): Promise<void> {
     if (userTurnCount < config.flushMinTurns) return;
 
     let entries;
@@ -35,6 +79,29 @@ export function setupSessionFlush(
     }
 
     const parts = collectMessageParts(entries, config.flushRecentMessages);
+
+    if (usesDirectTransport(config)) {
+      try {
+        const directResult = await runDirect(
+          ctx,
+          store,
+          projectStore,
+          {
+            systemPrompt: DIRECT_FLUSH_SYSTEM_PROMPT,
+            userPrompt: buildDirectFlushUserPrompt(store, projectStore, parts),
+            config,
+            timeoutMs,
+            signal,
+          },
+          dbManager,
+          projectName,
+        );
+        if (directResult.ok) return;
+      } catch {
+        // Fall through to subprocess below.
+      }
+    }
+
     const flushMessage = [
       FLUSH_PROMPT,
       "",

package/src/handlers/sync-markdown-memories.ts

@@ -1,6 +1,6 @@
 /**
- * Markdown memory sync command — /memory-sync-markdown imports existing
- * Markdown-backed memories into the SQLite search store.
+ * Markdown memory sync command — /memory-sync-markdown reconciles the SQLite
+ * search mirror with authoritative Markdown memory files.
  */
 
 import fs from 'node:fs';
@@ -8,20 +8,30 @@ import path from 'node:path';
 import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 import { DatabaseManager } from '../store/db.js';
 import {
-  parseMarkdownMemoryEntry,
-  syncMemoryEntry,
+  reconcileMarkdownFailureScopes,
+  reconcileMarkdownMemoryScope,
 } from '../store/sqlite-memory-store.js';
 import { ENTRY_DELIMITER, MEMORY_FILE, USER_FILE } from '../constants.js';
 import { AGENT_ROOT } from '../paths.js';
+import {
+  migrateExtensionRoot,
+  type ExtensionRootMigrationOptions,
+} from '../extension-root-migration.js';
+import { withMarkdownMutationLock } from '../store/markdown-mutation-lock.js';
 
 export interface BackfillCounters {
   filesScanned: number;
   entriesScanned: number;
   imported: number;
   skipped: number;
+  removed: number;
   warnings: string[];
 }
 
+export interface MigrationSyncOptions extends ExtensionRootMigrationOptions {
+  onMigrationSucceeded?: () => void;
+}
+
 function readEntries(filePath: string): string[] {
   if (!fs.existsSync(filePath)) return [];
   const raw = fs.readFileSync(filePath, 'utf-8').trim();
@@ -29,37 +39,15 @@ function readEntries(filePath: string): string[] {
   return raw.split(ENTRY_DELIMITER).map((entry) => entry.trim()).filter(Boolean);
 }
 
-function importEntries(
-  dbManager: DatabaseManager,
-  counters: BackfillCounters,
-  entries: string[],
-  target: 'memory' | 'user' | 'failure',
-  project: string | null = null,
-): void {
-  for (const rawEntry of entries) {
-    counters.entriesScanned++;
-    try {
-      const parsed = parseMarkdownMemoryEntry(rawEntry, target, project);
-      const result = syncMemoryEntry(dbManager, parsed);
-      if (result.action === 'inserted') counters.imported++;
-      else counters.skipped++;
-    } catch (err) {
-      counters.warnings.push(
-        `${path.basename(project ?? 'global')}/${target}: ${err instanceof Error ? err.message : String(err)}`,
-      );
-    }
-  }
-}
-
 function scanProjectDirs(agentRoot: string, globalDir: string, projectsMemoryDir = "projects-memory"): Array<{ name: string; memoryFile: string }> {
-  const projectsRoot = path.join(agentRoot, projectsMemoryDir);
+  const projectsRoot = path.resolve(agentRoot, projectsMemoryDir);
   const projects = new Map<string, string>();
 
   if (fs.existsSync(projectsRoot)) {
     for (const name of fs.readdirSync(projectsRoot)) {
-      const dir = path.join(projectsRoot, name);
-      const memoryFile = path.join(dir, MEMORY_FILE);
-      if (fs.existsSync(dir) && fs.statSync(dir).isDirectory() && fs.existsSync(memoryFile)) {
+      if (!isSafeProjectName(name, projectsRoot)) continue;
+      const memoryFile = resolveAuthoritativeMemoryFile(projectsRoot, name);
+      if (memoryFile) {
         projects.set(name, memoryFile);
       }
     }
@@ -74,9 +62,9 @@ function scanProjectDirs(agentRoot: string, globalDir: string, projectsMemoryDir
     for (const name of fs.readdirSync(agentRoot)) {
       if ((globalDirName && name === globalDirName) || name === projectsMemoryDir || name === 'skills' || name.startsWith('.')) continue;
       if (projects.has(name)) continue;
-      const dir = path.join(agentRoot, name);
-      const memoryFile = path.join(dir, MEMORY_FILE);
-      if (fs.existsSync(dir) && fs.statSync(dir).isDirectory() && fs.existsSync(memoryFile)) {
+      if (!isSafeProjectName(name, resolvedAgentRoot)) continue;
+      const memoryFile = resolveAuthoritativeMemoryFile(resolvedAgentRoot, name);
+      if (memoryFile) {
         projects.set(name, memoryFile);
       }
     }
@@ -87,17 +75,73 @@ function scanProjectDirs(agentRoot: string, globalDir: string, projectsMemoryDir
     .filter(({ memoryFile }) => fs.existsSync(memoryFile));
 }
 
-export function syncMarkdownMemoriesToSqlite(
+function realpathIfPresent(filePath: string): string {
+  try {
+    return fs.realpathSync(filePath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return path.resolve(filePath);
+    throw error;
+  }
+}
+
+function resolveAuthoritativeMemoryFile(root: string, projectName: string): string | null {
+  const canonicalRoot = realpathIfPresent(root);
+  if (!isSafeProjectName(projectName, path.resolve(root))) return null;
+
+  const projectDir = path.join(root, projectName);
+  let projectStat: fs.Stats;
+  try {
+    projectStat = fs.lstatSync(projectDir);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return path.join(canonicalRoot, projectName, MEMORY_FILE);
+    }
+    throw error;
+  }
+  if (projectStat.isSymbolicLink() || !projectStat.isDirectory()) return null;
+
+  const canonicalProjectDir = fs.realpathSync(projectDir);
+  if (path.dirname(canonicalProjectDir) !== canonicalRoot) return null;
+
+  const memoryFile = path.join(projectDir, MEMORY_FILE);
+  let memoryStat: fs.Stats;
+  try {
+    memoryStat = fs.lstatSync(memoryFile);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return path.join(canonicalProjectDir, MEMORY_FILE);
+    }
+    throw error;
+  }
+  if (memoryStat.isSymbolicLink() || !memoryStat.isFile()) return null;
+
+  const canonicalMemoryFile = fs.realpathSync(memoryFile);
+  if (path.dirname(canonicalMemoryFile) !== canonicalProjectDir || path.basename(canonicalMemoryFile) !== MEMORY_FILE) {
+    return null;
+  }
+  return canonicalMemoryFile;
+}
+
+function isSafeProjectName(name: string, projectsRoot: string): boolean {
+  if (!name || name === '.' || name === '..' || name.includes('/') || name.includes('\\') || path.isAbsolute(name)) {
+    return false;
+  }
+  const projectDir = path.resolve(projectsRoot, name);
+  return path.dirname(projectDir) === projectsRoot && path.basename(projectDir) === name;
+}
+
+export async function syncMarkdownMemoriesToSqlite(
   dbManager: DatabaseManager,
   globalDir: string,
   projectsMemoryDir?: string,
   agentRoot = AGENT_ROOT,
-): BackfillCounters & { projectCount: number } {
+): Promise<BackfillCounters & { projectCount: number }> {
   const counters: BackfillCounters = {
     filesScanned: 0,
     entriesScanned: 0,
     imported: 0,
     skipped: 0,
+    removed: 0,
     warnings: [],
   };
 
@@ -105,27 +149,74 @@ export function syncMarkdownMemoriesToSqlite(
   const globalUserFile = path.join(globalDir, USER_FILE);
   const globalFailureFile = path.join(globalDir, 'failures.md');
 
-  const importFile = (
-    filePath: string,
+  const reconcileFile = async (
+    filePath: string | null,
     target: 'memory' | 'user' | 'failure',
     project: string | null = null,
   ) => {
-    if (!fs.existsSync(filePath)) return;
-    counters.filesScanned++;
-    const entries = readEntries(filePath);
-    importEntries(dbManager, counters, entries, target, project);
+    const reconcile = () => {
+      if (filePath && fs.existsSync(filePath)) counters.filesScanned++;
+      const entries = filePath ? readEntries(filePath) : [];
+      counters.entriesScanned += entries.length;
+      try {
+        const result = target === 'failure'
+          ? reconcileMarkdownFailureScopes(dbManager, entries)
+          : reconcileMarkdownMemoryScope(dbManager, entries, target, project);
+        counters.imported += result.inserted;
+        counters.skipped += result.existing;
+        counters.removed += result.removed;
+      } catch (err) {
+        counters.warnings.push(
+          `${path.basename(project ?? 'global')}/${target}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    };
+    if (filePath) await withMarkdownMutationLock(filePath, reconcile);
+    else reconcile();
   };
 
-  importFile(globalMemoryFile, 'memory');
-  importFile(globalUserFile, 'user');
-  importFile(globalFailureFile, 'failure');
+  await reconcileFile(globalMemoryFile, 'memory');
+  await reconcileFile(globalUserFile, 'user');
+  await reconcileFile(globalFailureFile, 'failure');
 
   const projects = scanProjectDirs(agentRoot, globalDir, projectsMemoryDir);
-  for (const project of projects) {
-    importFile(project.memoryFile, 'memory', project.name);
+  const projectFiles = new Map(projects.map((project) => [project.name, project.memoryFile]));
+  const mirroredProjects = dbManager.getDb().prepare(`
+    SELECT DISTINCT project
+    FROM memories
+    WHERE project IS NOT NULL AND target = 'memory'
+  `).all() as Array<{ project: string }>;
+  const projectNames = new Set([
+    ...projectFiles.keys(),
+    ...mirroredProjects.map(({ project }) => project),
+  ]);
+  const projectsRoot = path.resolve(agentRoot, projectsMemoryDir ?? 'projects-memory');
+  for (const projectName of projectNames) {
+    const memoryFile = projectFiles.get(projectName)
+      ?? resolveAuthoritativeMemoryFile(projectsRoot, projectName);
+    await reconcileFile(memoryFile, 'memory', projectName);
   }
 
-  return { ...counters, projectCount: projects.length };
+  return { ...counters, projectCount: projectNames.size };
+}
+
+export async function migrateThenSyncMarkdownMemories(
+  dbManager: DatabaseManager,
+  legacyGlobalDir: string | null,
+  globalDir: string,
+  projectsMemoryDir?: string,
+  agentRoot = AGENT_ROOT,
+  migrationOptions: MigrationSyncOptions = {},
+): Promise<BackfillCounters & { projectCount: number }> {
+  if (legacyGlobalDir) {
+    const migration = await migrateExtensionRoot(legacyGlobalDir, globalDir, migrationOptions);
+    const sessionsFailure = migration.criticalFailures.find((failure) => failure.name === 'sessions.db');
+    if (sessionsFailure) {
+      throw new Error(`sessions.db migration failed: ${sessionsFailure.message}`);
+    }
+    migrationOptions.onMigrationSucceeded?.();
+  }
+  return await syncMarkdownMemoriesToSqlite(dbManager, globalDir, projectsMemoryDir, agentRoot);
 }
 
 export function registerSyncMarkdownMemoriesCommand(
@@ -136,19 +227,20 @@ export function registerSyncMarkdownMemoriesCommand(
   agentRoot = AGENT_ROOT,
 ): void {
   pi.registerCommand('memory-sync-markdown', {
-    description: 'Backfill Markdown memories into the SQLite search store',
+    description: 'Reconcile the SQLite search mirror with Markdown memories',
     handler: async (_args, ctx: ExtensionCommandContext) => {
-      ctx.ui.notify('🔄 Scanning Markdown memory files for SQLite backfill...', 'info');
+      ctx.ui.notify('🔄 Reconciling the SQLite search mirror with Markdown memories...', 'info');
 
       try {
-        const counters = syncMarkdownMemoriesToSqlite(dbManager, globalDir, projectsMemoryDir, agentRoot);
+        const counters = await syncMarkdownMemoriesToSqlite(dbManager, globalDir, projectsMemoryDir, agentRoot);
 
         let output = `\n✅ Markdown → SQLite sync complete!\n\n`;
         output += `📊 Results:\n`;
         output += `├─ Files scanned: ${counters.filesScanned}\n`;
         output += `├─ Entries scanned: ${counters.entriesScanned}\n`;
         output += `├─ Imported into SQLite: ${counters.imported}\n`;
-        output += `└─ Skipped as duplicates: ${counters.skipped}\n`;
+        output += `├─ Skipped as duplicates: ${counters.skipped}\n`;
+        output += `└─ Removed orphaned rows: ${counters.removed}\n`;
 
         if (counters.projectCount > 0) {
           output += `\n📁 Project memories scanned: ${counters.projectCount}\n`;

package/src/index.ts

@@ -23,6 +23,7 @@
  */
 
 import * as path from "node:path";
+import * as fs from "node:fs";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "./store/memory-store.js";
 import { SkillStore } from "./store/skill-store.js";
@@ -45,14 +46,14 @@ import { registerInterviewCommand } from "./handlers/interview.js";
 import { registerSwitchProjectCommand } from "./handlers/switch-project.js";
 import { registerIndexSessionsCommand } from "./handlers/index-sessions.js";
 import { registerLearnMemoryCommand } from "./handlers/learn-memory.js";
-import { registerSyncMarkdownMemoriesCommand, syncMarkdownMemoriesToSqlite } from "./handlers/sync-markdown-memories.js";
+import { migrateThenSyncMarkdownMemories, registerSyncMarkdownMemoriesCommand } from "./handlers/sync-markdown-memories.js";
 import { registerPreviewContextCommand } from "./handlers/preview-context.js";
 import { loadConfig } from "./config.js";
 import { detectProject, detectProjectSkills } from "./project.js";
 import { buildPromptContext } from "./prompt-context.js";
 import { migrateLegacyProjectMemoryDirs } from "./project-memory-migration.js";
-import { migrateExtensionRoot } from "./extension-root-migration.js";
 import { AGENT_ROOT } from "./paths.js";
+import { isDatabaseMigrationPending } from "./extension-root-migration.js";
 
 export function resolveProjectSkillDiscovery(
   skillStore: SkillStore,
@@ -95,7 +96,7 @@ export default function (pi: ExtensionAPI) {
     : configuredMemoryDir;
 
   const shouldMigrateExtensionRoot = !configuredMemoryDir || pointsToLegacyMemoryDir;
-  let extensionRootMigrated = false;
+  let persistenceInitialized = false;
 
   const store = new MemoryStore({ ...config, memoryDir: globalDir });
   const project = detectProject(config.projectsMemoryDir);
@@ -109,6 +110,15 @@ export default function (pi: ExtensionAPI) {
     migrationSentinelPath: path.join(globalDir, ".skills-migrated-to-extension-storage"),
   });
   const dbManager = new DatabaseManager(globalDir);
+  let databaseMigrationPending = shouldMigrateExtensionRoot
+    && isDatabaseMigrationPending(legacyGlobalDir, globalDir);
+  if (databaseMigrationPending) {
+    dbManager.setOpenGuard(() => {
+      if (databaseMigrationPending) {
+        throw new Error("Legacy sessions.db migration is pending");
+      }
+    });
+  }
   const sessionsDir = path.join(agentRoot, "sessions");
 
   const refreshSkillProjectContext = (cwd?: string) => {
@@ -124,12 +134,6 @@ export default function (pi: ExtensionAPI) {
   // ~/.pi/agent/<project>/ layout. This is non-destructive: legacy folders
   // remain in place while entries are copied/merged into projects-memory/.
   migrateLegacyProjectMemoryDirs(agentRoot, config.projectsMemoryDir);
-  try {
-    syncMarkdownMemoriesToSqlite(dbManager, globalDir, config.projectsMemoryDir, agentRoot);
-  } catch {
-    // Best-effort only: failed SQLite backfill should not block extension startup.
-  }
-
   // Detect project from cwd using shared helper
   // Project-scoped store: ~/.pi/agent/<projectsMemoryDir>/<project_name>/
   const projectConfig = project.memoryDir
@@ -139,13 +143,25 @@ export default function (pi: ExtensionAPI) {
 
   // ── 1. Load memory from disk on session start ──
   pi.on("session_start", async (_event, ctx) => {
-    if (shouldMigrateExtensionRoot && !extensionRootMigrated) {
+    if (!persistenceInitialized) {
       try {
-        await migrateExtensionRoot(legacyGlobalDir, globalDir);
+        await migrateThenSyncMarkdownMemories(
+          dbManager,
+          shouldMigrateExtensionRoot ? legacyGlobalDir : null,
+          globalDir,
+          config.projectsMemoryDir,
+          agentRoot,
+          {
+            onMigrationSucceeded: () => {
+              databaseMigrationPending = false;
+              dbManager.setOpenGuard(null);
+            },
+          },
+        );
+        persistenceInitialized = true;
       } catch {
-        // best effort migration only
+        // Best-effort only: migration or SQLite backfill must not block startup.
       }
-      extensionRootMigrated = true;
     }
 
     refreshSkillProjectContext(ctx.cwd);
@@ -154,7 +170,7 @@ export default function (pi: ExtensionAPI) {
     await store.loadFromDisk();
     if (projectStore) await projectStore.loadFromDisk();
 
-    scheduleSessionBackfill(dbManager, sessionsDir, {
+    if (persistenceInitialized) scheduleSessionBackfill(dbManager, sessionsDir, {
       notify: (message, level) => {
         const ui = (ctx as { ui?: { notify?: (message: string, level?: string) => void } }).ui;
         if (ui?.notify) {
@@ -194,7 +210,7 @@ export default function (pi: ExtensionAPI) {
   });
 
   // ── 6. Setup session-end flush ──
-  setupSessionFlush(pi, store, projectStore, config);
+  setupSessionFlush(pi, store, projectStore, config, dbManager, projectName);
 
   // ── 7. Setup auto-consolidation (inject consolidator into stores) ──
   store.setConsolidator(async (target, signal) => {
@@ -206,7 +222,7 @@ export default function (pi: ExtensionAPI) {
       return triggerConsolidation(pi, projectStore, target, signal, config.consolidationTimeoutMs, toolTarget, config);
     });
   }
-  registerConsolidateCommand(pi, store, config.consolidationTimeoutMs, projectStore, projectName, config);
+  registerConsolidateCommand(pi, store, config.consolidationTimeoutMs, projectStore, projectName, config, dbManager);
 
   // ── 8. Setup correction detection ──
   setupCorrectionDetector(pi, store, projectStore, config, dbManager, projectName);