pi-hermes-memory 0.7.23 → 0.8.0

package/src/handlers/auto-consolidate.ts

@@ -2,19 +2,72 @@
  * Auto-consolidation — when memory hits capacity, trigger automatic
  * consolidation instead of returning an error.
  *
- * Uses pi.exec() to spawn a one-shot consolidation process.
- * The child process modifies files on disk, so the parent MUST reload
- * from disk after consolidation completes.
+ * Default transport: in-process direct completion (same mechanism as
+ * background review — see review-memory-ops.ts), used only when a caller
+ * supplies model/modelRegistry access (the manual `/memory-consolidate`
+ * command has it; the automatic over-capacity consolidator registered on
+ * MemoryStore does not, since MemoryStore itself has no extension-runtime
+ * access, so that path stays subprocess-only). Falls back to a `pi -p`
+ * subprocess when direct mode is unavailable, declines, or fails.
+ *
+ * The subprocess child process modifies files on disk, so the parent MUST
+ * reload from disk after a subprocess-based consolidation completes.
  */
 
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { createHash } from "node:crypto";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "../store/memory-store.js";
-import { CONSOLIDATION_PROMPT, ENTRY_DELIMITER } from "../constants.js";
+import { DatabaseManager } from "../store/db.js";
+import { CONSOLIDATION_PROMPT, DIRECT_CONSOLIDATION_SYSTEM_PROMPT, ENTRY_DELIMITER } from "../constants.js";
 import type { ConsolidationResult, MemoryConfig } from "../types.js";
+import { AGENT_ROOT } from "../paths.js";
 import { execChildPrompt } from "./pi-child-process.js";
+import { runDirectMemoryCompletion, usesDirectTransport } from "./review-memory-ops.js";
+import { AtomicLockCoordinator } from "../store/atomic-lock-coordinator.js";
 
 type MemoryTarget = "memory" | "user" | "failure";
 type ToolMemoryTarget = MemoryTarget | "project";
+type ConsolidationLlmConfig = Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride" | "reviewTransport">;
+
+const CONSOLIDATION_LOCK_STALE_GRACE_MS = 30000;
+const CONSOLIDATION_LOCK_ENV = "PI_HERMES_CONSOLIDATION_LOCK_DIR";
+
+interface ConsolidationLock {
+  release: () => Promise<void>;
+}
+
+function consolidationLockRoot(): string {
+  return process.env[CONSOLIDATION_LOCK_ENV]?.trim()
+    || path.join(AGENT_ROOT, "pi-hermes-memory", ".consolidation-locks");
+}
+
+function sanitizeLockPart(value: string): string {
+  return value.replace(/[^a-z0-9._-]+/gi, "_").slice(0, 80) || "unknown";
+}
+
+function consolidationLockKey(target: MemoryTarget, toolTarget: ToolMemoryTarget, storageIdentity: string): string {
+  const storageHash = createHash("sha256").update(storageIdentity).digest("hex");
+  return `${sanitizeLockPart(toolTarget)}:${sanitizeLockPart(target)}:${storageHash}`;
+}
+
+async function tryAcquireConsolidationLock(
+  store: MemoryStore,
+  target: MemoryTarget,
+  toolTarget: ToolMemoryTarget,
+  timeoutMs: number,
+): Promise<ConsolidationLock | null> {
+  const storageIdentity = await store.getStorageIdentity(target);
+  const root = consolidationLockRoot();
+  await fs.mkdir(root, { recursive: true });
+  const coordinator = new AtomicLockCoordinator(path.join(root, "locks.sqlite"));
+  const lease = coordinator.tryAcquire(
+    consolidationLockKey(target, toolTarget, storageIdentity),
+    { staleMs: Math.max(timeoutMs, 0) + CONSOLIDATION_LOCK_STALE_GRACE_MS },
+  );
+  return lease ? { release: async () => lease.release() } : null;
+}
 
 function entriesForTarget(store: MemoryStore, target: MemoryTarget): string[] {
   if (target === "user") return store.getUserEntries();
@@ -34,7 +87,7 @@ function describeConsolidationFailure(
   timeoutMs: number,
 ): string {
   const stderr = result.stderr?.trim();
-  const terminated = result.killed || result.code === 143;
+  const terminated = result.killed || result.code === 124 || result.code === 143;
 
   if (terminated) {
     return `Consolidation subprocess was terminated (likely timeout or cancellation). Timeout: ${timeoutMs}ms. Consider increasing consolidationTimeoutMs if this is a manual run.`;
@@ -50,10 +103,47 @@ export async function triggerConsolidation(
   signal?: AbortSignal,
   timeoutMs: number = 60000,
   toolTarget: ToolMemoryTarget = target,
-  llmConfig: Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride"> = {},
+  llmConfig: ConsolidationLlmConfig = {},
+  directCtx: Pick<ExtensionContext, "model" | "modelRegistry"> | null = null,
+  dbManager: DatabaseManager | null = null,
+  projectName?: string | null,
+  deps: { runDirectMemoryCompletion?: typeof runDirectMemoryCompletion } = {},
 ): Promise<ConsolidationResult> {
   const entries = entriesForTarget(store, target);
   const currentContent = entries.join(ENTRY_DELIMITER);
+  const runDirect = deps.runDirectMemoryCompletion ?? runDirectMemoryCompletion;
+
+  if (directCtx && usesDirectTransport(llmConfig)) {
+    try {
+      const directResult = await runDirect(
+        directCtx,
+        store,
+        toolTarget === "project" ? store : null,
+        {
+          systemPrompt: DIRECT_CONSOLIDATION_SYSTEM_PROMPT,
+          userPrompt: [
+            `--- Current ${labelForTarget(target, toolTarget)} Entries (target: '${toolTarget}') ---`,
+            currentContent || "(empty)",
+            "",
+            `Only emit operations with "target": "${toolTarget}".`,
+          ].join("\n"),
+          config: llmConfig,
+          timeoutMs,
+          signal,
+        },
+        dbManager,
+        projectName,
+      );
+      // Consolidation only did its job if it actually freed space — unlike
+      // review/flush/correction, an empty or fully-skipped result here is a
+      // failure worth falling back to subprocess for, not a normal outcome.
+      if (directResult.ok && directResult.appliedCount > 0) {
+        return { consolidated: true };
+      }
+    } catch {
+      // Fall through to subprocess below.
+    }
+  }
 
   const prompt = [
     CONSOLIDATION_PROMPT,
@@ -64,7 +154,17 @@ export async function triggerConsolidation(
     `Use the memory tool to consolidate. Target: '${toolTarget}'`,
   ].join("\n");
 
+  let lock: ConsolidationLock | null = null;
+
   try {
+    lock = await tryAcquireConsolidationLock(store, target, toolTarget, timeoutMs);
+    if (!lock) {
+      return {
+        consolidated: false,
+        error: `Consolidation already in progress for target '${toolTarget}'. Skipping duplicate subprocess.`,
+      };
+    }
+
     const result = await execChildPrompt(pi, prompt, llmConfig, {
       signal,
       timeoutMs,
@@ -83,6 +183,10 @@ export async function triggerConsolidation(
       consolidated: false,
       error: `Consolidation failed: ${String(err).slice(0, 200)}`,
     };
+  } finally {
+    if (lock) {
+      try { await lock.release(); } catch { /* best-effort cleanup */ }
+    }
   }
 }
 
@@ -95,7 +199,9 @@ export function registerConsolidateCommand(
   timeoutMs: number = 60000,
   projectStore: MemoryStore | null = null,
   projectName?: string | null,
-  llmConfig: Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride"> = {},
+  llmConfig: ConsolidationLlmConfig = {},
+  dbManager: DatabaseManager | null = null,
+  deps: { runDirectMemoryCompletion?: typeof runDirectMemoryCompletion } = {},
 ): void {
   pi.registerCommand("memory-consolidate", {
     description: "Manually trigger memory consolidation to free up space",
@@ -157,6 +263,10 @@ export function registerConsolidateCommand(
           manualTimeoutMs,
           item.toolTarget,
           llmConfig,
+          ctx,
+          dbManager,
+          projectName,
+          deps,
         );
 
         if (result.consolidated) {

package/src/handlers/background-review.ts

@@ -8,13 +8,13 @@
  */
 
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { COMBINED_REVIEW_PROMPT } from "../constants.js";
+import { COMBINED_REVIEW_PROMPT, DIRECT_REVIEW_SYSTEM_PROMPT } from "../constants.js";
 import { MemoryStore } from "../store/memory-store.js";
 import { DatabaseManager } from "../store/db.js";
 import type { MemoryConfig } from "../types.js";
 import { applyRecentMessageLimit, collectMessageParts } from "./message-parts.js";
 import { execChildPrompt } from "./pi-child-process.js";
-import { runDirectBackgroundReview, type DirectReviewResult } from "./review-memory-ops.js";
+import { runDirectMemoryCompletion, usesDirectTransport, type DirectReviewResult } from "./review-memory-ops.js";
 
 export interface BackgroundReviewOptions {
   dbManager?: DatabaseManager | null;
@@ -23,8 +23,12 @@ export interface BackgroundReviewOptions {
 }
 
 export interface BackgroundReviewDeps {
-  runDirectReview?: typeof runDirectBackgroundReview;
+  runDirectReview?: typeof runDirectMemoryCompletion;
   execChildPrompt?: typeof execChildPrompt;
+  /** Test-only hook: called once runReview() has fully settled (after the
+   * fire-and-forget review work completes and reviewInProgress resets),
+   * since production callers never await runReview() directly. */
+  onReviewSettled?: () => void;
 }
 
 export interface ReviewPromptInput {
@@ -97,10 +101,6 @@ function shouldNotifySubprocess(stdout: string | undefined): boolean {
   return !!output && !output.toLowerCase().includes("nothing to save");
 }
 
-function usesDirectTransport(config: MemoryConfig): boolean {
-  return (config.reviewTransport ?? "direct") === "direct";
-}
-
 async function runSubprocessReview(
   pi: ExtensionAPI,
   prompt: string,
@@ -122,8 +122,9 @@ export function setupBackgroundReview(
 ): void {
   const dbManager = options.dbManager ?? null;
   const projectName = options.projectName ?? null;
-  const runDirectReview = options.deps?.runDirectReview ?? runDirectBackgroundReview;
+  const runDirectReview = options.deps?.runDirectReview ?? runDirectMemoryCompletion;
   const execChild = options.deps?.execChildPrompt ?? execChildPrompt;
+  const onReviewSettled = options.deps?.onReviewSettled;
 
   let turnsSinceReview = 0;
   let toolCallsSinceReview = 0;
@@ -194,6 +195,7 @@ export function setupBackgroundReview(
 
     const finishReview = () => {
       reviewInProgress = false;
+      onReviewSettled?.();
     };
 
     const notifyIfSaved = (saved: boolean) => {
@@ -204,22 +206,26 @@ export function setupBackgroundReview(
 
     const runReview = async (): Promise<void> => {
       if (usesDirectTransport(config)) {
-        const directResult = await runDirectReview(
-          ctx as Pick<ExtensionContext, "model" | "modelRegistry">,
-          store,
-          projectStore,
-          { userPrompt: directPrompt, config, timeoutMs: 120000 },
-          dbManager,
-          projectName,
-        );
-
-        if (directResult.ok) {
-          notifyIfSaved(shouldNotifyDirect(directResult));
-          return;
-        }
+        try {
+          const directResult = await runDirectReview(
+            ctx as Pick<ExtensionContext, "model" | "modelRegistry">,
+            store,
+            projectStore,
+            { userPrompt: directPrompt, systemPrompt: DIRECT_REVIEW_SYSTEM_PROMPT, config, timeoutMs: 120000 },
+            dbManager,
+            projectName,
+          );
+
+          if (directResult.ok) {
+            notifyIfSaved(shouldNotifyDirect(directResult));
+            return;
+          }
 
-        if (directResult.fallbackReason === "empty") {
-          return;
+          if (directResult.fallbackReason === "empty") {
+            return;
+          }
+        } catch {
+          // Fall through to subprocess below.
         }
       }
 

package/src/handlers/child-process-watchdog.mjs

@@ -0,0 +1,90 @@
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+
+const [timeoutValue, cancellationPath, command, ...args] = process.argv.slice(2);
+const timeoutMs = Number(timeoutValue);
+
+if (!cancellationPath || !command || !Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+  process.stderr.write("pi-hermes-memory watchdog: invalid invocation\n");
+  process.exit(2);
+}
+
+const child = spawn(command, args, {
+  detached: process.platform !== "win32",
+  stdio: ["ignore", "pipe", "pipe"],
+});
+
+child.stdout?.pipe(process.stdout);
+child.stderr?.pipe(process.stderr);
+
+let timedOut = false;
+let cancelled = false;
+let terminating = false;
+let forceTimer;
+
+function signalTree(signal) {
+  if (!child.pid) return;
+  if (process.platform === "win32") {
+    const killer = spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
+      stdio: "ignore",
+      windowsHide: true,
+    });
+    killer.unref();
+    return;
+  }
+  try {
+    process.kill(-child.pid, signal);
+  } catch {
+    try { child.kill(signal); } catch {}
+  }
+}
+
+function terminateTree() {
+  if (terminating) return;
+  terminating = true;
+  signalTree("SIGTERM");
+  forceTimer = setTimeout(() => signalTree("SIGKILL"), 500);
+  forceTimer.unref();
+}
+
+const timeout = setTimeout(() => {
+  timedOut = true;
+  process.stderr.write(`[pi-hermes-memory] child timed out after ${timeoutMs}ms; terminating process tree\n`);
+  terminateTree();
+}, timeoutMs);
+timeout.unref();
+
+const cancellationPoll = cancellationPath === "-" ? undefined : setInterval(() => {
+  if (!existsSync(cancellationPath)) return;
+  cancelled = true;
+  process.stderr.write("[pi-hermes-memory] child cancellation requested; terminating process tree\n");
+  terminateTree();
+}, 25);
+cancellationPoll?.unref();
+
+for (const signal of ["SIGTERM", "SIGINT"]) {
+  process.on(signal, terminateTree);
+}
+
+child.once("error", (error) => {
+  clearTimeout(timeout);
+  if (cancellationPoll) clearInterval(cancellationPoll);
+  if (forceTimer) clearTimeout(forceTimer);
+  process.stderr.write(`pi-hermes-memory watchdog: ${error.message}\n`);
+  process.exitCode = timedOut ? 124 : cancelled ? 143 : 127;
+});
+
+child.once("close", (code, signal) => {
+  clearTimeout(timeout);
+  if (cancellationPoll) clearInterval(cancellationPoll);
+  if (forceTimer) clearTimeout(forceTimer);
+  if (timedOut) {
+    process.exitCode = 124;
+  } else if (cancelled) {
+    process.exitCode = 143;
+  } else if (typeof code === "number") {
+    process.exitCode = code;
+  } else {
+    process.exitCode = signal === "SIGTERM" ? 143 : 1;
+  }
+});

package/src/handlers/correction-detector.ts

@@ -11,21 +11,19 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "../store/memory-store.js";
 import { DatabaseManager } from "../store/db.js";
-import {
-  formatFailureMemoryContent,
-  syncMemoryEntry,
-} from "../store/sqlite-memory-store.js";
 import {
   CORRECTION_SAVE_PROMPT,
   CORRECTION_STRONG_PATTERNS,
   CORRECTION_WEAK_PATTERNS,
   CORRECTION_NEGATIVE_PATTERNS,
   CORRECTION_DIRECTIVE_WORDS,
+  DIRECT_CORRECTION_SYSTEM_PROMPT,
   ENTRY_DELIMITER,
 } from "../constants.js";
 import type { MemoryConfig } from "../types.js";
 import { getMessageText } from "../types.js";
 import { execChildPrompt } from "./pi-child-process.js";
+import { runDirectMemoryCompletion, usesDirectTransport } from "./review-memory-ops.js";
 
 /**
  * Extract the directive part from a correction message.
@@ -129,12 +127,14 @@ export function setupCorrectionDetector(
   config: MemoryConfig,
   dbManager: DatabaseManager | null = null,
   projectName?: string | null,
+  deps: { runDirectMemoryCompletion?: typeof runDirectMemoryCompletion } = {},
 ): void {
   if (!config.correctionDetection) return;
 
   let pendingCorrection = false;
   let turnsSinceLastCorrection = 3; // Start at threshold so first correction can fire immediately
   let correctionInProgress = false;
+  const runDirect = deps.runDirectMemoryCompletion ?? runDirectMemoryCompletion;
 
   // Flag on message_end (user role)
   pi.on("message_end", async (event, _ctx) => {
@@ -182,9 +182,7 @@ export function setupCorrectionDetector(
       const currentUser = store.getUserEntries().join(ENTRY_DELIMITER);
       const currentProject = projectStore ? projectStore.getMemoryEntries().join(ENTRY_DELIMITER) : null;
 
-      const prompt = [
-        CORRECTION_SAVE_PROMPT,
-        "",
+      const promptBody = [
         "--- Current Memory ---",
         currentMemory || "(empty)",
         "",
@@ -193,31 +191,67 @@ export function setupCorrectionDetector(
       ];
 
       if (currentProject !== null) {
-        prompt.push(
+        promptBody.push(
           "",
           "--- Current Project Memory ---",
           currentProject || "(empty)",
         );
       }
 
-      prompt.push(
+      promptBody.push(
         "",
         "--- Recent Conversation ---",
         recentParts.join("\n\n"),
       );
 
-      const result = await execChildPrompt(pi, prompt.join("\n"), config, {
-        signal: ctx.signal,
-        timeoutMs: 30000,
-      });
+      let savedViaLlm = false;
+
+      const runSubprocessCorrection = async (): Promise<void> => {
+        const subprocessPrompt = [CORRECTION_SAVE_PROMPT, "", ...promptBody].join("\n");
+        const result = await execChildPrompt(pi, subprocessPrompt, config, {
+          signal: ctx.signal,
+          timeoutMs: 30000,
+        });
+        if (result.code === 0 && result.stdout) {
+          const output = result.stdout.trim();
+          savedViaLlm = !!output && !output.toLowerCase().includes("nothing to save");
+        }
+      };
 
-      if (result.code === 0 && result.stdout) {
-        const output = result.stdout.trim();
-        if (output && !output.toLowerCase().includes("nothing to save")) {
-          ctx.ui.notify("🔧 Correction detected — memory updated", "info");
+      let handledDirect = false;
+      if (usesDirectTransport(config)) {
+        try {
+          const directResult = await runDirect(
+            ctx,
+            store,
+            projectStore,
+            {
+              systemPrompt: DIRECT_CORRECTION_SYSTEM_PROMPT,
+              userPrompt: promptBody.join("\n"),
+              config,
+              timeoutMs: 30000,
+              signal: ctx.signal,
+            },
+            dbManager,
+            projectName,
+          );
+          if (directResult.ok) {
+            savedViaLlm = directResult.appliedCount > 0;
+            handledDirect = true;
+          }
+        } catch {
+          // Fall through to subprocess below.
         }
       }
 
+      if (!handledDirect) {
+        await runSubprocessCorrection();
+      }
+
+      if (savedViaLlm) {
+        ctx.ui.notify("🔧 Correction detected — memory updated", "info");
+      }
+
       // Also save as a failure memory for learning
       try {
         const lastUserMsg = recentParts.find(p => p.startsWith("[USER]"));
@@ -226,29 +260,11 @@ export function setupCorrectionDetector(
           const directive = extractCorrectionDirective(correctionText);
           const failureReason = "User corrected the agent";
           const scopedProjectName = projectStore ? projectName?.trim() || null : null;
-          const addResult = await store.addFailure(directive, {
+          await store.addFailure(directive, {
             category: "correction",
             failureReason,
             project: scopedProjectName ?? undefined,
           });
-
-          if (addResult.success && dbManager) {
-            try {
-              syncMemoryEntry(dbManager, {
-                content: formatFailureMemoryContent(directive, {
-                  category: "correction",
-                  failureReason,
-                  project: scopedProjectName,
-                }),
-                target: "failure",
-                project: scopedProjectName,
-                category: "correction",
-                failureReason,
-              });
-            } catch {
-              // Best-effort — searchable sync should not block correction capture
-            }
-          }
         }
       } catch {
         // Best-effort — don't block the session