pi-hermes-memory 0.7.23 → 0.8.0

package/src/handlers/pi-child-process.ts

@@ -1,15 +1,19 @@
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import type { MemoryConfig, ThinkingLevel } from "../types.js";
+import { AGENT_ROOT } from "../paths.js";
 
-type ChildLlmConfig = Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride">;
+type ChildLlmConfig = Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride" | "childExtensionPaths">;
 
 interface PiExecResult {
   code: number;
   stdout?: string;
   stderr?: string;
+  killed?: boolean;
 }
 
 interface ExecChildPromptOptions {
@@ -18,6 +22,20 @@ interface ExecChildPromptOptions {
   retryWithoutOverrides?: boolean;
 }
 
+interface ExecChildPromptDependencies {
+  removeTemporaryDirectory: (dir: string) => Promise<void>;
+}
+
+const DEFAULT_EXEC_CHILD_PROMPT_DEPENDENCIES: ExecChildPromptDependencies = {
+  removeTemporaryDirectory: async (dir) => {
+    await fs.rm(dir, { recursive: true, force: true });
+  },
+};
+
+const WATCHDOG_EXIT_GRACE_MS = 5000;
+const CHILD_PROCESS_WATCHDOG_PATH = fileURLToPath(
+  new URL("./child-process-watchdog.mjs", import.meta.url),
+);
 export interface ChildPiInvocation {
   command: string;
   args: string[];
@@ -80,34 +98,158 @@ export function inheritedExtensionArgs(argv: string[] = process.argv.slice(2)):
   return args;
 }
 
-function appendOwnExtensionArgs(args: string[]): void {
+// Provider-auth-adapter packages (e.g. Anthropic/xAI/Codex OAuth) inject
+// subscription billing headers via pi.registerProvider(). --no-extensions
+// strips these from child `pi -p` subprocesses, which silently rebills
+// subscription usage as pay-as-you-go "extra usage" instead (see issue #94).
+//
+// pi has no runtime API to enumerate loaded extensions or map a registered
+// provider back to its extension file, so we can't ask pi "what adapter is
+// active" directly. Instead we mirror pi's OWN static package-discovery
+// convention (package.json -> "pi": { "extensions": [...] }, the same field
+// pi-hermes-memory's own package.json declares) and match sibling package
+// names against a naming convention, so a future xai-oauth-adapter or
+// pi-codex-oauth-adapter is picked up automatically without a code change
+// here — no code execution, just JSON reads of sibling package.json files.
+const AUTH_ADAPTER_NAME_PATTERNS: readonly RegExp[] = [
+  /(^|[-/])oauth-adapter$/,
+  /(^|[-/])auth-adapter$/,
+];
+
+function isAuthAdapterPackageName(name: string): boolean {
+  return AUTH_ADAPTER_NAME_PATTERNS.some((pattern) => pattern.test(name));
+}
+
+// Read a sibling package's "pi": { "extensions": [...] } manifest field —
+// the same field pi's own loader reads — and resolve declared paths
+// relative to that package's directory. Mirrors loader.js#resolveExtensionEntries.
+function readPackageExtensionEntries(packageDir: string): string[] {
+  const packageJsonPath = join(packageDir, "package.json");
+  if (!existsSync(packageJsonPath)) return [];
+
+  let manifest: unknown;
+  try {
+    manifest = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+  } catch {
+    return [];
+  }
+
+  const declaredExtensions = (manifest as { pi?: { extensions?: unknown } } | null)?.pi?.extensions;
+  if (!Array.isArray(declaredExtensions)) return [];
+
+  const entries: string[] = [];
+  for (const relativePath of declaredExtensions) {
+    if (typeof relativePath !== "string") continue;
+    const resolved = resolve(packageDir, relativePath);
+    if (existsSync(resolved)) entries.push(resolved);
+  }
+  return entries;
+}
+
+function scanRootForAuthAdapters(root: string): string[] {
+  let entries: string[];
+  try {
+    entries = readdirSync(root);
+  } catch {
+    return [];
+  }
+
+  const detected: string[] = [];
+  for (const entry of entries) {
+    if (entry.startsWith("@")) {
+      // Scoped org, e.g. @xai/pi-oauth-adapter — one extra level, no deeper.
+      const scopeDir = join(root, entry);
+      let scopedPackages: string[];
+      try {
+        scopedPackages = readdirSync(scopeDir);
+      } catch {
+        continue;
+      }
+      for (const scopedName of scopedPackages) {
+        if (!isAuthAdapterPackageName(scopedName)) continue;
+        detected.push(...readPackageExtensionEntries(join(scopeDir, scopedName)));
+      }
+      continue;
+    }
+
+    if (!isAuthAdapterPackageName(entry)) continue;
+    detected.push(...readPackageExtensionEntries(join(root, entry)));
+  }
+  return detected;
+}
+
+// `roots` is overridable so tests can point this at fixture directories
+// instead of the real sibling-packages trees. Two roots are scanned by
+// default: packages installed alongside pi-hermes-memory's own package
+// (the common npm-managed-extensions layout), and AGENT_ROOT's npm
+// directory (covers pi-hermes-memory being loaded from elsewhere, e.g. a
+// local dev checkout via -e, while the adapter is still npm-managed).
+export function detectAuthAdapterExtensionPaths(roots?: string[]): string[] {
+  const searchRoots = roots ?? [
+    OWN_EXTENSION_PATH ? resolve(dirname(dirname(OWN_EXTENSION_PATH)), "..") : "",
+    join(AGENT_ROOT, "npm", "node_modules"),
+  ].filter((root) => root.length > 0);
+
+  const seenRoots: string[] = [];
+  const detected: string[] = [];
+  for (const root of searchRoots) {
+    if (seenRoots.includes(root)) continue;
+    seenRoots.push(root);
+    detected.push(...scanRootForAuthAdapters(root));
+  }
+  return detected;
+}
+
+function childExtensionPaths(config: ChildLlmConfig): string[] {
+  const candidates = [
+    OWN_EXTENSION_PATH,
+    ...(config.childExtensionPaths ?? []),
+    ...detectAuthAdapterExtensionPaths(),
+  ];
+  const seen = new Set<string>();
+  const paths: string[] = [];
+  for (const candidate of candidates) {
+    const trimmed = candidate?.trim();
+    if (!trimmed) continue;
+    const normalized = resolve(trimmed);
+    if (seen.has(normalized) || !existsSync(normalized)) continue;
+    seen.add(normalized);
+    paths.push(normalized);
+  }
+  return paths;
+}
+
+function appendOwnExtensionArgs(args: string[], config: ChildLlmConfig): void {
   // Skip all packages from settings.json (--no-extensions) — the subprocess
-  // only needs pi-hermes-memory to access the memory tool. Loading every
-  // plugin (context-mode, pi-lens, pi-web-access, pi-review, …) wastes
-  // prompt tokens and startup CPU for simple one-shot memory tasks.
-  if (OWN_EXTENSION_PATH) {
-    args.push("--no-extensions", "-e", OWN_EXTENSION_PATH);
+  // loads only Hermes and explicitly required provider adapters.
+  args.push("--no-extensions");
+  for (const extensionPath of childExtensionPaths(config)) {
+    args.push("-e", extensionPath);
   }
 }
 
-export function buildChildPiPromptArgs(prompt: string, config: ChildLlmConfig, _argv?: string[]): string[] {
+export function buildChildPiPromptArgs(
+  prompt: string,
+  config: ChildLlmConfig,
+  _argv: string[] = process.argv.slice(2),
+): string[] {
   const args = ["-p", "--no-session"];
   const model = normalizedModelOverride(config);
   const thinking = effectiveThinkingOverride(config);
 
   if (model) args.push("--model", model);
   if (thinking) args.push("--thinking", thinking);
-  appendOwnExtensionArgs(args);
+  appendOwnExtensionArgs(args, config);
   args.push(prompt);
 
   return args;
 }
 
-function basePromptArgs(prompt: string): string[] {
+function basePromptArgs(prompt: string, config: ChildLlmConfig): string[] {
   // Always use --no-extensions + own path so the retry also avoids loading
   // all settings.json packages — matching the primary code path.
   const args = ["-p", "--no-session"];
-  appendOwnExtensionArgs(args);
+  appendOwnExtensionArgs(args, config);
   args.push(prompt);
   return args;
 }
@@ -142,6 +284,40 @@ function resolvedPiCliPath(options: ResolveChildPiInvocationOptions): string | u
   return resolvedInstalledPiCliPath();
 }
 
+function resolvedWindowsPiInvocation(
+  args: string[],
+  execPath: string,
+): ChildPiInvocation | undefined {
+  const pathEntries = (process.env.PATH ?? process.env.Path ?? "")
+    .split(";")
+    .map((entry) => entry.trim().replace(/^"|"$/g, ""))
+    .filter(Boolean);
+
+  for (const directory of pathEntries) {
+    for (const executableName of ["pi.exe", "pi.com"]) {
+      const executablePath = join(directory, executableName);
+      if (existsSync(executablePath)) {
+        return { command: executablePath, args };
+      }
+    }
+
+    if (!existsSync(join(directory, "pi.cmd")) && !existsSync(join(directory, "pi.bat"))) {
+      continue;
+    }
+
+    for (const cliPath of [
+      join(directory, "node_modules", "@earendil-works", "pi-coding-agent", "dist", "cli.js"),
+      join(directory, "node_modules", "@earendil-works", "pi-coding-agent", "cli.js"),
+    ]) {
+      if (existsSync(cliPath)) {
+        return { command: execPath, args: [cliPath, ...args] };
+      }
+    }
+  }
+
+  return undefined;
+}
+
 export function resolveChildPiInvocation(
   args: string[],
   options: ResolveChildPiInvocationOptions = {},
@@ -152,13 +328,33 @@ export function resolveChildPiInvocation(
   }
 
   const piCliPath = resolvedPiCliPath(options);
-  if (!piCliPath) {
-    return { command: "pi", args };
+  if (piCliPath) {
+    return {
+      command: options.execPath ?? process.execPath,
+      args: [piCliPath, ...args],
+    };
   }
 
+  const fallback = resolvedWindowsPiInvocation(args, options.execPath ?? process.execPath);
+  if (fallback) return fallback;
+
+  throw new Error("Unable to resolve a directly executable Pi CLI on Windows");
+}
+
+export function resolveWatchedChildPiInvocation(
+  invocation: ChildPiInvocation,
+  timeoutMs: number,
+  cancellationPath = "-",
+): ChildPiInvocation {
   return {
-    command: options.execPath ?? process.execPath,
-    args: [piCliPath, ...args],
+    command: process.execPath,
+    args: [
+      CHILD_PROCESS_WATCHDOG_PATH,
+      String(timeoutMs),
+      cancellationPath,
+      invocation.command,
+      ...invocation.args,
+    ],
   };
 }
 
@@ -175,38 +371,75 @@ function shouldRetryWithoutOverridesForError(error: unknown): boolean {
   return shouldRetryWithoutOverridesFromText(String(error));
 }
 
+async function writePromptToTemporaryFile(prompt: string): Promise<{ dir: string; filePath: string }> {
+  const dir = await fs.mkdtemp(join(os.tmpdir(), "pi-hermes-prompt-"));
+  const filePath = join(dir, "prompt.md");
+  try {
+    await fs.writeFile(filePath, prompt, { encoding: "utf-8", mode: 0o600 });
+    return { dir, filePath };
+  } catch (error) {
+    try { await fs.rm(dir, { recursive: true, force: true }); } catch {}
+    throw error;
+  }
+}
+
 export async function execChildPrompt(
   pi: Pick<ExtensionAPI, "exec">,
   prompt: string,
   config: ChildLlmConfig,
   options: ExecChildPromptOptions,
+  dependencies: ExecChildPromptDependencies = DEFAULT_EXEC_CHILD_PROMPT_DEPENDENCIES,
 ): Promise<PiExecResult> {
   const execOptions = {
-    signal: options.signal,
-    timeout: options.timeoutMs,
+    timeout: options.timeoutMs + WATCHDOG_EXIT_GRACE_MS,
+  };
+  const temporaryPrompt = await writePromptToTemporaryFile(prompt);
+  const promptReference = `@${temporaryPrompt.filePath}`;
+  const cancellationPath = join(temporaryPrompt.dir, "cancel");
+  const requestCancellation = () => {
+    void fs.writeFile(cancellationPath, "", { mode: 0o600 }).catch(() => {});
   };
+  options.signal?.addEventListener("abort", requestCancellation, { once: true });
+  if (options.signal?.aborted) requestCancellation();
 
   try {
-    const invocation = resolveChildPiInvocation(buildChildPiPromptArgs(prompt, config));
-    const result = await pi.exec(invocation.command, invocation.args, execOptions) as PiExecResult;
-    if (
-      result.code === 0 ||
-      !options.retryWithoutOverrides ||
-      !hasChildLlmOverrides(config) ||
-      !shouldRetryWithoutOverrides(result)
-    ) {
-      return result;
+    try {
+      const invocation = resolveWatchedChildPiInvocation(
+        resolveChildPiInvocation(buildChildPiPromptArgs(promptReference, config)),
+        options.timeoutMs,
+        cancellationPath,
+      );
+      const result = await pi.exec(invocation.command, invocation.args, execOptions) as PiExecResult;
+      if (
+        result.code === 0 ||
+        !options.retryWithoutOverrides ||
+        !hasChildLlmOverrides(config) ||
+        !shouldRetryWithoutOverrides(result)
+      ) {
+        return result;
+      }
+    } catch (error) {
+      if (
+        !options.retryWithoutOverrides ||
+        !hasChildLlmOverrides(config) ||
+        !shouldRetryWithoutOverridesForError(error)
+      ) {
+        throw error;
+      }
     }
-  } catch (error) {
-    if (
-      !options.retryWithoutOverrides ||
-      !hasChildLlmOverrides(config) ||
-      !shouldRetryWithoutOverridesForError(error)
-    ) {
-      throw error;
+
+    const retryInvocation = resolveWatchedChildPiInvocation(
+      resolveChildPiInvocation(basePromptArgs(promptReference, config)),
+      options.timeoutMs,
+      cancellationPath,
+    );
+    return await pi.exec(retryInvocation.command, retryInvocation.args, execOptions) as PiExecResult;
+  } finally {
+    options.signal?.removeEventListener("abort", requestCancellation);
+    try {
+      await dependencies.removeTemporaryDirectory(temporaryPrompt.dir);
+    } catch {
+      try { await fs.unlink(temporaryPrompt.filePath); } catch {}
     }
   }
-
-  const retryInvocation = resolveChildPiInvocation(basePromptArgs(prompt));
-  return pi.exec(retryInvocation.command, retryInvocation.args, execOptions) as Promise<PiExecResult>;
 }

package/src/handlers/review-memory-ops.ts

@@ -5,16 +5,8 @@
 import type { Api, Model } from "@earendil-works/pi-ai";
 import { completeSimple, type Message, type SimpleStreamOptions } from "@earendil-works/pi-ai/compat";
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { DIRECT_REVIEW_SYSTEM_PROMPT } from "../constants.js";
 import { MemoryStore } from "../store/memory-store.js";
-import { DatabaseManager } from "../store/db.js";
-import {
-  formatFailureMemoryContent,
-  removeExactSyncedMemories,
-  removeSyncedMemories,
-  replaceSyncedMemories,
-  syncMemoryEntry,
-} from "../store/sqlite-memory-store.js";
+import type { DatabaseManager } from "../store/db.js";
 import type { MemoryCategory, MemoryConfig, MemoryResult, ThinkingLevel } from "../types.js";
 
 export interface ReviewMemoryOperation {
@@ -38,13 +30,21 @@ export interface DirectReviewResult {
   error?: string;
 }
 
-export interface RunDirectBackgroundReviewOptions {
+export interface RunDirectMemoryCompletionOptions {
   userPrompt: string;
+  systemPrompt: string;
   config: Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride">;
   timeoutMs?: number;
   signal?: AbortSignal;
 }
 
+/** Shared transport gate: review/flush/consolidation/correction all default to
+ * the in-process direct completion path and fall back to a `pi -p` subprocess
+ * only on failure, unless the user forces `reviewTransport: "subprocess"`. */
+export function usesDirectTransport(config: Pick<MemoryConfig, "reviewTransport">): boolean {
+  return (config.reviewTransport ?? "direct") === "direct";
+}
+
 type ReviewLlmConfig = Pick<MemoryConfig, "llmModelOverride" | "llmThinkingOverride">;
 
 function findExactModelReferenceMatch(modelReference: string, availableModels: Model<Api>[]): Model<Api> | undefined {
@@ -203,107 +203,12 @@ export function parseReviewOperations(text: string): ReviewMemoryOperation[] | n
   return parsed;
 }
 
-function sqliteProjectFor(
-  rawTarget: ReviewMemoryOperation["target"],
-  projectName?: string | null,
-): string | null | undefined {
-  if (rawTarget === "project") return projectName?.trim() || null;
-  if (rawTarget === "memory" || rawTarget === "user" || rawTarget === "failure") return null;
-  return undefined;
-}
-
-function sqliteTargetFor(rawTarget: ReviewMemoryOperation["target"]): "memory" | "user" | "failure" {
-  return rawTarget === "user" ? "user" : rawTarget === "failure" ? "failure" : "memory";
-}
-
-async function syncAdd(
-  rawTarget: ReviewMemoryOperation["target"],
-  content: string,
-  category: MemoryCategory | undefined,
-  failureReason: string | undefined,
-  dbManager: DatabaseManager | null,
-  projectName?: string | null,
-): Promise<void> {
-  if (!dbManager) return;
-
-  const sqliteTarget = sqliteTargetFor(rawTarget);
-  const sqliteProject = sqliteProjectFor(rawTarget, projectName);
-
-  if (rawTarget === "failure") {
-    const failureCategory = category ?? "failure";
-    syncMemoryEntry(dbManager, {
-      content: formatFailureMemoryContent(content, {
-        category: failureCategory,
-        failureReason,
-      }),
-      target: "failure",
-      project: sqliteProject ?? null,
-      category: failureCategory,
-      failureReason,
-    });
-    return;
-  }
-
-  syncMemoryEntry(dbManager, {
-    content,
-    target: sqliteTarget,
-    project: sqliteProject ?? null,
-  });
-}
-
-async function syncReplace(
-  rawTarget: ReviewMemoryOperation["target"],
-  oldText: string,
-  newContent: string,
-  dbManager: DatabaseManager | null,
-  projectName?: string | null,
-): Promise<void> {
-  if (!dbManager) return;
-  replaceSyncedMemories(dbManager, oldText, {
-    content: newContent,
-    target: sqliteTargetFor(rawTarget),
-    project: sqliteProjectFor(rawTarget, projectName),
-  });
-}
-
-async function syncRemove(
-  rawTarget: ReviewMemoryOperation["target"],
-  oldText: string,
-  dbManager: DatabaseManager | null,
-  projectName?: string | null,
-): Promise<void> {
-  if (!dbManager) return;
-  removeSyncedMemories(dbManager, oldText, {
-    target: sqliteTargetFor(rawTarget),
-    project: sqliteProjectFor(rawTarget, projectName),
-  });
-}
-
-async function syncEvictions(
-  rawTarget: ReviewMemoryOperation["target"],
-  evictedEntries: string[] | undefined,
-  dbManager: DatabaseManager | null,
-  projectName?: string | null,
-): Promise<void> {
-  if (!dbManager || !evictedEntries?.length) return;
-  for (const entry of evictedEntries) {
-    try {
-      removeExactSyncedMemories(dbManager, entry, {
-        target: sqliteTargetFor(rawTarget),
-        project: sqliteProjectFor(rawTarget, projectName),
-      });
-    } catch {
-      // best effort
-    }
-  }
-}
-
 export async function applyReviewOperations(
   store: MemoryStore,
   projectStore: MemoryStore | null,
   operations: ReviewMemoryOperation[],
-  dbManager: DatabaseManager | null = null,
-  projectName?: string | null,
+  _dbManager: DatabaseManager | null = null,
+  _projectName?: string | null,
 ): Promise<ApplyReviewOperationsResult> {
   let appliedCount = 0;
   let skippedCount = 0;
@@ -332,7 +237,6 @@ export async function applyReviewOperations(
             failureReason: op.failure_reason,
           });
           if (result.success) {
-            await syncAdd(rawTarget, op.content, category, op.failure_reason, dbManager, projectName);
             appliedCount++;
           } else {
             skippedCount++;
@@ -340,8 +244,6 @@ export async function applyReviewOperations(
         } else {
           result = await activeStore.add(memoryTarget, op.content);
           if (result.success) {
-            await syncEvictions(rawTarget, result.evicted_entries, dbManager, projectName);
-            await syncAdd(rawTarget, op.content, undefined, undefined, dbManager, projectName);
             appliedCount++;
           } else {
             skippedCount++;
@@ -356,7 +258,6 @@ export async function applyReviewOperations(
         }
         result = await activeStore.replace(memoryTarget, op.old_text, op.content);
         if (result.success) {
-          await syncReplace(rawTarget, op.old_text, op.content, dbManager, projectName);
           appliedCount++;
         } else {
           skippedCount++;
@@ -370,7 +271,6 @@ export async function applyReviewOperations(
         }
         result = await activeStore.remove(memoryTarget, op.old_text);
         if (result.success) {
-          await syncRemove(rawTarget, op.old_text, dbManager, projectName);
           appliedCount++;
         } else {
           skippedCount++;
@@ -379,7 +279,9 @@ export async function applyReviewOperations(
       }
       default:
         skippedCount++;
+        continue;
     }
+
   }
 
   return { appliedCount, skippedCount };
@@ -395,11 +297,11 @@ function responseText(content: unknown): string {
     .join("\n");
 }
 
-export async function runDirectBackgroundReview(
+export async function runDirectMemoryCompletion(
   ctx: Pick<ExtensionContext, "model" | "modelRegistry">,
   store: MemoryStore,
   projectStore: MemoryStore | null,
-  options: RunDirectBackgroundReviewOptions,
+  options: RunDirectMemoryCompletionOptions,
   dbManager: DatabaseManager | null = null,
   projectName?: string | null,
 ): Promise<DirectReviewResult> {
@@ -435,7 +337,7 @@ export async function runDirectBackgroundReview(
   try {
     const response = await completeSimple(
       model,
-      { systemPrompt: DIRECT_REVIEW_SYSTEM_PROMPT, messages: [userMessage] },
+      { systemPrompt: options.systemPrompt, messages: [userMessage] },
       buildDirectReviewCompletionOptions(
         model,
         { apiKey: auth.apiKey, headers: auth.headers, env: auth.env },
@@ -478,4 +380,4 @@ export async function runDirectBackgroundReview(
   } finally {
     clearTimeout(timeout);
   }
-}
+}