pi-famulus 0.1.8 → 0.1.9

package/bin/pi-famulus.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
-import { nativePackageName, resolveNativeManagerPath } from "../src/native-manager.js";
+import { ensureNativeManagerInstalled, nativePackageName } from "../src/native-manager.js";
 
-const binary = resolveNativeManagerPath();
+const binary = ensureNativeManagerInstalled();
 if (!binary) {
   const name = nativePackageName(process.platform, process.arch);
   console.error(name

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-famulus",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "description": "Keep your main agent's context clean: subagents on cheaper models, long bash in the background, and monitors that wake the agent. One Rust daemon runs the shells.",
   "keywords": [
     "pi-package",
@@ -54,12 +54,12 @@
     "pi-famulus": "./bin/pi-famulus.js"
   },
   "optionalDependencies": {
-    "pi-famulus-linux-x64": "0.1.8",
-    "pi-famulus-linux-arm64": "0.1.8",
-    "pi-famulus-darwin-x64": "0.1.8",
-    "pi-famulus-darwin-arm64": "0.1.8",
-    "pi-famulus-win-x64": "0.1.8",
-    "pi-famulus-win-arm64": "0.1.8"
+    "pi-famulus-linux-x64": "0.1.9",
+    "pi-famulus-linux-arm64": "0.1.9",
+    "pi-famulus-darwin-x64": "0.1.9",
+    "pi-famulus-darwin-arm64": "0.1.9",
+    "pi-famulus-win-x64": "0.1.9",
+    "pi-famulus-win-arm64": "0.1.9"
   },
   "pi": {
     "extensions": [

package/src/config.ts

@@ -7,7 +7,7 @@
 import { accessSync, constants, readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { delimiter, join, win32 } from "node:path";
-import { nativePackageName, resolveNativeManagerPath } from "./native-manager.js";
+import { describeNativeLookup, resolveNativeManagerPath } from "./native-manager.js";
 
 export interface FamulusConfig {
   /** Foreground budget for bash before auto-backgrounding (ms). */
@@ -290,11 +290,7 @@ export function describeManagerSearch(
   const tried: string[] = [];
   if (config.managerPath) tried.push(`config managerPath ${describeCandidate(config.managerPath)}`);
   if (env.PI_FAMULUS_MANAGER_PATH) tried.push(`PI_FAMULUS_MANAGER_PATH ${describeCandidate(env.PI_FAMULUS_MANAGER_PATH)}`);
-  const nativeName = nativePackageName(process.platform, process.arch);
-  const native = resolveNativeManagerPath();
-  tried.push(nativeName
-    ? `npm ${nativeName}${native ? ` ${native}` : " (missing or unusable, ignored)"}`
-    : `npm native manager (unsupported ${process.platform}/${process.arch})`);
+  tried.push(describeNativeLookup());
   tried.push(describeCandidate(join(home, "bin", MANAGER_FILE_NAME)));
   tried.push("pi-famulus on PATH");
   return tried.join("; ");

package/src/index.ts

@@ -24,6 +24,7 @@ import { createRegistryCommsHost } from "./comms/registry-host";
 import { createAgentMessageTool, createContactSupervisorTool } from "./comms/tools";
 import { registerReplyCommand } from "./comms/reply-command";
 import { describeManagerSearch, getFamulusHome, loadConfig, resolveManagerPath, resolveSubagentConfig } from "./config";
+import { ensureNativeManagerInstalled } from "./native-manager.js";
 import type { TaskExitInfo } from "./format";
 import { EXTENSION_VERSION, ManagerClient, type ManagerEvent, type TaskRecord } from "./manager-client";
 import { createMonitorTool, exitEventFromRecord, MonitorRegistry } from "./monitor";
@@ -99,7 +100,28 @@ function createFamulusRuntime(retiring?: Promise<unknown>): FamulusRuntime {
   registerBehaviorGuidelines(pi);
   const home = getFamulusHome();
   const config = loadConfig(home);
-  const managerPath = resolveManagerPath(config, home);
+  // pi update reinstalls extensions with npm. omit=optional drops the platform
+  // package during that install. Fetch it in the background so a slow or
+  // unreachable registry cannot stall extension startup.
+  let managerPath = resolveManagerPath(config, home);
+  let famulusInfo: {
+    extensionVersion: string;
+    managerPath: string | null;
+    describeSearch: () => string;
+    getClient: () => ManagerClient | null;
+  } | null = null;
+  const nativeRestore: Promise<string | null> = managerPath
+    ? Promise.resolve(managerPath)
+    : Promise.resolve()
+        .then(() => ensureNativeManagerInstalled())
+        .then((path) => {
+          if (path) {
+            managerPath = path;
+            if (famulusInfo) famulusInfo.managerPath = path;
+          }
+          return path;
+        })
+        .catch(() => null);
   const clock = realClock;
 
   // Claude-style transcript pills for notifications (TUI only; no-ops elsewhere).
@@ -392,12 +414,13 @@ function createFamulusRuntime(retiring?: Promise<unknown>): FamulusRuntime {
     clock,
     syncWithManager: deps.syncWithManager,
   });
-  registerFamulusCommand(pi, {
+  famulusInfo = {
     extensionVersion: EXTENSION_VERSION,
     managerPath,
     describeSearch: () => describeManagerSearch(config, home),
     getClient: () => client,
-  });
+  };
+  registerFamulusCommand(pi, famulusInfo);
   monitorRegistry.onChange(() => {
     for (const mon of monitorRegistry.listActive()) {
       const existing = workIndex.get(mon.taskId);
@@ -819,11 +842,18 @@ function createFamulusRuntime(retiring?: Promise<unknown>): FamulusRuntime {
     startFleetWidget(startCtx);
 
     // Connect in the background: a cold manager spawn must not stall the
-    // first prompt. Tools call ensureAvailable() lazily before use.
+    // first prompt. The cache install, if one is running, finishes first so a
+    // missing optional package is not reported before it can be restored.
+    // Tools call ensureAvailable() lazily before use.
     const c = client;
-    void c
-      .connect()
+    void nativeRestore
+      .then((path) => {
+        if (client !== c || !c) return null;
+        if (path) c.useManagerPath(path);
+        return c.connect();
+      })
       .then((ok) => {
+        if (ok === null || client !== c || !c) return;
         if (ok) {
           if (c.protocolLevel() >= 4) void reconcileAgentLeases(c).catch(() => {});
           return;

package/src/manager-client.ts

@@ -18,7 +18,7 @@ import { famulusPaths } from "./config";
 import { realClock, type Clock, type ClockTimer } from "./clock";
 
 const MAX_FRAME_BYTES = 4 * 1024 * 1024; // 4 MiB (§3.3)
-const EXTENSION_VERSION = "0.1.8";
+const EXTENSION_VERSION = "0.1.9";
 export { EXTENSION_VERSION };
 // 5: speaks per-kind agent admission and queued acquire.
 const PROTOCOL = 5;
@@ -285,7 +285,7 @@ function delay(clock: Clock, ms: number): Promise<void> {
 export class ManagerClient {
   private readonly home: string;
   private readonly sessionId: string;
-  private readonly managerPath: string | null;
+  private managerPath: string | null;
   private readonly piPid: number;
   private readonly cwd: string | undefined;
   private readonly log: (message: string) => void;
@@ -327,6 +327,11 @@ export class ManagerClient {
     return this.state === "connected";
   }
 
+  /** A binary found after construction. The first path wins. */
+  useManagerPath(path: string): void {
+    if (!this.managerPath) this.managerPath = path;
+  }
+
   /** Maximum protocol level reported by the connected manager; 0 means unknown/legacy. */
   protocolLevel(): number {
     return this.protocolLevel_;

package/src/native-manager.d.ts

@@ -3,7 +3,37 @@ export interface NativeManagerOptions {
   arch?: string;
   /** Node package resolver; tests can use createRequire from an isolated consumer. */
   resolve?: (specifier: string) => string;
+  /** Walk node_modules from this file instead of this module. */
+  from?: string;
+  /** Do not walk above this directory. */
+  stopAt?: string;
+  /** Versioned platform cache. `false` disables the cache lookup. */
+  cacheDir?: string | false;
+}
+export interface NativeCacheOptions {
+  version?: string;
+  platform?: string;
+  arch?: string;
+  env?: NodeJS.ProcessEnv;
+  home?: string;
+}
+export interface EnsureNativeOptions extends NativeManagerOptions {
+  env?: NodeJS.ProcessEnv;
+  npmCommand?: string;
+  spawn?: (
+    command: string,
+    args: string[],
+    options?: { timeout?: number; windowsHide?: boolean; encoding?: "utf8" },
+  ) => { status: number | null; error?: Error };
 }
 export function nativePackageName(platform: string, arch: string): string | null;
+export function nativeCacheDir(options?: NativeCacheOptions): string;
 /** Null when unsupported, missing, corrupt, non-executable, or a different version. */
 export function resolveNativeManagerPath(options?: NativeManagerOptions): string | null;
+/** `npm <package> <path>` or why the exact platform package was not usable. */
+export function describeNativeLookup(options?: NativeManagerOptions): string;
+/**
+ * Install the exact platform package into the versioned cache when it is missing.
+ * Returns the binary path, or null when it is still unusable.
+ */
+export function ensureNativeManagerInstalled(options?: EnsureNativeOptions): string | null;

package/src/native-manager.js

@@ -1,5 +1,9 @@
-import { accessSync, constants, readFileSync, statSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { accessSync, closeSync, constants, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync, writeSync } from "node:fs";
 import { createRequire } from "node:module";
+import { homedir } from "node:os";
+import { dirname, join, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
 import { getSystemErrorMap } from "node:util";
 
 // Filesystem failures mean this optional package is unusable, regardless of the syscall code.
@@ -11,8 +15,10 @@ const resolutionErrorCodes = new Set([
   "ERR_INVALID_MODULE_SPECIFIER", "ERR_UNSUPPORTED_RESOLVE_REQUEST",
 ]);
 
+const modulePath = fileURLToPath(import.meta.url);
 const packageVersion = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
 const resolveInstalled = createRequire(import.meta.url).resolve;
+const RESTORE_HINT = "not installed; run `npm install --include=optional pi-famulus`";
 
 /** The native optional packages are selected by npm's os/cpu constraints. */
 export function nativePackageName(platform, arch) {
@@ -23,20 +29,263 @@ export function nativePackageName(platform, arch) {
   return `pi-famulus-${packageOs}-${arch}`;
 }
 
+/**
+ * Cache for the exact platform package. `pi update --extensions` reinstalls
+ * with npm, and `omit=optional` (user or project npmrc) deletes optional
+ * dependencies during that install. The cache is outside the famulus runtime
+ * home and outside pi's package tree, so the next update does not remove it.
+ */
+export function nativeCacheDir({
+  version = packageVersion,
+  platform = process.platform,
+  arch = process.arch,
+  env = process.env,
+  home = homedir(),
+} = {}) {
+  const base = platform === "win32"
+    ? join(env.LOCALAPPDATA || join(home, "AppData", "Local"), "pi-famulus", "native")
+    : join(env.XDG_CACHE_HOME || join(home, ".cache"), "pi-famulus", "native");
+  return join(base, version, `${platform}-${arch}`);
+}
+
+function isIgnoredResolveError(error) {
+  return error instanceof SyntaxError || systemErrorCodes.has(error?.code) || resolutionErrorCodes.has(error?.code);
+}
+
+function executableFile(path) {
+  try {
+    const stat = statSync(path);
+    if (!stat.isFile() || stat.size <= 0) return false;
+    accessSync(path, constants.X_OK);
+    return true;
+  } catch (error) {
+    if (isIgnoredResolveError(error)) return false;
+    throw error;
+  }
+}
+
+/** Usable exact-version binary inside one package directory, or null. */
+function binaryInPackage(dir, name) {
+  let metadata;
+  try {
+    metadata = JSON.parse(readFileSync(join(dir, "package.json"), "utf8"));
+  } catch (error) {
+    if (isIgnoredResolveError(error)) return null;
+    throw error;
+  }
+  if (!metadata || metadata.name !== name || metadata.version !== packageVersion) return null;
+  const exported = metadata.exports?.["./bin/pi-famulus"];
+  const rel = typeof exported === "string" ? exported : metadata.main;
+  if (typeof rel !== "string" || !rel.startsWith("./") || rel.split("/").includes("..")) return null;
+  const binary = join(dir, rel);
+  const root = dir.endsWith(sep) ? dir : dir + sep;
+  if (!binary.startsWith(root)) return null;
+  return executableFile(binary) ? binary : null;
+}
+
+function walkNodeModules(startFile, stopAt) {
+  const dirs = [];
+  let dir = dirname(startFile);
+  const seen = new Set();
+  const boundary = stopAt ? resolve(stopAt) : null;
+  while (!seen.has(dir)) {
+    seen.add(dir);
+    dirs.push(join(dir, "node_modules"));
+    if (boundary && (dir === boundary || !dir.startsWith(boundary.endsWith(sep) ? boundary : boundary + sep))) break;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return dirs;
+}
+
+function legacyResolve(name, resolve) {
+  const metadata = JSON.parse(readFileSync(resolve(`${name}/package.json`), "utf8"));
+  if (!metadata || metadata.name !== name || metadata.version !== packageVersion) return null;
+  const binary = resolve(`${name}/bin/pi-famulus`);
+  const stat = statSync(binary);
+  if (!stat.isFile() || stat.size <= 0) return null;
+  accessSync(binary, constants.X_OK);
+  return binary;
+}
+
+function searchNative({ platform = process.platform, arch = process.arch, from, stopAt, cacheDir } = {}) {
+  const name = nativePackageName(platform, arch);
+  if (!name) return { name: null, path: null, detail: `unsupported ${platform}/${arch}` };
+  let start = from ?? modulePath;
+  if (!from) {
+    try { start = realpathSync(modulePath); } catch { /* keep the URL path */ }
+  }
+  const seen = new Set();
+  for (const nodeModules of walkNodeModules(start, stopAt)) {
+    const dir = join(nodeModules, name);
+    if (seen.has(dir)) continue;
+    seen.add(dir);
+    const binary = binaryInPackage(dir, name);
+    if (binary) return { name, path: binary, detail: "" };
+  }
+  // NODE_PATH and other resolver roots. An explicit `from` is a hermetic walk.
+  if (!from) {
+    try {
+      const resolved = dirname(resolveInstalled(`${name}/package.json`));
+      if (!seen.has(resolved)) {
+        const binary = binaryInPackage(resolved, name);
+        if (binary) return { name, path: binary, detail: "" };
+      }
+    } catch (error) {
+      if (!isIgnoredResolveError(error)) throw error;
+    }
+  }
+  if (cacheDir !== false) {
+    const dir = join(cacheDir ?? nativeCacheDir({ platform, arch }), "node_modules", name);
+    const binary = binaryInPackage(dir, name);
+    if (binary) return { name, path: binary, detail: "" };
+  }
+  return { name, path: null, detail: RESTORE_HINT };
+}
+
 /** Resolve the exact-version optional package without loading native executable bytes. */
-export function resolveNativeManagerPath({ platform = process.platform, arch = process.arch, resolve = resolveInstalled } = {}) {
+export function resolveNativeManagerPath({ platform = process.platform, arch = process.arch, resolve, from, stopAt, cacheDir } = {}) {
   const name = nativePackageName(platform, arch);
   if (!name) return null;
+  // Injected resolvers stay hermetic: tests use them to simulate one install tree.
+  if (resolve) {
+    try {
+      return legacyResolve(name, resolve);
+    } catch (error) {
+      if (isIgnoredResolveError(error)) return null;
+      throw error;
+    }
+  }
+  return searchNative({ platform, arch, from, stopAt, cacheDir }).path;
+}
+
+/** `npm <package> <path>` or why that exact platform package was not usable. */
+export function describeNativeLookup(options) {
+  const name = nativePackageName(options?.platform ?? process.platform, options?.arch ?? process.arch);
+  if (!name) return `npm native manager (unsupported ${options?.platform ?? process.platform}/${options?.arch ?? process.arch})`;
+  if (options?.resolve) {
+    const path = resolveNativeManagerPath(options);
+    return path ? `npm ${name} ${path}` : `npm ${name} (missing or unusable, ignored)`;
+  }
+  const found = searchNative(options);
+  return found.path ? `npm ${name} ${found.path}` : `npm ${name} (${found.detail})`;
+}
+
+function pidAlive(pid) {
   try {
-    const metadata = JSON.parse(readFileSync(resolve(`${name}/package.json`), "utf8"));
-    if (!metadata || metadata.name !== name || metadata.version !== packageVersion) return null;
-    const binary = resolve(`${name}/bin/pi-famulus`);
-    const stat = statSync(binary);
-    if (!stat.isFile() || stat.size <= 0) return null;
-    accessSync(binary, constants.X_OK);
-    return binary;
+    process.kill(pid, 0);
+    return true;
   } catch (error) {
-    if (error instanceof SyntaxError || systemErrorCodes.has(error?.code) || resolutionErrorCodes.has(error?.code)) return null;
-    throw error;
+    return error?.code === "EPERM";
+  }
+}
+
+function sleepMs(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/** A lock left by a dead process, or held far longer than one npm install, can be replaced. */
+function installLockIsStale(lockPath) {
+  try {
+    const age = Date.now() - statSync(lockPath).mtimeMs;
+    const pid = Number(readFileSync(lockPath, "utf8").trim());
+    if (Number.isInteger(pid) && pid > 0) return !pidAlive(pid) || age > 150_000;
+    return age > 5_000;
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * One npm install per cache directory. Another startup either waits until that
+ * install publishes a binary or takes over a stale lock. Returns the open lock,
+ * or null when waiting itself produced a usable binary.
+ */
+function acquireInstallLock(dir, lookup) {
+  mkdirSync(dir, { recursive: true });
+  const lockPath = `${dir}.install.lock`;
+  const deadline = Date.now() + 130_000;
+  while (Date.now() <= deadline) {
+    try {
+      const fd = openSync(lockPath, "wx");
+      try { writeSync(fd, String(process.pid)); } catch { /* the fd is still the lock */ }
+      return { fd, lockPath };
+    } catch (error) {
+      if (error?.code !== "EEXIST") throw error;
+      const ready = resolveNativeManagerPath(lookup);
+      if (ready) return null;
+      if (installLockIsStale(lockPath)) {
+        rmSync(lockPath, { force: true });
+        continue;
+      }
+      sleepMs(200);
+    }
+  }
+  return null;
+}
+
+function releaseInstallLock(lock) {
+  if (!lock) return;
+  try { closeSync(lock.fd); } catch { /* already closed */ }
+  rmSync(lock.lockPath, { force: true });
+}
+
+/**
+ * Install the exact platform package into the versioned cache when the
+ * optional dependency is missing. No-ops when a usable binary is already
+ * resolvable. `PI_FAMULUS_NO_NATIVE_FETCH=1`, `PI_OFFLINE=1`, and Vitest skip
+ * the install. Directory, manifest, and npm failures return null so startup
+ * degrades instead of throwing.
+ */
+export function ensureNativeManagerInstalled({
+  platform = process.platform,
+  arch = process.arch,
+  cacheDir,
+  env = process.env,
+  npmCommand,
+  spawn = spawnSync,
+  from,
+  stopAt,
+} = {}) {
+  const lookup = { platform, arch, cacheDir, from, stopAt };
+  try {
+    const existing = resolveNativeManagerPath(lookup);
+    if (existing) return existing;
+    if (!nativePackageName(platform, arch)) return null;
+    if (env.PI_FAMULUS_NO_NATIVE_FETCH === "1" || env.PI_OFFLINE === "1" || env.VITEST) return null;
+    if (cacheDir === false) return null;
+    const dir = cacheDir ?? nativeCacheDir({ platform, arch, env });
+    const lockedLookup = { ...lookup, cacheDir: dir };
+    const lock = acquireInstallLock(dir, lockedLookup);
+    if (!lock) return resolveNativeManagerPath(lockedLookup);
+    try {
+      const manifest = join(dir, "package.json");
+      try {
+        readFileSync(manifest);
+      } catch {
+        writeFileSync(manifest, `${JSON.stringify({ private: true, name: "pi-famulus-native-cache", version: "0.0.0" })}\n`);
+      }
+      const command = npmCommand ?? (platform === "win32" ? "npm.cmd" : "npm");
+      const name = nativePackageName(platform, arch);
+      let result;
+      try {
+        result = spawn(command, [
+          "install", `${name}@${packageVersion}`,
+          "--prefix", dir,
+          "--ignore-scripts", "--no-fund", "--no-audit", "--loglevel=error",
+        ], { timeout: 120_000, windowsHide: true, encoding: "utf8" });
+      } catch {
+        return resolveNativeManagerPath(lockedLookup);
+      }
+      const found = resolveNativeManagerPath(lockedLookup);
+      if (found) return found;
+      if (!result || result.error || result.status !== 0) return null;
+      return null;
+    } finally {
+      releaseInstallLock(lock);
+    }
+  } catch {
+    return null;
   }
 }