pi-orchestrator-config 4.5.0 → 4.6.1

package/README.md

@@ -65,6 +65,7 @@ Single extension that provides:
 | `/issue-review`                                                                                 | Review a GitHub issue spec and fix it                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | `/release [flags]`                                                                              | Create GitHub release with changelog and version bumping                                                                                                                                                                                                                                                                                                                                                                                                   |
 | `/review-local [branch]`                                                                        | Review local uncommitted or branch changes                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| `/qodo-review [--autofix] [--fast\|--deep] [--ticket <url>] [path ...]`                         | Review local changes with Qodo, attach session context, and optionally autofix (max 3 cycles)                                                                                                                                                                                                                                                                                                                                                            |
 | `/review-handler [url] [--autorabbit] [--autoqodo]`                                             | Process PR review comments, fix approved items                                                                                                                                                                                                                                                                                                                                                                                                             |
 | `/domain-model [focus area]`                                                                    | Scan codebase and build/update a CONTEXT.md domain glossary for consistent AI vocabulary                                                                                                                                                                                                                                                                                                                                                                   |
 | `/refine-review <url>`                                                                          | Refine and improve existing PR review comments                                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -76,7 +77,7 @@ Single extension that provides:
 | `/dream`                                                                                        | Run memory consolidation — extract, deduplicate, maintain topic-based memory                                                                                                                                                                                                                                                                                                                                                                               |
 | `/remember <what>`                                                                              | Save a memory for future sessions                                                                                                                                                                                                                                                                                                                                                                                                                          |
 | `/dream-auto`                                                                                   | Toggle automatic memory dreaming (every 3h + session end)                                                                                                                                                                                                                                                                                                                                                                                                  |
-| `/cron add\|list\|list-all\|remove [--persist]`                                                         | Schedule recurring tasks. Session jobs end with the session; `--persist` keeps a job across Pi sessions in this project. Add/remove mutations still work when leader election fails closed on an unsupported platform. For example: `/cron add every 2h check for new issues --persist` or `/cron remove persist:<uuid>`. `/cron list` opens an overlay for the selected jobs (or both when omitted). Persistent delivery is local at-least-once: a crash or leader recovery can repeat a run. |
+| `/cron add\|list\|list-all\|remove [--persist]`                                                         | Schedule recurring tasks. Session jobs end with the session; `--persist` keeps a job across Pi sessions in this project. Add/remove mutations still work when leader election fails closed on an unsupported platform. For example: `/cron add every 2h check for new issues --persist` or `/cron remove persist:<uuid>`. `/cron list` opens an overlay for the selected jobs (or both when omitted). Persistent delivery is local at-least-once: a crash or leader recovery can repeat a run. Project cron writes use a stable OS `flock` (included in Docker via `util-linux`) across the entire transaction; native installations need `flock` and a shared filesystem that supports advisory locks. A busy or unavailable lock fails closed after two seconds. **Upgrade gate (enforced):** stop and drain every old Pi process writing crons on the shared mount before starting a version with this kernel lock; old directory-lock writers do not honor `flock` and must not resume. If a legacy `crons.json.mutation.lock` directory exists, new writes fail closed even if its PID is invisible or its heartbeat expired. Only after verifying every old writer cannot resume may an operator remove that stale directory (not `crons.json` or cron tasks). Never remove it merely because a write timed out. |
 | `/async-kill [name\|id\|all]`                                                                   | Kill async agents (overlay picker or by name/id)                                                                                                                                                                                                                                                                                                                                                                                                           |
 | `/pi-config-settings [project\|global]`                                                         | Interactive TUI settings editor — browse all pi-config settings with scope indicators, smart pickers for providers/models, and immediate save                                                                                                                                                                                                                                                                                                              |
 | `/status`                                                                                       | Unified session snapshot — async agents, cron tasks, git branch, context usage                                                                                                                                                                                                                                                                                                                                                                             |
@@ -466,6 +467,13 @@ settings; it reads the project locally and stores its regenerable graph in `./gr
 Native installations require Node.js 22+, npm 12+, Python 3, `make`, and G++; the
 installer uses the same audited strict installation and failure remains non-fatal.
 
+**Persistent cron locking:** The image includes `flock` from `util-linux` for kernel-backed,
+cross-container store mutations on shared mounts. The shared filesystem must support
+advisory locks. Before deploying this version alongside old Pi containers, stop and
+drain every legacy cron writer. An existing legacy mutation-lock directory blocks
+new writes until an operator verifies old writers cannot resume and removes only
+that directory. Do not stop running containers merely to test this PR.
+
 **CLI provider binaries (optional `cli_agents`):** The image installs the CLIs used by
 `cli-*` providers — `claude` (Claude Code), `gemini` (`@google/gemini-cli`), and
 `agent` (Cursor Agent CLI). Enable with `cli_agents` in settings (e.g.
@@ -710,6 +718,7 @@ PI_PIDIFF_ENABLE=false pi
 | `-v "$HOME/.claude/credentials.json":"$HOME/.claude/credentials.json":ro`                                                       | Claude CLI auth (for acpx-claude / cli-claude models) |
 | `-v "$HOME/.config/glab-cli":"$HOME/.config/glab-cli":ro`                                                                       | GitLab CLI config (auth tokens, host settings)        |
 | `-v "$HOME/.coderabbit":"$HOME/.coderabbit":rw`                                                                                 | CodeRabbit CLI auth and review data                   |
+| `-v "$HOME/.qodo":"$HOME/.qodo":rw`                                                                                             | Qodo CLI auth directory (mount at the container user's `$HOME/.qodo`; with `PI_HOST_USER`, that is `/home/$PI_HOST_USER/.qodo`). Create it on the host first with `mkdir -p "$HOME/.qodo"`; inside the container run `qodo read whoami` and, if signed out, `qodo login`. Host OS-keychain credentials may not work inside the container. |
 | `-v "$HOME/screenshots":"$HOME/screenshots"`                                                                                    | Share screenshots/images with the agent               |
 | `-v /var/run/docker.sock:/var/run/docker.sock:ro` + `--group-add $(stat -c '%g' /var/run/docker.sock)`                          | Docker container inspection via `docker-safe`         |
 | `-v /var/run/podman/podman.sock:/var/run/podman/podman.sock:ro`                                                                 | Podman container inspection via `docker-safe`         |
@@ -740,6 +749,7 @@ PI_PIDIFF_ENABLE=false pi
 | `jq`                | JSON processing                                                                                                                                                                                                                                                                                                                |
 | `rg` (ripgrep)      | Fast recursive search                                                                                                                                                                                                                                                                                                          |
 | `cr`                | CodeRabbit CLI — local AI code reviews                                                                                                                                                                                                                                                                                         |
+| `qodo`              | Qodo CLI — local code reviews (`qodo --version`; mount `~/.qodo` as above and authenticate inside the container if needed)                                                                                                                                                                                                   |
 | `curl`              | HTTP requests                                                                                                                                                                                                                                                                                                                  |
 
 #### `myk-pi-tools` subcommands
@@ -904,6 +914,8 @@ A standalone HTTP sidecar wrapping the Pi SDK, living under `packages/pi-sidecar
 
 CLI commands: `npx pi-sidecar` (start server), `npx pi-sidecar-start` (background start/stop).
 
+`GET /providers` lists registered providers and whether each supports a session API key, even when no models or credentials are available. The Python client exposes this as `await SidecarClient.get_providers()`.
+
 See [`packages/pi-sidecar/README.md`](packages/pi-sidecar/README.md) for full documentation.
 See [`packages/pi-sidecar/CONSUMER-GUIDE.md`](packages/pi-sidecar/CONSUMER-GUIDE.md) for integration best practices.
 

package/extensions/openai-compatible-discovery/index.ts

@@ -1,19 +1,17 @@
-import type { AuthResult, Model, Provider } from "@earendil-works/pi-ai";
+import type { AuthResult, Model, Provider, RefreshModelsContext } from "@earendil-works/pi-ai";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { createRuntimeProvider } from "../shared/create-runtime-provider.js";
 import { createLogger } from "../shared/logger.js";
 import {
-  OpenAiCompatibleDiscoveryCache,
-  buildLiteLlmCapabilitiesUrl,
+  buildOpenAiCompatibleCapabilitiesUrl,
   buildOpenAiCompatibleModelsRequest,
-  enrichLiteLlmReasoning,
+  enrichOpenAiCompatibleReasoning,
   findEligibleOpenAiCompatibleProviderConfigsResult,
   formatOpenAiCompatibleDiscoverySummary,
   materializeOpenAiCompatibleModels,
   openAiCompatibleConnectionFingerprint,
   redactOpenAiCompatibleDiagnostic,
   resolveStaticOpenAiCompatibleHeaders,
-  type LiteLlmCapabilityRecord,
+  type OpenAiCompatibleCapabilityRecord,
   type OpenAiCompatibleModelRecord,
   type ResolvedOpenAiCompatibleConnection,
 } from "../shared/openai-compatible-discovery.js";
@@ -31,16 +29,14 @@ function responseRecords(payload: unknown): OpenAiCompatibleModelRecord[] {
   });
 }
 
-function capabilityRecords(payload: unknown): LiteLlmCapabilityRecord[] {
+function capabilityRecords(payload: unknown): OpenAiCompatibleCapabilityRecord[] {
   if (!payload || typeof payload !== "object" || !Array.isArray((payload as { data?: unknown }).data))
-    throw new Error("LiteLLM capability response must contain a data array");
-  return (payload as { data: unknown[] }).data.filter(
-    (record): record is LiteLlmCapabilityRecord => Boolean(record) && typeof record === "object" && !Array.isArray(record),
+    throw new Error("OpenAI-compatible capability response must contain a data array");
+  const records = (payload as { data: unknown[] }).data.filter(
+    (record): record is OpenAiCompatibleCapabilityRecord => Boolean(record) && typeof record === "object" && !Array.isArray(record),
   );
-}
-
-function isLiteLlmProvider(sourceProviderId: string, source: Provider): boolean {
-  return `${sourceProviderId} ${source.name ?? ""}`.toLowerCase().includes("litellm");
+  log.debug("parsed capability records", { count: records.length });
+  return records;
 }
 
 function resolvedConnection(source: Provider, auth?: AuthResult, staticHeaders?: Record<string, string>): ResolvedOpenAiCompatibleConnection {
@@ -58,100 +54,86 @@ function combineModels(sourceProviderId: string, staticModels: readonly Model[],
   return [...staticModels, ...discovered];
 }
 
-async function discoverProvider(
+async function refreshProviderModels(
   pi: ExtensionAPI,
   ctx: any,
   sourceProviderId: string,
-  rawStaticHeaders: Record<string, string> | undefined,
-  cache: OpenAiCompatibleDiscoveryCache<OpenAiCompatibleModelRecord>,
-  signal: AbortSignal,
-  isCurrent: () => boolean,
-  registeredSources: Set<string>,
-): Promise<void> {
-  if (!isCurrent()) return;
-  const source = ctx.modelRegistry.getProvider(sourceProviderId) as Provider | undefined;
-  if (!source) {
-    log.warn(`${sourceProviderId}: configured provider was not resolved`);
-    return;
-  }
-  let auth: AuthResult | undefined;
-  let staticHeaders: Record<string, string> | undefined;
-  try {
-    auth = await ctx.modelRegistry.getProviderAuth(sourceProviderId);
-    staticHeaders = await resolveStaticOpenAiCompatibleHeaders(rawStaticHeaders);
-  } catch {
-    log.warn(`${sourceProviderId}: authentication or configured headers could not be resolved`);
-    return;
-  }
-  const connection = resolvedConnection(source, auth, staticHeaders);
-  let request;
+  source: Provider,
+  staticModels: readonly Model[],
+  refresh: RefreshModelsContext,
+  appendSummary: () => boolean,
+  discoverModelCapabilities: boolean,
+  rawStaticHeaders?: Record<string, string>,
+  snapshot: { discovered: Model[]; fingerprint?: string; generation: number },
+  lifecycle: { signal: AbortSignal; isCurrent: () => boolean },
+): Promise<Model[]> {
+  const generation = ++snapshot.generation;
+  const active = () => {
+    const current = lifecycle.isCurrent();
+    log.debug("discovery refresh eligibility", { provider: sourceProviderId, generation, current, aborted: refresh.signal.aborted, latest: generation === snapshot.generation });
+    return current && !refresh.signal.aborted && generation === snapshot.generation;
+  };
+  let connection: ResolvedOpenAiCompatibleConnection = { baseUrl: source.baseUrl };
+  let scopeResolved = false;
   try {
-    request = buildOpenAiCompatibleModelsRequest(connection);
-  } catch (error) {
-    log.warn(`${sourceProviderId}: discovery inactive`, redactOpenAiCompatibleDiagnostic(error instanceof Error ? error.message : String(error), connection));
-    return;
-  }
-  const records = await cache.get(async () => {
-    try {
-      const response = await fetch(request.url, {
-        headers: request.headers,
-        redirect: "error",
-        signal: AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]),
-      });
-      if (!response.ok) throw new Error(`OpenAI-compatible /v1/models returned HTTP ${response.status}`);
-      const records = responseRecords(await response.json());
-      if (!isLiteLlmProvider(sourceProviderId, source)) return records;
+    const auth = await ctx.modelRegistry.getProviderAuth(sourceProviderId) as AuthResult | undefined;
+    const staticHeaders = await resolveStaticOpenAiCompatibleHeaders(rawStaticHeaders);
+    connection = resolvedConnection(ctx.modelRegistry.getProvider(sourceProviderId) ?? source, auth, staticHeaders);
+    if (!active()) return [...staticModels];
+    const fingerprint = openAiCompatibleConnectionFingerprint(connection);
+    if (snapshot.fingerprint !== fingerprint) {
+      snapshot.discovered = [];
+      snapshot.fingerprint = fingerprint;
+      log.debug("discovery connection scope changed", { provider: sourceProviderId });
+    }
+    scopeResolved = true;
+    if (!refresh.allowNetwork) return [...staticModels, ...snapshot.discovered];
+    const request = buildOpenAiCompatibleModelsRequest(connection);
+    const response = await fetch(request.url, {
+      headers: request.headers, redirect: "error",
+      signal: AbortSignal.any([refresh.signal, lifecycle.signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]),
+    });
+    if (!response.ok) throw new Error(`OpenAI-compatible /v1/models returned HTTP ${response.status}`);
+    let records = responseRecords(await response.json());
+
+    if (discoverModelCapabilities) {
       try {
-        const capabilityResponse = await fetch(buildLiteLlmCapabilitiesUrl(request.url), {
+        const capabilityResponse = await fetch(buildOpenAiCompatibleCapabilitiesUrl(request.url), {
           headers: request.headers,
           redirect: "error",
-          signal: AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]),
+          signal: AbortSignal.any([refresh.signal, lifecycle.signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]),
         });
-        if (!capabilityResponse.ok) throw new Error(`LiteLLM capability endpoint returned HTTP ${capabilityResponse.status}`);
-        return enrichLiteLlmReasoning(records, capabilityRecords(await capabilityResponse.json()));
+        if (!capabilityResponse.ok) throw new Error(`OpenAI-compatible capability endpoint returned HTTP ${capabilityResponse.status}`);
+        records = enrichOpenAiCompatibleReasoning(records, capabilityRecords(await capabilityResponse.json()));
       } catch (error) {
+        if (!active()) return [...staticModels];
         log.warn(`${sourceProviderId}: reasoning capability enrichment unavailable`, redactOpenAiCompatibleDiagnostic(error instanceof Error ? error.message : String(error), connection));
-        return records;
       }
-    } catch (error) {
-      log.warn(`${sourceProviderId}: discovery refresh failed`, redactOpenAiCompatibleDiagnostic(error instanceof Error ? error.message : String(error), connection));
-      throw error;
     }
-  }, { cacheKey: openAiCompatibleConnectionFingerprint(connection) });
-  if (!isCurrent() || !records.hasSnapshot) return;
-  if (records.stale) log.warn(`${sourceProviderId}: refresh failed; using last-known-good discovery result`);
 
-  const staticModels = (ctx.modelRegistry.getAll() as Model[])
-    .filter((model) => model.provider === sourceProviderId);
-  const models = combineModels(sourceProviderId, staticModels, records.models, request.streamBaseUrl);
-  try {
-    const provider = await createRuntimeProvider({
-      id: sourceProviderId,
-      name: source.name,
-      baseUrl: request.streamBaseUrl,
-      auth: source.auth,
-      models,
-      api: {
-        stream: source.stream.bind(source),
-        streamSimple: source.streamSimple.bind(source),
-      },
-    });
-    if (!isCurrent()) return;
-    // Pi composes this native provider with the existing models.json entry. The
-    // source key is therefore both the ordinary picker label and routing key.
-    pi.registerProvider(provider);
-    registeredSources.add(sourceProviderId);
-    try {
-      pi.appendEntry<OpenAiCompatibleDiscoverySummary>(
-        "openai-compatible-discovery-summary",
-        { summary: formatOpenAiCompatibleDiscoverySummary(sourceProviderId, records.models.length) },
-      );
-    } catch (error) {
-      log.warn(`${sourceProviderId}: discovery summary append failed`, error instanceof Error ? error.name : typeof error);
+    if (!active()) return [...staticModels];
+    const models = combineModels(sourceProviderId, staticModels, records, request.streamBaseUrl);
+    snapshot.discovered = models.slice(staticModels.length);
+    if (active() && appendSummary()) {
+      try {
+        pi.appendEntry<OpenAiCompatibleDiscoverySummary>(
+          "openai-compatible-discovery-summary",
+          { summary: formatOpenAiCompatibleDiscoverySummary(sourceProviderId, records.length) },
+        );
+      } catch (error) {
+        log.warn(`${sourceProviderId}: discovery summary append failed`, error instanceof Error ? error.name : typeof error);
+      }
     }
-    log.info(`${sourceProviderId}: registered ${records.models.length} discovered model(s) on configured provider`);
+    log.info(`${sourceProviderId}: refreshed ${records.length} discovered model(s) on configured provider`);
+    return models;
   } catch (error) {
-    log.warn(`${sourceProviderId}: provider augmentation failed`, redactOpenAiCompatibleDiagnostic(error instanceof Error ? error.message : String(error), connection));
+    if (active())
+      log.warn(`${sourceProviderId}: discovery refresh failed`, scopeResolved
+        ? redactOpenAiCompatibleDiagnostic(error instanceof Error ? error.message : String(error), connection)
+        : { phase: "connection resolution", cause: error instanceof Error ? error.name : typeof error });
+    if (!active()) return [...staticModels];
+    if (!scopeResolved) { snapshot.discovered = []; snapshot.fingerprint = undefined; }
+    return [...staticModels, ...snapshot.discovered];
   }
 }
 
@@ -176,16 +158,14 @@ export default function (pi: ExtensionAPI) {
     );
   }
 
-  const cache = new OpenAiCompatibleDiscoveryCache<OpenAiCompatibleModelRecord>();
   const registeredSources = new Set<string>();
-  const controllers = new Set<AbortController>();
-  let active = true;
-  let generation = 0;
-
+  let lifecycleGeneration = 0;
+  let lifecycleController = new AbortController();
   const restoreSources = () => {
+    lifecycleController.abort();
+    lifecycleGeneration++;
     for (const id of registeredSources) {
       try {
-        // Removing our native overlay makes Pi recompose the unchanged static provider.
         pi.unregisterProvider(id);
       } catch (error) {
         log.warn("provider augmentation cleanup failed", { provider: id, cause: error instanceof Error ? error.name : typeof error });
@@ -194,39 +174,68 @@ export default function (pi: ExtensionAPI) {
     }
   };
 
-  pi.on("session_shutdown", () => {
-    active = false;
-    generation += 1;
-    for (const controller of controllers) controller.abort();
-    controllers.clear();
-    cache.clear();
-    restoreSources();
-  });
+  pi.on("session_shutdown", restoreSources);
 
   pi.on("session_start", async (_event, ctx) => {
-    if (ctx?.mode !== "tui" || ctx?.hasUI !== true) return;
-    active = true;
-    generation += 1;
-    for (const controller of controllers) controller.abort();
-    controllers.clear();
-    cache.clear();
     restoreSources();
-    const currentGeneration = generation;
-    const controller = new AbortController();
-    controllers.add(controller);
-    const isCurrent = () => active && currentGeneration === generation;
+    lifecycleController = new AbortController();
+    const generation = lifecycleGeneration;
+    const lifecycle = {
+      signal: lifecycleController.signal,
+      isCurrent: () => {
+        const current = generation === lifecycleGeneration;
+        log.debug("discovery lifecycle eligibility", { generation, lifecycleGeneration, current });
+        return current;
+      },
+    };
     const configResult = findEligibleOpenAiCompatibleProviderConfigsResult();
     if (configResult.providers.length === 0) {
       const diagnostic = { modelsConfig: configResult.status };
       if (configResult.status === "unreadable" || configResult.status === "malformed")
         log.warn("inactive: OpenAI-compatible discovery configuration unavailable", diagnostic);
       else log.debug("inactive: no opted-in OpenAI-compatible providers", diagnostic);
-      controllers.delete(controller);
       return;
     }
-    await Promise.all(configResult.providers.map(({ id, headers }) =>
-      discoverProvider(pi, ctx, id, headers, cache, controller.signal, isCurrent, registeredSources),
-    ));
-    controllers.delete(controller);
+
+    let startupRefresh = ctx?.mode === "tui" && ctx?.hasUI === true;
+    const registrationRefreshes: Promise<void>[] = [];
+    const activeProviders: string[] = [];
+    for (const { id, headers, discoverModelCapabilities } of configResult.providers) {
+      const source = ctx.modelRegistry.getProvider(id) as Provider | undefined;
+      if (!source) {
+        log.warn(`${id}: configured provider was not resolved`);
+        continue;
+      }
+      const staticModels = (ctx.modelRegistry.getAll() as Model[])
+        .filter((model) => model.provider === id);
+      const snapshot = { discovered: [] as Model[], fingerprint: undefined as string | undefined, generation: 0 };
+      // Registration starts an unawaited offline refresh. Its first callback signals
+      // that it has acquired its generation, so the network pass can safely supersede it.
+      let registrationStarted!: () => void;
+      registrationRefreshes.push(new Promise<void>((resolve) => { registrationStarted = resolve; }));
+      pi.registerProvider(id, {
+        refreshModels: (refresh) => {
+          log.debug("provider discovery refresh requested", { provider: id, generation: snapshot.generation + 1, allowNetwork: refresh.allowNetwork, aborted: refresh.signal.aborted });
+          registrationStarted();
+          return refreshProviderModels(
+            pi, ctx, id, source, staticModels, refresh, () => startupRefresh,
+            discoverModelCapabilities, headers, snapshot, lifecycle,
+          );
+        },
+      });
+      registeredSources.add(id);
+      activeProviders.push(id);
+    }
+    try {
+      await Promise.all(registrationRefreshes);
+      log.debug("registration offline refreshes started", { providers: activeProviders });
+      await ctx.modelRegistry.refresh({
+        providers: activeProviders,
+        allowNetwork: true,
+        force: true,
+      });
+    } finally {
+      startupRefresh = false;
+    }
   });
 }

package/extensions/orchestrator/cron-store.ts

@@ -1,7 +1,7 @@
 /** Durable cron storage and local-filesystem leader locks. Delivery is at-least-once. */
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { execFileSync } from "node:child_process";
+import { execFileSync, spawnSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { parseProcStartTime } from "./utils.js";
 import { createLogger } from "../shared/logger.js";
@@ -21,7 +21,7 @@ export interface DurableCronTask {
   nextRun?: number;
 }
 export interface DurableCronEnvelope { version: 1; tasks: DurableCronTask[]; }
-export interface CronLockOwner { pid: number; process_start_token: string; instance_id: string; heartbeat_at: string; fd?: number; }
+export interface CronLockOwner { pid: number; process_start_token: string; instance_id: string; heartbeat_at: string; pid_namespace?: string; fd?: number; }
 
 type StartTokenDeps = { readFileSync: (path: string, encoding: BufferEncoding) => string; execFileSync: (file: string, args: readonly string[], options: { encoding: BufferEncoding; windowsHide: boolean }) => string; platform: NodeJS.Platform };
 const startTokenDeps: StartTokenDeps = { readFileSync: fs.readFileSync, execFileSync, platform: process.platform };
@@ -52,18 +52,45 @@ function ownerPath(dir: string) { return path.join(dir, "owner.json"); }
 /** True only when the OS can prove this record cannot still own the lock. */
 function ownerIsProvenDead(owner: CronLockOwner): boolean {
   if (!owner?.pid || !Number.isInteger(owner.pid) || owner.pid < 1) return false;
+  // A PID from another namespace (or a legacy record without namespace) is
+  // not evidence of death, even when kill(pid, 0) reports ESRCH.
+  const namespace = processPidNamespace();
+  if (namespace ? !owner.pid_namespace || owner.pid_namespace !== namespace : owner.pid_namespace?.startsWith("pid:[")) {
+    log.debug("cron_lock_owner_foreign", { pid: owner.pid, instance_id: owner.instance_id, namespace: owner.pid_namespace });
+    return false;
+  }
   try {
     process.kill(owner.pid, 0);
   } catch (error: any) {
     // EPERM means the process exists but is not signalable; all other failures
     // are treated conservatively except the OS's definitive "no such process".
-    return error?.code === "ESRCH";
+    const dead = error?.code === "ESRCH";
+    log.debug("cron_lock_owner_pid_check", { pid: owner.pid, instance_id: owner.instance_id, dead, reason: error?.code });
+    return dead;
   }
   if (!owner.process_start_token || owner.process_start_token === "unknown") return false;
   const token = resolveProcessStartToken(owner.pid);
   // An unavailable identity source fails closed. A different issued token
   // proves PID reuse, so this record belongs to a dead process.
-  return token !== null && token !== owner.process_start_token;
+  const dead = token !== null && token !== owner.process_start_token;
+  log.debug("cron_lock_owner_token_check", { pid: owner.pid, instance_id: owner.instance_id, dead });
+  return dead;
+}
+function processPidNamespace(): string | null {
+  try { return fs.readlinkSync("/proc/self/ns/pid"); }
+  catch (error: any) { log.debug("cron_pid_namespace_unavailable", { pid: process.pid, reason: error?.code || "read_error" }); return null; }
+}
+function leaseIsExpired(owner: CronLockOwner): boolean {
+  const ageMs = Date.now() - Date.parse(owner.heartbeat_at);
+  const expired = Number.isFinite(ageMs) && ageMs >= STALE_LOCK_MS;
+  log.debug("cron_lock_lease_age", { instance_id: owner.instance_id, age_ms: ageMs, expired });
+  return expired;
+}
+function canReclaim(owner: CronLockOwner): boolean {
+  const dead = ownerIsProvenDead(owner);
+  const expired = leaseIsExpired(owner);
+  log.debug("cron_lock_reclaim_decision", { instance_id: owner.instance_id, dead, expired });
+  return dead || expired;
 }
 function readOwner(dir: string): CronLockOwner | null {
   try {
@@ -74,7 +101,9 @@ function readOwner(dir: string): CronLockOwner | null {
   } catch (error: any) { log.debug("cron_lock_owner_unreadable", { dir, reason: error?.code || "parse_error" }); return null; }
 }
 function sameOwner(a: CronLockOwner | null, b: CronLockOwner) {
-  return !!a && a.instance_id === b.instance_id && a.pid === b.pid && a.process_start_token === b.process_start_token;
+  const same = !!a && a.instance_id === b.instance_id && a.pid === b.pid && a.process_start_token === b.process_start_token && a.pid_namespace === b.pid_namespace;
+  log.debug("cron_lock_owner_compare", { instance_id: b.instance_id, current_instance_id: a?.instance_id, same });
+  return same;
 }
 function serializedOwner(owner: CronLockOwner) {
   const { fd: _fd, ...record } = owner;
@@ -83,24 +112,30 @@ function serializedOwner(owner: CronLockOwner) {
 function claimIsOld(claim: string) {
   try {
     const ageMs = Date.now() - fs.statSync(claim).mtimeMs;
-    const old = ageMs >= STALE_LOCK_MS;
-    log.debug("cron_lock_claim_age", { claim, age_ms: ageMs, old });
+    const file = ownerPath(claim);
+    const ownerAgeMs = fs.existsSync(file) ? Date.now() - fs.statSync(file).mtimeMs : ageMs;
+    const old = ageMs >= STALE_LOCK_MS && ownerAgeMs >= STALE_LOCK_MS;
+    log.debug("cron_lock_claim_age", { claim, age_ms: ageMs, owner_age_ms: ownerAgeMs, old });
     return old;
   } catch (error: any) { log.warn("cron_lock_claim_stat_failed", { claim, reason: error?.code || "stat_error" }); return false; }
 }
 function restoreClaim(claim: string, dir: string) {
+  if (fs.existsSync(dir)) {
+    log.warn("cron_lock_claim_restore_failed", { claim, dir, reason: "canonical_lock_exists" });
+    return;
+  }
   try { fs.renameSync(claim, dir); } catch { log.warn("cron_lock_claim_restore_failed", { claim, dir }); }
 }
 /** Read the canonical record first; detach and delete only that proven-stale record. */
-function reclaimStaleLock(dir: string, instanceId: string, kind: "leader" | "mutation"): boolean {
+function reclaimStaleLock(dir: string, instanceId: string): boolean {
   const expected = readOwner(dir);
-  if (!(expected ? ownerIsProvenDead(expected) : claimIsOld(dir))) return false;
+  if (!(expected ? canReclaim(expected) : claimIsOld(dir))) return false;
   const claim = `${dir}.claim-${randomUUID()}`;
   try { fs.renameSync(dir, claim); } catch { return false; }
   const current = readOwner(claim);
-  if (expected ? !sameOwner(current, expected) : current || !claimIsOld(claim)) { restoreClaim(claim, dir); return false; }
-  try { fs.rmSync(claim, { recursive: true, force: false }); log.info(`cron_${kind}_lock_reclaim`, { dir, instance_id: instanceId, outcome: "reclaimed" }); return true; }
-  catch (error: any) { log.warn(`cron_${kind}_lock_reclaim`, { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "remove_error" }); restoreClaim(claim, dir); return false; }
+  if (expected ? !sameOwner(current, expected) || !canReclaim(current!) : current || !claimIsOld(claim)) { restoreClaim(claim, dir); return false; }
+  try { fs.rmSync(claim, { recursive: true, force: false }); log.info("cron_leader_lock_reclaim", { dir, instance_id: instanceId, outcome: "reclaimed" }); return true; }
+  catch (error: any) { log.warn("cron_leader_lock_reclaim", { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "remove_error" }); restoreClaim(claim, dir); return false; }
 }
 function safelyRemoveLock(dir: string, expected?: CronLockOwner): boolean {
   if (expected && !sameOwner(readOwner(dir), expected)) { log.debug("cron_lock_remove", { dir, outcome: "not_owner" }); return false; }
@@ -114,7 +149,7 @@ function acquireDirectoryLock(dir: string, instanceId: string, reclaimStale: boo
   const token = resolveProcessStartToken(process.pid, deps);
   // Without a PID-reuse-safe token, durable leadership is disabled rather than unsafe.
   if (!token) { log.warn("cron_leader_lock_unavailable", { dir, instance_id: instanceId, reason: "no_process_token" }); return null; }
-  const owner: CronLockOwner = { pid: process.pid, process_start_token: token, instance_id: instanceId, heartbeat_at: new Date().toISOString() };
+  const owner: CronLockOwner = { pid: process.pid, process_start_token: token, instance_id: instanceId, heartbeat_at: new Date().toISOString(), pid_namespace: processPidNamespace() || undefined };
   try {
     fs.mkdirSync(dir, { mode: 0o700 });
     owner.fd = fs.openSync(ownerPath(dir), "wx", 0o600);
@@ -128,7 +163,7 @@ function acquireDirectoryLock(dir: string, instanceId: string, reclaimStale: boo
   } catch (error: any) {
     if (owner.fd !== undefined) try { fs.closeSync(owner.fd); } catch {}
     if (error?.code !== "EEXIST" || !reclaimStale) { (error?.code === "EEXIST" ? log.debug : log.warn)("cron_leader_lock_acquire", { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "lock_error" }); return null; }
-    if (!reclaimStaleLock(dir, instanceId, "leader")) return null;
+    if (!reclaimStaleLock(dir, instanceId)) return null;
     return acquireDirectoryLock(dir, instanceId, false);
   }
 }
@@ -156,46 +191,66 @@ function atomicWrite(file: string, content: string) {
   fs.writeFileSync(temp, content, { mode: 0o600 });
   fs.renameSync(temp, file);
 }
-function acquireMutationLock(dir: string, instanceId = randomUUID()): CronLockOwner | null {
-  const token = resolveProcessStartToken(process.pid);
-  if (!token) { log.warn("cron_mutation_lock_unavailable", { dir, instance_id: instanceId, reason: "no_process_token" }); return null; }
-  const owner: CronLockOwner = { pid: process.pid, process_start_token: token, instance_id: instanceId, heartbeat_at: new Date().toISOString() };
-  try {
-    fs.mkdirSync(dir, { mode: 0o700 });
-    fs.writeFileSync(ownerPath(dir), serializedOwner(owner), { flag: "wx", mode: 0o600 });
-    const acquired = sameOwner(readOwner(dir), owner);
-    log.debug("cron_mutation_lock_acquire", { dir, instance_id: instanceId, outcome: acquired ? "acquired" : "verification_failed" });
-    return acquired ? owner : null;
-  } catch (error: any) {
-    if (error?.code !== "EEXIST") { log.warn("cron_mutation_lock_acquire", { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "lock_error" }); return null; }
-    if (!reclaimStaleLock(dir, instanceId, "mutation")) return null;
-    return acquireMutationLock(dir, instanceId);
+function acquireKernelMutationLock(store: string): number {
+  fs.mkdirSync(path.dirname(store), { recursive: true, mode: 0o700 });
+  const lock = `${store}.mutation.flock`;
+  const fd = fs.openSync(lock, "a", 0o600);
+  // flock(1) locks the inherited open file description. The parent keeps its
+  // fd open across the whole synchronous transaction; close releases it even
+  // after the short-lived flock process exits. Never replace this lock inode.
+  const result = spawnSync("flock", ["-x", "-w", "2", "3"], { stdio: ["ignore", "ignore", "pipe", fd], encoding: "utf8" });
+  if (result.status !== 0) {
+    fs.closeSync(fd);
+    log.error("cron_mutation_kernel_lock", { lock, outcome: "failed", reason: result.error?.message || result.stderr });
+    throw new Error("Timed out waiting for cron storage lock");
+  }
+  // A previous pi version may still hold its independent directory lock.
+  // Never infer death from an invisible PID or an old heartbeat: only the
+  // operator can confirm all legacy writers have stopped and remove it.
+  if (fs.existsSync(lockPath(store, "mutation"))) {
+    fs.closeSync(fd);
+    log.error("cron_mutation_legacy_lock", { store, outcome: "blocked_until_verified_migration" });
+    throw new Error("legacy cron storage lock present; stop all old cron writers and verify they cannot resume before removing the stale mutation-lock directory");
   }
+  log.debug("cron_mutation_kernel_lock", { lock, outcome: "acquired" });
+  return fd;
 }
-function releaseMutationLock(dir: string, owner: CronLockOwner) { safelyRemoveLock(dir, owner); }
 export function mutateDurableCronStore(store: string, change: (tasks: DurableCronTask[]) => DurableCronTask[]): DurableCronEnvelope {
-  const lock = lockPath(store, "mutation"); let owner: CronLockOwner | null = null;
-  for (let attempt = 0; attempt < 100; attempt++) { owner = acquireMutationLock(lock); if (owner) break; sleep(20); }
-  if (!owner) throw new Error("Timed out waiting for cron storage lock");
+  const kernel = acquireKernelMutationLock(store);
   try {
     // Corrupt records must not prevent valid tasks from being updated. Schedulers
     // log each skipped record before calling here.
     const current = readDurableCronStore(store).tasks.filter((task) => { try { validateDurableCronTask(task); return true; } catch { return false; } });
-    const tasks = change(current); validate(tasks); const envelope: DurableCronEnvelope = { version: 1, tasks }; atomicWrite(store, JSON.stringify(envelope)); return envelope;
+    const tasks = change(current); validate(tasks);
+    const envelope: DurableCronEnvelope = { version: 1, tasks }; atomicWrite(store, JSON.stringify(envelope)); return envelope;
+  }
+  finally {
+    fs.closeSync(kernel);
+    log.debug("cron_mutation_kernel_lock", { store, outcome: "released" });
   }
-  finally { releaseMutationLock(lock, owner); }
 }
 export function acquireLeaderLock(store: string, instanceId: string, deps: StartTokenDeps = startTokenDeps): CronLockOwner | null { return acquireDirectoryLock(lockPath(store, "leader"), instanceId, true, deps); }
+export function isLeaderLockCurrent(store: string, owner: CronLockOwner): boolean {
+  const current = readOwner(lockPath(store, "leader"));
+  const valid = !!current && sameOwner(current, owner) && !leaseIsExpired(current);
+  log.debug("cron_leader_lock_current", { store, instance_id: owner.instance_id, valid });
+  return valid;
+}
 export function refreshLeaderLock(store: string, owner: CronLockOwner): boolean {
   const dir = lockPath(store, "leader");
-  if (owner.fd === undefined || !sameOwner(readOwner(dir), owner)) return false;
+  if (owner.fd === undefined || !isLeaderLockCurrent(store, owner)) {
+    log.warn("cron_leader_lock_refresh", { dir, instance_id: owner.instance_id, outcome: "not_current" });
+    return false;
+  }
   owner.heartbeat_at = new Date().toISOString();
   try {
     // Write through the exclusively-created inode, never a lock pathname that
     // could have been atomically reclaimed and recreated by another process.
     fs.ftruncateSync(owner.fd, 0); fs.writeSync(owner.fd, serializedOwner(owner), 0, "utf8"); fs.fsyncSync(owner.fd);
-    return sameOwner(readOwner(dir), owner);
-  } catch { return false; }
+    const renewed = sameOwner(readOwner(dir), owner);
+    log.debug("cron_leader_lock_refresh", { dir, instance_id: owner.instance_id, renewed });
+    return renewed;
+  } catch (error: any) { log.warn("cron_leader_lock_refresh", { dir, instance_id: owner.instance_id, reason: error?.code || "write_error" }); return false; }
 }
 export function releaseLeaderLock(store: string, owner: CronLockOwner) {
   const dir = lockPath(store, "leader");