routstrd 0.4.0 → 0.4.2

package/src/daemon/wallet/coco-client.ts

@@ -1,10 +1,11 @@
 import {
-  initializeCoco,
+  Manager,
   getEncodedToken,
   normalizeMintUrl,
 } from "@cashu/coco-core";
 import type {
   HistoryEntry,
+  ReceiveOperation,
   Logger as CocoLogger,
   Plugin as CocoPlugin,
 } from "@cashu/coco-core";
@@ -30,7 +31,25 @@ import type {
   CocodState,
   NpcAddress,
   NpcUsernameResult,
+  WalletCleanupOptions,
+  WalletCleanupResult,
+  WalletRecoveryProgress,
 } from "./cocod-client";
+import { selectCleanupOperations } from "./cleanup";
+import {
+  clearInterruptedReceiveReservations,
+  deleteReceiveTokenReservation,
+  getReceiveReconcileBackup,
+  initReceiveDedupSchema,
+  listProcessingReceiveTokens,
+  receiveInputFingerprint,
+  reconcileExecutingReceives,
+  releaseReceiveToken,
+  reserveReceiveToken,
+  setReceiveReconcileBackup,
+  updateReceiveToken,
+  type ReceiveReconcileSource,
+} from "./receive-dedup";
 import { cocoLogger, logger } from "../../utils/logger";
 import {
   legacyCocodPidPath,
@@ -56,12 +75,7 @@ type LegacyCocodFetch = (
 
 export interface LegacyCocodGuardOptions {
   socketPath?: string;
-  pidFilePath?: string;
   pathExists?: (path: string) => boolean;
-  readFile?: (path: string) => string;
-  isProcessRunning?: (pid: number) => boolean;
-  /** PID owned by the caller's already-acquired legacy exclusion lock. */
-  ignorePid?: number;
   fetchImpl?: LegacyCocodFetch;
   timeoutMs?: number;
 }
@@ -69,6 +83,8 @@ export interface LegacyCocodGuardOptions {
 export interface LegacyCocodPidClaimOptions {
   pidFilePath?: string;
   pid?: number;
+  /** Human-readable lock name used in contention errors. */
+  label?: string;
   openExclusive?: (path: string) => number;
   writePid?: (fd: number, pid: number) => void;
   closeFile?: (fd: number) => void;
@@ -110,6 +126,20 @@ function startupProgress(message: string): void {
   console.log(`${STARTUP_LOG_PREFIX} ${message}`);
 }
 
+// Set only while the background wallet recovery sweeps are running. While set,
+// coco logger messages that indicate a stalled/failed per-mint check are
+// forwarded to the startup stream (see createCocoLogger).
+let surfacingRecoveryProgress = false;
+
+// These fire once per operation when a mint is unreachable or otherwise fails
+// to reconcile. They are the only per-mint signal coco emits, and they happen
+// exactly when recovery is slow.
+const RECOVERY_STALL_MARKERS = new Map<string, string>([
+  ["SendOperationService\u0000Could not reach mint for recovery, will retry later", "Send recovery: mint unreachable"],
+  ["ReceiveOperationService\u0000Could not reach mint for receive recovery, will retry later", "Receive recovery: mint unreachable"],
+  ["MintOperationService\u0000Failed to reconcile stale pending mint operation", "Mint recovery: pending operation check failed"],
+]);
+
 const SAFE_COCO_LOG_FIELDS = new Set([
   "module",
   "mintUrl",
@@ -149,6 +179,19 @@ function createCocoLogger(bindings: Record<string, unknown> = {}): CocoLogger {
     // useful without copying wallet material into routstrd's logs. Written to
     // ~/.routstrd/coco-logs/ so wallet-engine noise stays out of the main logs.
     const metadata = safeCocoMetadata([bindings, ...meta]);
+
+    if (surfacingRecoveryProgress) {
+      const module = typeof bindings.module === "string" ? bindings.module : undefined;
+      if (module) {
+        const stall = RECOVERY_STALL_MARKERS.get(`${module}\u0000${message}`);
+        if (stall) {
+          const mintUrl =
+            typeof metadata.mintUrl === "string" ? metadata.mintUrl : undefined;
+          startupProgress(mintUrl ? `${stall} (${mintUrl})` : stall);
+        }
+      }
+    }
+
     cocoLogger[level](
       `[coco] ${message}`,
       ...(Object.keys(metadata).length > 0 ? [metadata] : []),
@@ -199,13 +242,35 @@ function saveConfig(config: CocodConfig, configFile: string): void {
   }
 }
 
+export function isZombieProcess(
+  pid: number,
+  readFile: (path: string) => string = (path) =>
+    readFileSync(path, "utf-8"),
+): boolean {
+  try {
+    // Linux exposes zombie state as the character following the final `)` in
+    // /proc/<pid>/stat. Use the final parenthesis because process names may
+    // themselves contain spaces or parentheses. Other platforms simply fall
+    // back to process.kill(pid, 0) below.
+    const stat = readFile(`/proc/${pid}/stat`);
+    const commandEnd = stat.lastIndexOf(")");
+    return commandEnd >= 0 && stat.charAt(commandEnd + 2) === "Z";
+  } catch {
+    return false;
+  }
+}
+
 function defaultIsProcessRunning(pid: number): boolean {
   try {
     process.kill(pid, 0);
-    return true;
   } catch (error) {
     return (error as NodeJS.ErrnoException).code === "EPERM";
   }
+
+  // kill(pid, 0) also succeeds for dead-but-unreaped processes. Zombies hold
+  // no database or socket resources, so treating them as dead allows stale
+  // wallet locks to be reclaimed (notably under non-reaping Docker PID 1s).
+  return !isZombieProcess(pid);
 }
 
 function hasErrorCode(error: unknown, codes: Set<string>): boolean {
@@ -225,9 +290,15 @@ function hasErrorCode(error: unknown, codes: Set<string>): boolean {
 }
 
 /**
- * Refuse to open coco.db while the legacy cocod daemon owns its Unix socket.
+ * Refuse to open coco.db while a daemon answers on legacy cocod's Unix socket.
  * Two independent wallet engines must never operate on the same proof database.
  *
+ * cocod.pid is deliberately shared: routstrd writes its own PID there while
+ * the in-process wallet is open, fencing old cocod binaries from starting. A
+ * live PID in that file therefore cannot identify cocod. Socket responsiveness
+ * is the authoritative identity check; the atomic PID-file claim below closes
+ * the race when cocod is still starting and has not opened its socket yet.
+ *
  * A socket left behind after a crash is safe to ignore only when connecting
  * fails with ENOENT or ECONNREFUSED. Other probe failures are treated as unsafe
  * because they do not prove that cocod has stopped.
@@ -236,37 +307,7 @@ export async function assertLegacyCocodNotRunning(
   options: LegacyCocodGuardOptions = {},
 ): Promise<void> {
   const socketPath = options.socketPath || legacyCocodSocketPath();
-  const pidFilePath = options.pidFilePath || legacyCocodPidPath();
   const pathExists = options.pathExists || existsSync;
-  const readFile = options.readFile || ((path) => readFileSync(path, "utf-8"));
-  const isProcessRunning = options.isProcessRunning || defaultIsProcessRunning;
-
-  const getRunningLegacyPid = (): number | null => {
-    if (!pathExists(pidFilePath)) return null;
-
-    try {
-      const pid = Number.parseInt(readFile(pidFilePath).trim(), 10);
-      return Number.isInteger(pid) &&
-        pid > 0 &&
-        pid !== options.ignorePid &&
-        isProcessRunning(pid)
-        ? pid
-        : null;
-    } catch {
-      // An unreadable or malformed PID file does not prove that cocod is alive;
-      // the socket probe below remains the authoritative fallback.
-      return null;
-    }
-  };
-
-  const runningPid = getRunningLegacyPid();
-  if (runningPid !== null) {
-    throw new Error(
-      `Legacy cocod daemon is still running with PID ${runningPid}. ` +
-        "Refusing to open the wallet database because cocod and coco-core cannot safely use it at the same time. " +
-        `Run 'cocod stop' or 'kill ${runningPid}' and try again.`,
-    );
-  }
 
   if (!pathExists(socketPath)) return;
 
@@ -281,19 +322,8 @@ export async function assertLegacyCocodNotRunning(
     await response.body?.cancel();
   } catch (error) {
     if (hasErrorCode(error, STALE_SOCKET_ERROR_CODES)) {
-      // Recheck after the failed probe in case cocod started concurrently.
-      const newlyRunningPid = getRunningLegacyPid();
-      if (newlyRunningPid === null) {
-        logger.debug(`Ignoring stale legacy cocod socket at ${socketPath}`);
-        return;
-      }
-
-      throw new Error(
-        `Legacy cocod daemon is still running with PID ${newlyRunningPid}. ` +
-          "Refusing to open the wallet database because cocod and coco-core cannot safely use it at the same time. " +
-          `Run 'cocod stop' or 'kill ${newlyRunningPid}' and try again.`,
-        { cause: error },
-      );
+      logger.debug(`Ignoring stale legacy cocod socket at ${socketPath}`);
+      return;
     }
 
     throw new Error(
@@ -418,12 +448,14 @@ export function claimLegacyCocodPidFile(
   return claimPidFile({
     ...options,
     pidFilePath: options.pidFilePath || legacyCocodPidPath(),
+    label: options.label || "legacy cocod exclusion lock",
   });
 }
 
 function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: string }): () => void {
   const pidFilePath = options.pidFilePath;
   const pid = options.pid ?? process.pid;
+  const label = options.label || "wallet process lock";
   const openExclusive =
     options.openExclusive || ((path: string) => openSync(path, "wx", 0o600));
   const writePid =
@@ -449,7 +481,7 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
       stalePid = Number.parseInt(readFile(pidFilePath).trim(), 10);
     } catch {
       throw new Error(
-        `Cannot claim the wallet process lock at ${pidFilePath}. ` +
+        `Cannot claim the ${label} at ${pidFilePath}. ` +
           "Another cocod or routstrd process may be starting. Stop it and try again.",
         { cause: error },
       );
@@ -460,9 +492,13 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
       stalePid <= 0 ||
       isProcessRunning(stalePid)
     ) {
+      const ownerMessage =
+        Number.isInteger(stalePid) && stalePid > 0
+          ? `PID ${stalePid} is still running and holds it. ` +
+            `Stop that process first ('routstrd stop' or 'kill ${stalePid}').`
+          : "Another cocod or routstrd process may be starting. Stop it and try again.";
       throw new Error(
-        `Cannot claim the wallet process lock at ${pidFilePath}. ` +
-          "Another cocod or routstrd process may be starting. Stop it and try again.",
+        `Cannot claim the ${label} at ${pidFilePath}: ${ownerMessage}`,
         { cause: error },
       );
     }
@@ -472,7 +508,7 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
       fd = openExclusive(pidFilePath);
     } catch (retryError) {
       throw new Error(
-        `Cannot claim the wallet process lock at ${pidFilePath}. ` +
+        `Cannot claim the ${label} at ${pidFilePath}. ` +
           "Another cocod or routstrd process may be starting. Stop it and try again.",
         { cause: retryError },
       );
@@ -493,9 +529,10 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
   }
 
   let released = false;
-  return () => {
+  const release = () => {
     if (released) return;
     released = true;
+    process.removeListener("exit", release);
 
     try {
       if (readFile(pidFilePath).trim() === String(pid)) {
@@ -504,12 +541,37 @@ function claimPidFile(options: LegacyCocodPidClaimOptions & { pidFilePath: strin
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
         logger.warn(
-          `Failed to release wallet process lock at ${pidFilePath}:`,
+          `Failed to release ${label} at ${pidFilePath}:`,
           error,
         );
       }
     }
   };
+
+  // process.exit() and natural shutdown still run synchronous exit handlers.
+  // This prevents migration or startup failures from stranding our PID files.
+  process.once("exit", release);
+  return release;
+}
+
+/**
+ * Minimal structural view of coco-core's MintOperationService.
+ * The service is private on the exported Manager class, so the in-process
+ * client reaches it through this narrow cast. Both methods reload the latest
+ * persisted row before mutating anything, so a bare operation id is enough.
+ */
+interface MintOperationServiceCleanup {
+  failPendingOperation(
+    op: { id: string },
+    terminalFailure: { reason: string; retryable?: boolean; observedAt: number },
+  ): Promise<unknown>;
+  /**
+   * Ask the mint for a pending quote's current state and persist the
+   * observation. "waiting" means the mint still reports the quote as unpaid.
+   */
+  observePendingOperation(
+    operationId: string,
+  ): Promise<{ category: "waiting" | "ready" | "completed" | "terminal" }>;
 }
 
 export interface CreateCocoClientOptions {
@@ -528,6 +590,337 @@ export interface CreateCocoClientOptions {
   npcBaseUrl?: string;
 }
 
+/**
+ * Build a usable coco Manager without running the blocking recovery sweeps.
+ *
+ * This replicates `initializeCoco()` up to (but not including) the send/melt/
+ * receive/mint recovery passes, so the daemon can serve wallet reads while
+ * recovery proceeds in the background.
+ */
+function constructCocoManager(
+  repo: SqliteRepositories,
+  seed: Uint8Array,
+): Manager {
+  return new Manager(repo, async () => seed, createCocoLogger());
+}
+
+async function enableCocoManager(coco: Manager): Promise<void> {
+  await coco.initPlugins();
+  await coco.reconcileLegacyMintQuotes();
+  await coco.enableMintOperationWatcher();
+  await coco.enableProofStateWatcher();
+  await coco.enableMintOperationProcessor();
+}
+
+/**
+ * Shared wall-clock budget for checking expired mint quotes with their mints
+ * during background recovery. coco-core issues mint requests without a
+ * timeout, so a hung mint could otherwise stall this phase (and with it the
+ * recovery promise that gates value-moving operations) far longer than this.
+ */
+const EXPIRED_MINT_OBSERVATION_DEADLINE_MS = 15_000;
+
+/** Rejects when `timeoutMs` elapses before `promise` settles. */
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(
+      () => reject(new Error("Timed out contacting mint")),
+      timeoutMs,
+    );
+  });
+  return Promise.race([promise, timeout]).finally(() => {
+    if (timer !== undefined) clearTimeout(timer);
+  });
+}
+
+/** Structural subset of coco's Manager used by expired-quote settlement. */
+export interface ExpiredMintQuoteSource {
+  ops: {
+    mint: {
+      listPending(): Promise<
+        Array<{
+          id: string;
+          mintUrl: string;
+          quoteId?: string;
+          state: string;
+          /** Quote expiry in epoch seconds. */
+          expiry: number;
+          updatedAt: number;
+          lastObservedRemoteState?: string;
+        }>
+      >;
+    };
+  };
+  mintOperationService: MintOperationServiceCleanup;
+}
+
+export interface ExpiredMintSettlement {
+  /** Quotes their mint confirmed as UNPAID, failed locally. */
+  failed: number;
+  /** Quotes observed as PAID/ISSUED, left for mint recovery to finalize. */
+  leftForRecovery: number;
+  /** Quotes whose mint could not be checked in time, left pending. */
+  unobserved: number;
+}
+
+/**
+ * Settle expired pending mint quotes before the mint recovery sweep runs.
+ *
+ * An expired bolt11 invoice can never be paid again, so a quote the mint
+ * still reports as UNPAID is guaranteed never to be issued and is failed
+ * locally. That local fail is what keeps coco-core's mint recovery sweep
+ * quick: the sweep treats UNPAID as "waiting" and would otherwise re-contact
+ * every dead quote's mint on every startup.
+ *
+ * The observation round is what makes the local fail safe: a quote can have
+ * been paid before expiry while the daemon was down, leaving no local
+ * observation behind. Failing such a quote without asking the mint would
+ * strand the paid funds, because failed operations are skipped by recovery.
+ * Asking the mint first closes that hole: PAID/ISSUED quotes are left for
+ * the sweep to finalize, and quotes whose mint is unreachable or too slow
+ * are left pending so a later startup can still recover them.
+ */
+export async function settleExpiredMintQuotes(
+  source: ExpiredMintQuoteSource,
+  nowMs: number,
+  deadlineMs: number = EXPIRED_MINT_OBSERVATION_DEADLINE_MS,
+): Promise<ExpiredMintSettlement> {
+  const pendingMints = await source.ops.mint.listPending();
+  const selection = selectCleanupOperations({
+    mints: pendingMints,
+    sends: [],
+    melts: [],
+    nowMs,
+    minAgeMs: 0,
+  });
+
+  const settlement: ExpiredMintSettlement = {
+    failed: 0,
+    leftForRecovery: 0,
+    unobserved: 0,
+  };
+  const candidates = selection.mintsToFail;
+  if (candidates.length === 0) return settlement;
+
+  const startedAt = Date.now();
+  for (const op of candidates) {
+    const remainingMs = deadlineMs - (Date.now() - startedAt);
+    if (remainingMs <= 0) {
+      const skipped =
+        candidates.length -
+        settlement.failed -
+        settlement.leftForRecovery -
+        settlement.unobserved;
+      settlement.unobserved += skipped;
+      startupProgress(
+        `Expired mint quote check budget exhausted; ${skipped} quote(s) left for mint recovery.`,
+      );
+      break;
+    }
+
+    try {
+      const result = await withTimeout(
+        source.mintOperationService.observePendingOperation(op.id),
+        remainingMs,
+      );
+      if (result.category === "waiting") {
+        // The mint confirms the expired quote is still unpaid: it can never
+        // be issued now, so failing it locally cannot strand funds.
+        await source.mintOperationService.failPendingOperation(
+          { id: op.id },
+          {
+            reason: "Expired mint quote confirmed unpaid by mint",
+            retryable: false,
+            observedAt: Date.now(),
+          },
+        );
+        settlement.failed++;
+      } else {
+        // PAID/ISSUED (or terminally failed) at the mint: normal recovery
+        // must see this quote so paid proofs get claimed.
+        settlement.leftForRecovery++;
+        const observed =
+          result.category === "ready"
+            ? "was paid at the mint"
+            : result.category === "completed"
+              ? "was already issued at the mint"
+              : "failed terminally at the mint";
+        startupProgress(
+          `Expired mint quote ${op.quoteId ?? op.id} at ${op.mintUrl} ${observed}; leaving it for mint recovery.`,
+        );
+      }
+    } catch (error) {
+      // Mint unreachable, too slow, or the quote unknown to it: leave the
+      // operation pending so a later startup can still recover it.
+      settlement.unobserved++;
+      logger.warn("Could not check expired mint quote; leaving it pending", {
+        operationId: op.id,
+        mintUrl: op.mintUrl,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  return settlement;
+}
+
+interface ReceiveRecoveryInternals {
+  receiveOperationService: {
+    checkProofStatesWithMint(
+      mintUrl: string,
+      proofs: Array<{ secret: string }>,
+    ): Promise<Array<{ state: string }>>;
+    hasSavedOutputs(operation: unknown): Promise<boolean>;
+    markAsRolledBack(operation: unknown, error: string): Promise<unknown>;
+  };
+  mintAdapter: {
+    getCashuMint(mintUrl: string): {
+      restore(input: { outputs: Array<{ amount: number; id: string; B_: string }> }): Promise<{
+        outputs: Array<{ B_: string }>;
+      }>;
+    };
+  };
+}
+
+async function reconcileDuplicateReceiveOperations(
+  coco: Manager,
+  repo: SqliteRepositories,
+): Promise<Awaited<ReturnType<typeof reconcileExecutingReceives>>> {
+  const internals = coco as unknown as ReceiveRecoveryInternals;
+  const unavailableMints = new Set<string>();
+  const deadline = Date.now() + 45_000;
+  const ensureMintBudget = (mintUrl: string): void => {
+    if (Date.now() >= deadline) throw new Error("Receive cleanup time budget exhausted");
+    if (unavailableMints.has(mintUrl)) throw new Error("Mint already failed receive cleanup");
+  };
+  const markMintFailure = (mintUrl: string, error: unknown): never => {
+    unavailableMints.add(mintUrl);
+    throw error;
+  };
+  const source: ReceiveReconcileSource = {
+    listExecuting: () => repo.receiveOperationRepository.getByState("executing"),
+    checkProofStates: async (operation) => {
+      ensureMintBudget(operation.mintUrl);
+      try {
+        return await withTimeout(
+          internals.receiveOperationService.checkProofStatesWithMint(
+            operation.mintUrl,
+            operation.inputProofs,
+          ),
+          Math.min(15_000, Math.max(1, deadline - Date.now())),
+        );
+      } catch (error) {
+        return markMintFailure(operation.mintUrl, error);
+      }
+    },
+    restoreOutputs: async (mintUrl, outputs) => {
+      ensureMintBudget(mintUrl);
+      // Probe deterministic outputs in bounded batches. The Cashu restore
+      // response echoes only blinded messages with stored signatures, which
+      // identifies the owning operation. Coco later performs the real proof
+      // recovery for the retained operation.
+      const restored: Array<{ B_: string }> = [];
+      const mint = internals.mintAdapter.getCashuMint(mintUrl);
+      for (let index = 0; index < outputs.length; index += 300) {
+        try {
+          const response = await withTimeout(
+            mint.restore({ outputs: outputs.slice(index, index + 300) }),
+            Math.min(15_000, Math.max(1, deadline - Date.now())),
+          );
+          restored.push(...response.outputs);
+        } catch (error) {
+          return markMintFailure(mintUrl, error);
+        }
+      }
+      return restored;
+    },
+    hasSavedOutputs: (operation) =>
+      internals.receiveOperationService.hasSavedOutputs(operation),
+    rollBack: async (operation, reason) => {
+      await internals.receiveOperationService.markAsRolledBack(operation, reason);
+    },
+  };
+  return reconcileExecutingReceives(source);
+}
+
+interface RecoveryPhaseProgress {
+  phase: string;
+  failedMintQuotes: number;
+}
+
+/**
+ * Run the wallet recovery sweeps in order, reporting phase changes.
+ *
+ * Expired mint quotes are settled first: quotes their mint confirms as unpaid
+ * are failed locally so `recoverPendingMintOperations()` skips them, while
+ * paid/issued and unreachable-mint quotes stay pending for the sweep.
+ */
+async function runWalletRecovery(
+  coco: Manager,
+  onProgress: (progress: RecoveryPhaseProgress) => void,
+  receiveOperationIds?: string[],
+): Promise<void> {
+  surfacingRecoveryProgress = true;
+  let failedMintQuotes = 0;
+  try {
+    onProgress({ phase: "Settling expired mint quotes", failedMintQuotes });
+    const settlement = await settleExpiredMintQuotes(
+      {
+        ops: coco.ops,
+        mintOperationService: (
+          coco as unknown as {
+            mintOperationService: MintOperationServiceCleanup;
+          }
+        ).mintOperationService,
+      },
+      Date.now(),
+    );
+    failedMintQuotes = settlement.failed;
+    if (settlement.leftForRecovery > 0 || settlement.unobserved > 0) {
+      startupProgress(
+        `Expired mint quotes: ${settlement.failed} failed locally, ` +
+          `${settlement.leftForRecovery} paid/issued (kept for recovery), ` +
+          `${settlement.unobserved} unverifiable (kept pending).`,
+      );
+    }
+    onProgress({ phase: "Settled expired mint quotes", failedMintQuotes });
+
+    onProgress({ phase: "Send recovery", failedMintQuotes });
+    await coco.ops.send.recovery.run();
+
+    onProgress({ phase: "Melt recovery", failedMintQuotes });
+    await coco.ops.melt.recovery.run();
+
+    onProgress({ phase: "Receive recovery", failedMintQuotes });
+    if (receiveOperationIds) {
+      // The pre-check already classified every executing receive by unique
+      // input set. Recover only the conclusive retained operations; unresolved
+      // groups stay untouched instead of falling back to Coco 1's expensive
+      // per-row sweep on this startup.
+      for (const operationId of receiveOperationIds) {
+        try {
+          await withTimeout(coco.ops.receive.refresh(operationId), 15_000);
+        } catch (error) {
+          logger.warn("Targeted receive recovery did not complete", {
+            operationId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    } else {
+      await coco.ops.receive.recovery.run();
+    }
+
+    onProgress({ phase: "Mint recovery", failedMintQuotes });
+    await coco.recoverPendingMintOperations();
+
+    onProgress({ phase: "done", failedMintQuotes });
+  } finally {
+    surfacingRecoveryProgress = false;
+  }
+}
+
 export async function createCocoClient(
   options: CreateCocoClientOptions = {},
 ): Promise<CocodClient> {
@@ -544,14 +937,14 @@ export async function createCocoClient(
   const npcBaseUrl = options.npcBaseUrl || NPC_DEFAULT_BASE_URL;
   const npcAddressDomain = new URL(npcBaseUrl).host;
 
-  await assertLegacyCocodNotRunning({
-    socketPath: legacySocket,
-    pidFilePath: legacyPidFile,
-  });
+  await assertLegacyCocodNotRunning({ socketPath: legacySocket });
   // The canonical wallet directory is created by initialization/migration.
   // Keep a legacy PID claim as an exclusion fence for old cocod binaries.
   mkdirSync(dirname(legacyPidFile), { recursive: true, mode: 0o700 });
-  const releaseWalletPidClaim = claimPidFile({ pidFilePath: walletPidFile });
+  const releaseWalletPidClaim = claimPidFile({
+    pidFilePath: walletPidFile,
+    label: "routstrd wallet lock",
+  });
   let releaseLegacyPidClaim: () => void;
   try {
     releaseLegacyPidClaim = claimLegacyCocodPidFile({
@@ -563,9 +956,26 @@ export async function createCocoClient(
   }
 
   let database: Database | undefined;
-  let coco: Awaited<ReturnType<typeof initializeCoco>> | undefined;
+  let coco: Manager | undefined;
+  let findFinalizedReceiveSibling: (
+    operation: ReceiveOperation | null,
+  ) => Promise<string | null> = async () => null;
   let walletConfig = loadConfig(configFile);
 
+  let recoveryPhase = "queued";
+  let recoveryFailedMintQuotes = 0;
+  let recoveryDone = false;
+  let recoveryError: string | undefined;
+  const recoveryCounts = {
+    pendingSends: 0,
+    inflightProofs: 0,
+    pendingMints: 0,
+  };
+  let recoveryResolve: (() => void) | undefined;
+  const recoveryPromise = new Promise<void>((resolve) => {
+    recoveryResolve = resolve;
+  });
+
   try {
     startupProgress("Opening Cashu wallet database...");
 
@@ -576,29 +986,125 @@ export async function createCocoClient(
     database = new Database(dbPath);
     const repo = new SqliteRepositories({ database });
     await repo.init();
+    initReceiveDedupSchema(database);
+    const interruptedReservations = clearInterruptedReceiveReservations(database);
+    if (interruptedReservations > 0) {
+      logger.warn("Cleared interrupted receive reservations with no Coco operation", {
+        count: interruptedReservations,
+      });
+    }
 
     const [pendingSends, inflightProofs, pendingMints] = await Promise.all([
       repo.sendOperationRepository.getPending(),
       repo.proofRepository.getInflightProofs(),
       repo.mintOperationRepository.getPending(),
     ]);
+    recoveryCounts.pendingSends = pendingSends.length;
+    recoveryCounts.inflightProofs = inflightProofs.length;
+    recoveryCounts.pendingMints = pendingMints.length;
     const recoveryCount =
-      pendingSends.length + inflightProofs.length + pendingMints.length;
+      recoveryCounts.pendingSends +
+      recoveryCounts.inflightProofs +
+      recoveryCounts.pendingMints;
+
     if (recoveryCount > 0) {
       startupProgress(
-        `Recovering wallet state: ${pendingSends.length} pending sends, ` +
-          `${inflightProofs.length} in-flight proofs, ${pendingMints.length} pending mints. ` +
-          "This may take a few minutes while Cashu mints are contacted.",
+        `Recovering wallet state in background: ${recoveryCounts.pendingSends} pending sends, ` +
+          `${recoveryCounts.inflightProofs} in-flight proofs, ${recoveryCounts.pendingMints} pending mints.`,
       );
     } else {
       startupProgress("Initializing Cashu wallet...");
     }
 
-    coco = await initializeCoco({
-      repo,
-      seedGetter: async () => seed,
-      logger: createCocoLogger(),
-    });
+    // Construct Coco so the pre-recovery checker can reuse its mint adapter,
+    // but do not enable watchers/processors until the backup and cleanup finish.
+    coco = constructCocoManager(repo, seed);
+
+    const executingReceives = await repo.receiveOperationRepository.getByState("executing");
+    let receiveRecoveryOperationIds: string[] | undefined;
+    if (executingReceives.length > 0) {
+      startupProgress(
+        `Checking ${executingReceives.length} unfinished Cashu receive operation(s) for duplicates...`,
+      );
+      const recordedBackup = getReceiveReconcileBackup(database);
+      if (!recordedBackup || !existsSync(recordedBackup)) {
+        const backupPath = `${dbPath}.pre-receive-reconcile-${Date.now()}`;
+        database.exec(`VACUUM INTO '${backupPath.replaceAll("'", "''")}'`);
+        setReceiveReconcileBackup(database, backupPath);
+        startupProgress(`Created wallet backup before receive cleanup: ${backupPath}`);
+      }
+      const receiveReconcile = await reconcileDuplicateReceiveOperations(coco, repo);
+      receiveRecoveryOperationIds = receiveReconcile.recoveryOperationIds;
+      startupProgress(
+        `Receive cleanup: ${receiveReconcile.executing} operation(s), ` +
+          `${receiveReconcile.uniqueGroups} unique input set(s), ` +
+          `${receiveReconcile.rolledBack} stale duplicate(s) retired, ` +
+          `${receiveReconcile.unresolved} unresolved.`,
+      );
+    }
+
+    await enableCocoManager(coco);
+    const openDatabase = database;
+
+    findFinalizedReceiveSibling = async (
+      operation: ReceiveOperation | null,
+    ): Promise<string | null> => {
+      if (!operation) return null;
+      const fingerprint = receiveInputFingerprint(operation);
+      const siblings = await repo.receiveOperationRepository.getByMintUrl(operation.mintUrl);
+      return (
+        siblings.find(
+          (candidate) =>
+            candidate.state === "finalized" &&
+            receiveInputFingerprint(candidate) === fingerprint,
+        )?.id ?? null
+      );
+    };
+
+    const syncReceiveReservations = async (): Promise<void> => {
+      for (const reservation of listProcessingReceiveTokens(openDatabase)) {
+        if (!reservation.operationId) continue;
+        try {
+          const operation = await coco!.ops.receive.get(reservation.operationId);
+          if (operation?.state === "finalized") {
+            updateReceiveToken(openDatabase, reservation.tokenHash, {
+              state: "succeeded",
+              operationId: operation.id,
+            });
+          } else if (operation?.state === "rolled_back") {
+            const finalizedSibling = await findFinalizedReceiveSibling(operation);
+            updateReceiveToken(openDatabase, reservation.tokenHash, finalizedSibling
+              ? {
+                  state: "succeeded",
+                  operationId: finalizedSibling,
+                }
+              : {
+                  state: "failed",
+                  operationId: operation.id,
+                  error: operation.error || "Token receive was rolled back",
+                });
+          } else if (operation?.state === "prepared") {
+            // A crash before execute had no mint side effect. Cancel the stale
+            // prepared operation and permit a fresh exact-token attempt.
+            await coco!.ops.receive.cancel(
+              operation.id,
+              "Cancelled interrupted receive before execution",
+            );
+            deleteReceiveTokenReservation(openDatabase, reservation.tokenHash);
+          } else if (!operation) {
+            deleteReceiveTokenReservation(openDatabase, reservation.tokenHash);
+          }
+        } catch (error) {
+          // One damaged/stale reservation must never block daemon startup or
+          // make every wallet write fail after recovery.
+          logger.warn("Could not reconcile receive token reservation", {
+            operationId: reservation.operationId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    };
+    await syncReceiveReservations();
 
     const trustedMints = await coco.mint.getAllTrustedMints();
     const configuredDefault = walletConfig.defaultMintUrl;
@@ -640,6 +1146,34 @@ export async function createCocoClient(
     }
 
     startupProgress("Cashu wallet ready.");
+
+    // Recovery runs in the background so the daemon can serve wallet reads
+    // immediately. Value-moving operations await the same promise below.
+    runWalletRecovery(
+      coco,
+      (progress) => {
+        recoveryPhase = progress.phase;
+        recoveryFailedMintQuotes = progress.failedMintQuotes;
+        if (progress.phase !== "done") {
+          startupProgress(`Wallet recovery: ${progress.phase}...`);
+        }
+      },
+      receiveRecoveryOperationIds,
+    )
+      .then(async () => {
+        await syncReceiveReservations();
+        recoveryDone = true;
+        recoveryPhase = "done";
+        recoveryResolve?.();
+        startupProgress("Wallet recovery complete.");
+      })
+      .catch((error) => {
+        recoveryDone = true;
+        recoveryPhase = "error";
+        recoveryError = error instanceof Error ? error.message : String(error);
+        recoveryResolve?.();
+        startupProgress(`Wallet recovery failed: ${recoveryError}`);
+      });
   } catch (error) {
     database?.close();
     releaseLegacyPidClaim();
@@ -660,6 +1194,18 @@ export async function createCocoClient(
   };
 
   let disposed = false;
+
+  /**
+   * Block a value-moving operation until background recovery has settled.
+   * Reads stay ungated so the daemon can report balances/status immediately.
+   */
+  const waitForRecovery = async (): Promise<void> => {
+    if (!recoveryDone) await recoveryPromise;
+    if (recoveryError) {
+      throw new Error(`Wallet is not ready: ${recoveryError}`);
+    }
+  };
+
   return {
     async ping(): Promise<boolean> {
       try {
@@ -671,6 +1217,8 @@ export async function createCocoClient(
     },
 
     async getStatus(): Promise<CocodState> {
+      if (recoveryError) return "ERROR";
+      if (!recoveryDone) return "RECOVERING";
       try {
         await coco.wallet.balances.total();
         return "UNLOCKED";
@@ -679,6 +1227,18 @@ export async function createCocoClient(
       }
     },
 
+    async getRecoveryProgress(): Promise<WalletRecoveryProgress> {
+      return {
+        state: recoveryError ? "ERROR" : recoveryDone ? "UNLOCKED" : "RECOVERING",
+        phase: recoveryPhase,
+        pendingSends: recoveryCounts.pendingSends,
+        inflightProofs: recoveryCounts.inflightProofs,
+        pendingMints: recoveryCounts.pendingMints,
+        failedMintQuotes: recoveryFailedMintQuotes,
+        ...(recoveryError ? { error: recoveryError } : {}),
+      };
+    },
+
     async unlock(_passphrase: string): Promise<string> {
       // coco-core does not support passphrase locking.
       // Wallet access is controlled by ~/.routstrd/wallet/config.json.
@@ -696,11 +1256,135 @@ export async function createCocoClient(
     },
 
     async receiveCashu(token: string): Promise<string> {
-      await coco.wallet.receive(token);
-      return "Token received successfully";
+      const reservation = reserveReceiveToken(database, token);
+      if (!reservation.acquired) {
+        if (reservation.existing?.state === "succeeded") {
+          return "Token already received successfully";
+        }
+        if (
+          reservation.existing?.state === "processing" &&
+          reservation.existing.operationId
+        ) {
+          // A prior request may have stopped after the mint call became
+          // uncertain. Re-drive that one Coco operation instead of creating a
+          // duplicate. Refresh is idempotent and uses its stored output data.
+          try {
+            const operation = await withTimeout(
+              coco.ops.receive.refresh(reservation.existing.operationId),
+              15_000,
+            );
+            if (operation.state === "finalized") {
+              updateReceiveToken(database, reservation.tokenHash, {
+                state: "succeeded",
+                operationId: operation.id,
+              });
+              return "Token received successfully";
+            }
+            if (operation.state === "rolled_back") {
+              const finalizedSibling = await findFinalizedReceiveSibling(operation);
+              if (finalizedSibling) {
+                updateReceiveToken(database, reservation.tokenHash, {
+                  state: "succeeded",
+                  operationId: finalizedSibling,
+                });
+                return "Token already received successfully";
+              }
+              updateReceiveToken(database, reservation.tokenHash, {
+                state: "failed",
+                operationId: operation.id,
+                error: operation.error || "Token receive was rolled back",
+              });
+              throw new Error(operation.error || "Token receive was rolled back");
+            }
+          } catch (error) {
+            const latest = await coco.ops.receive.get(reservation.existing.operationId);
+            if (latest?.state === "finalized") {
+              updateReceiveToken(database, reservation.tokenHash, {
+                state: "succeeded",
+                operationId: latest.id,
+              });
+              return "Token received successfully";
+            }
+            throw error;
+          }
+          throw new Error("Token receive is still unresolved");
+        }
+        if (reservation.existing?.state === "processing") {
+          throw new Error("Token receive is already in progress");
+        }
+        throw new Error(reservation.existing?.error || "Token receive previously failed");
+      }
+
+      let preparedOperationId: string | undefined;
+      try {
+        await waitForRecovery();
+        const prepared = await coco.ops.receive.prepare({ token });
+        preparedOperationId = prepared.id;
+        updateReceiveToken(database, reservation.tokenHash, {
+          state: "processing",
+          operationId: prepared.id,
+        });
+        await coco.ops.receive.execute(prepared.id);
+        updateReceiveToken(database, reservation.tokenHash, {
+          state: "succeeded",
+          operationId: prepared.id,
+        });
+        return "Token received successfully";
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (preparedOperationId) {
+          let latest: Awaited<ReturnType<typeof coco.ops.receive.get>> = null;
+          try {
+            latest = await coco.ops.receive.get(preparedOperationId);
+          } catch (lookupError) {
+            logger.warn("Could not inspect failed receive operation", {
+              operationId: preparedOperationId,
+              error:
+                lookupError instanceof Error ? lookupError.message : String(lookupError),
+            });
+          }
+          if (latest?.state === "finalized") {
+            updateReceiveToken(database, reservation.tokenHash, {
+              state: "succeeded",
+              operationId: preparedOperationId,
+            });
+            return "Token received successfully";
+          }
+          if (latest?.state === "executing") {
+            updateReceiveToken(database, reservation.tokenHash, {
+              state: "processing",
+              operationId: preparedOperationId,
+              error: message,
+            });
+          } else if (latest?.state === "rolled_back") {
+            const finalizedSibling = await findFinalizedReceiveSibling(latest);
+            if (finalizedSibling) {
+              updateReceiveToken(database, reservation.tokenHash, {
+                state: "succeeded",
+                operationId: finalizedSibling,
+              });
+              return "Token already received successfully";
+            }
+            updateReceiveToken(database, reservation.tokenHash, {
+              state: "failed",
+              operationId: preparedOperationId,
+              error: latest.error || message,
+            });
+          } else {
+            // A prepared or missing operation had no known mint side effect.
+            deleteReceiveTokenReservation(database, reservation.tokenHash);
+          }
+        } else {
+          // Decode/validation failed before coco created an operation. Do not
+          // permanently reserve malformed input or transient mint-fetch errors.
+          releaseReceiveToken(database, reservation.tokenHash);
+        }
+        throw error;
+      }
     },
 
     async receiveBolt11(amount: number, mintUrl?: string): Promise<string> {
+      await waitForRecovery();
       const targetMint = mintUrl
         ? normalizeMintUrl(mintUrl)
         : walletConfig.defaultMintUrl;
@@ -719,6 +1403,7 @@ export async function createCocoClient(
     },
 
     async sendCashu(amount: number, mintUrl?: string): Promise<string> {
+      await waitForRecovery();
       const targetMint = mintUrl
         ? normalizeMintUrl(mintUrl)
         : walletConfig.defaultMintUrl;
@@ -734,6 +1419,7 @@ export async function createCocoClient(
     },
 
     async sendBolt11(invoice: string, mintUrl?: string): Promise<string> {
+      await waitForRecovery();
       const targetMint = mintUrl
         ? normalizeMintUrl(mintUrl)
         : walletConfig.defaultMintUrl;
@@ -755,6 +1441,7 @@ export async function createCocoClient(
     },
 
     async addMint(url: string): Promise<string> {
+      await waitForRecovery();
       const mintUrl = normalizeMintUrl(url);
       await coco.mint.addMint(mintUrl, { trusted: true });
       return `Mint ${mintUrl} added successfully`;
@@ -769,6 +1456,7 @@ export async function createCocoClient(
     },
 
     async setDefaultMint(url: string): Promise<string> {
+      await waitForRecovery();
       const mintUrl = normalizeMintUrl(url);
       const trustedMints = await coco.mint.getAllTrustedMints();
       if (!trustedMints.some((mint) => mint.mintUrl === mintUrl)) {
@@ -784,6 +1472,9 @@ export async function createCocoClient(
       if (disposed) return;
       disposed = true;
       try {
+        // Let any in-flight recovery settle before closing the database from
+        // underneath it. The recovery promise resolves on success or failure.
+        await recoveryPromise;
         await coco.dispose();
       } finally {
         try {
@@ -817,6 +1508,7 @@ export async function createCocoClient(
       username: string,
       confirm?: boolean,
     ): Promise<NpcUsernameResult> {
+      await waitForRecovery();
       const result = await npcApi().setUsername(username, confirm === true);
       if (result.success) {
         return { success: true };
@@ -829,7 +1521,108 @@ export async function createCocoClient(
     },
 
     async syncNpc(): Promise<void> {
+      await waitForRecovery();
       await npcApi().sync();
     },
+
+    async cleanupStuckOperations(
+      options: WalletCleanupOptions = {},
+    ): Promise<WalletCleanupResult> {
+      await waitForRecovery();
+      const minAgeMs = options.minAgeMs ?? 7 * 24 * 60 * 60 * 1000;
+      const dryRun = options.dryRun === true;
+      const nowMs = Date.now();
+
+      const [pendingMints, inFlightSends, preparedMelts] = await Promise.all([
+        coco.ops.mint.listPending(),
+        coco.ops.send.listInFlight(),
+        coco.ops.melt.listPrepared(),
+      ]);
+
+      const filteredMints = options.mintUrl
+        ? pendingMints.filter((op) => op.mintUrl === options.mintUrl)
+        : pendingMints;
+      const filteredSends = options.mintUrl
+        ? inFlightSends.filter((op) => op.mintUrl === options.mintUrl)
+        : inFlightSends;
+      const filteredMelts = options.mintUrl
+        ? preparedMelts.filter((op) => op.mintUrl === options.mintUrl)
+        : preparedMelts;
+
+      const selection = selectCleanupOperations({
+        mints: filteredMints,
+        sends: filteredSends,
+        melts: filteredMelts,
+        nowMs,
+        minAgeMs,
+      });
+
+      const errors: WalletCleanupResult["errors"] = [];
+
+      if (!dryRun) {
+        const mintService = (
+          coco as unknown as {
+            mintOperationService: MintOperationServiceCleanup;
+          }
+        ).mintOperationService;
+
+        for (const op of selection.mintsToFail) {
+          try {
+            await mintService.failPendingOperation(
+              { id: op.id },
+              {
+                reason: "Expired unpaid mint quote cleaned up by routstrd",
+                retryable: false,
+                observedAt: nowMs,
+              },
+            );
+          } catch (error) {
+            errors.push({
+              operationId: op.id,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        }
+
+        for (const op of selection.sendsToReclaim) {
+          try {
+            await coco.ops.send.reclaim(op.id);
+          } catch (error) {
+            errors.push({
+              operationId: op.id,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        }
+
+        for (const op of selection.meltsToCancel) {
+          try {
+            await coco.ops.melt.cancel(op.id, "Cancelled by wallet cleanup");
+          } catch (error) {
+            errors.push({
+              operationId: op.id,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        }
+      }
+
+      const actedOn =
+        selection.mintsToFail.length +
+        selection.sendsToReclaim.length +
+        selection.meltsToCancel.length;
+      const skipped =
+        filteredMints.length + filteredSends.length + filteredMelts.length -
+        actedOn;
+
+      return {
+        dryRun,
+        failedMintQuotes: selection.mintsToFail.length,
+        reclaimedSends: selection.sendsToReclaim.length,
+        cancelledMelts: selection.meltsToCancel.length,
+        skipped,
+        errors,
+      };
+    },
   };
 }