stitchkit 0.116.0 → 0.117.0

package/dist/{index-n3v695sb.js → index-hdjmdwsj.js}

@@ -33,8 +33,9 @@ import {
   assistantStatus,
   ProviderJournalMessageSchema,
   ProviderJournalRequestSchema,
-  ProviderJournalResponseSchema
-} from "./index-fggxgbrk.js";
+  ProviderJournalResponseSchema,
+  isActiveRunState
+} from "./index-xx1j310k.js";
 import {
   messageOf
 } from "./index-g149e9pk.js";
@@ -52,6 +53,24 @@ import {
   transportResult
 } from "./index-9jfdm5tg.js";
 
+// src/agent-runtime/run-store-retry.ts
+var STORE_RETRY_DELAYS_MS = [25, 100, 400];
+function storeRetryDelay(failures) {
+  return STORE_RETRY_DELAYS_MS[failures];
+}
+async function withStoreRetries(work) {
+  for (let failures = 0;; failures += 1) {
+    try {
+      return await work();
+    } catch (error) {
+      const delay = storeRetryDelay(failures);
+      if (delay === undefined)
+        throw error;
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+}
+
 // src/agent-runtime/terminal-commit.ts
 class AgentRuntimeConflictError2 extends Error {
   constructor(operation) {
@@ -59,6 +78,18 @@ class AgentRuntimeConflictError2 extends Error {
     this.name = "AgentRuntimeConflictError";
   }
 }
+
+class AgentRunUnsettledError2 extends Error {
+  run;
+  ownerId;
+  constructor(run, ownerId, cause) {
+    const reason = cause instanceof Error ? cause.message : String(cause);
+    super(`Agent run ${run.id} is still ${run.state} with no executor: ${reason}`, { cause });
+    this.name = "AgentRunUnsettledError";
+    this.run = run;
+    this.ownerId = ownerId;
+  }
+}
 function appliedSnapshot(result, operation) {
   if (result.outcome === "applied" || result.outcome === "duplicate")
     return result.snapshot;
@@ -74,6 +105,12 @@ function viewOfSnapshot(snapshot, runId, retainedAssistant) {
     ...assistant && { assistant }
   };
 }
+function terminalOfSnapshot(snapshot, runId) {
+  return canonicalTerminal(viewOfSnapshot(snapshot, runId));
+}
+function terminalOfRunView(view, conversationId) {
+  return canonicalTerminal(viewOfRun(view, conversationId));
+}
 function viewOfRun(view, conversationId) {
   return {
     snapshotVersion: view.snapshotVersion,
@@ -122,55 +159,92 @@ function canRetryTerminal(current, previous, runtimeEpoch) {
   return (current.state === "running" || current.state === "interrupt_requested") && current.ownerId === runtimeEpoch && current.fencingToken === previous.fencingToken;
 }
 var TERMINAL_COMMIT_ATTEMPTS = 32;
+async function attemptTerminalCommit(input, candidate) {
+  const committed = await input.store.commitRunTerminal({
+    conversationId: candidate.run.conversationId,
+    runId: candidate.run.id,
+    expectedRevision: candidate.run.revision,
+    ownerId: input.runtimeEpoch,
+    ...candidate.run.fencingToken !== undefined && {
+      fencingToken: candidate.run.fencingToken
+    },
+    assistant: candidate.assistant,
+    reason: candidate.reason,
+    ...candidate.policyName && { policyName: candidate.policyName },
+    ...candidate.usage && { usage: candidate.usage },
+    ...candidate.absorb?.length && { absorb: candidate.absorb }
+  });
+  if (committed.outcome === "applied") {
+    const terminal = canonicalTerminal(viewOfSnapshot(committed.snapshot, candidate.run.id));
+    if (!terminal)
+      throw new AgentRuntimeConflictError2("terminal result projection");
+    const absorbed = committed.snapshot.runs.filter((run) => run.absorbedIntoRunId === candidate.run.id);
+    return {
+      settled: {
+        ...terminal,
+        committedByCaller: true,
+        ...absorbed.length > 0 && { absorbed }
+      }
+    };
+  }
+  if (committed.outcome === "duplicate") {
+    const terminal = canonicalTerminal(viewOfSnapshot(committed.snapshot, candidate.run.id, committed.assistant));
+    if (!terminal)
+      throw new AgentRuntimeConflictError2("terminal result projection");
+    return { settled: terminal };
+  }
+  if (committed.outcome !== "conflict") {
+    throw new AgentRuntimeConflictError2("terminal commit");
+  }
+  const latest = await input.store.loadRun({
+    conversationId: candidate.run.conversationId,
+    runId: candidate.run.id
+  });
+  const terminal = latest ? canonicalTerminal(viewOfRun(latest, candidate.run.conversationId)) : undefined;
+  if (terminal)
+    return { settled: terminal };
+  const current = latest?.run;
+  if (!current || !canRetryTerminal(current, candidate.run, input.runtimeEpoch)) {
+    throw new AgentRuntimeConflictError2("terminal commit");
+  }
+  return {
+    candidate: current.state === "interrupt_requested" ? interruptedCandidate(candidate, current, input.now) : { ...candidate, run: current }
+  };
+}
+async function absorbedBy(store, candidate) {
+  const views = await Promise.all((candidate.absorb ?? []).map(({ runId }) => store.loadRun({ conversationId: candidate.run.conversationId, runId }).catch(() => {
+    return;
+  })));
+  return views.flatMap((view) => view?.run.absorbedIntoRunId === candidate.run.id ? [view.run] : []);
+}
 async function commitAgentRunTerminal(input) {
   let candidate = input.candidate;
+  let storeFailures = 0;
   for (let attempt = 0;attempt < TERMINAL_COMMIT_ATTEMPTS; attempt += 1) {
-    const committed = await input.store.commitRunTerminal({
-      conversationId: candidate.run.conversationId,
-      runId: candidate.run.id,
-      expectedRevision: candidate.run.revision,
-      ownerId: input.runtimeEpoch,
-      ...candidate.run.fencingToken !== undefined && {
-        fencingToken: candidate.run.fencingToken
-      },
-      assistant: candidate.assistant,
-      reason: candidate.reason,
-      ...candidate.policyName && { policyName: candidate.policyName },
-      ...candidate.usage && { usage: candidate.usage },
-      ...candidate.absorb?.length && { absorb: candidate.absorb }
-    });
-    if (committed.outcome === "applied") {
-      const terminal = canonicalTerminal(viewOfSnapshot(committed.snapshot, candidate.run.id));
-      if (!terminal)
-        throw new AgentRuntimeConflictError2("terminal result projection");
-      const absorbed = committed.snapshot.runs.filter((run) => run.absorbedIntoRunId === candidate.run.id);
+    try {
+      const round = await attemptTerminalCommit(input, candidate);
+      if (!round.settled) {
+        candidate = round.candidate;
+        continue;
+      }
+      const owned = storeFailures > 0 && round.settled.run.ownerId === input.runtimeEpoch && round.settled.run.fencingToken === candidate.run.fencingToken;
+      if (!owned)
+        return round.settled;
+      const absorbed = await absorbedBy(input.store, candidate);
       return {
-        ...terminal,
+        ...round.settled,
         committedByCaller: true,
         ...absorbed.length > 0 && { absorbed }
       };
+    } catch (error) {
+      if (error instanceof AgentRuntimeConflictError2)
+        throw error;
+      const delay = storeRetryDelay(storeFailures);
+      if (delay === undefined)
+        throw error;
+      storeFailures += 1;
+      await new Promise((resolve) => setTimeout(resolve, delay));
     }
-    if (committed.outcome === "duplicate") {
-      const terminal = canonicalTerminal(viewOfSnapshot(committed.snapshot, candidate.run.id, committed.assistant));
-      if (!terminal)
-        throw new AgentRuntimeConflictError2("terminal result projection");
-      return terminal;
-    }
-    if (committed.outcome !== "conflict") {
-      throw new AgentRuntimeConflictError2("terminal commit");
-    }
-    const latest = await input.store.loadRun({
-      conversationId: candidate.run.conversationId,
-      runId: candidate.run.id
-    });
-    const terminal = latest ? canonicalTerminal(viewOfRun(latest, candidate.run.conversationId)) : undefined;
-    if (terminal)
-      return terminal;
-    const current = latest?.run;
-    if (!current || !canRetryTerminal(current, candidate.run, input.runtimeEpoch)) {
-      throw new AgentRuntimeConflictError2("terminal commit");
-    }
-    candidate = current.state === "interrupt_requested" ? interruptedCandidate(candidate, current, input.now) : { ...candidate, run: current };
   }
   throw new AgentRuntimeConflictError2("terminal commit");
 }
@@ -2545,19 +2619,20 @@ function createAgentRunOperationLifecycle(config) {
 
 // src/agent-runtime/run-acquisition.ts
 async function acquireRun(dependencies, input) {
-  const { config, publish, runtimeEpoch, generateId, now, idleTimeoutMs, injection } = dependencies;
+  const { config, runtimeEpoch, now, idleTimeoutMs, injection } = dependencies;
   injection?.withdraw(input.key, input.acceptedRun.id);
-  const queued = await config.store.loadRun({
+  const queued = await withStoreRetries(() => config.store.loadRun({
     conversationId: input.acceptedRun.conversationId,
     runId: input.acceptedRun.id
-  });
+  }));
   if (!queued)
     throw new AgentRuntimeConflictError2("run lookup");
   if (queued.run.terminalReason === "absorbed") {
-    const absorbing = queued.run.absorbedIntoRunId ? await config.store.loadRun({
+    const { absorbedIntoRunId } = queued.run;
+    const absorbing = absorbedIntoRunId ? await withStoreRetries(() => config.store.loadRun({
       conversationId: queued.run.conversationId,
-      runId: queued.run.absorbedIntoRunId
-    }) : undefined;
+      runId: absorbedIntoRunId
+    })) : undefined;
     if (!absorbing?.assistant || !absorbing.run.terminalReason) {
       throw new AgentRuntimeConflictError2("absorbed run resolution");
     }
@@ -2576,38 +2651,16 @@ async function acquireRun(dependencies, input) {
     };
   }
   const queuedRun = queued.run;
-  const acquired = appliedSnapshot(await config.store.acquireRun({
+  const acquired = appliedSnapshot(await withStoreRetries(() => config.store.acquireRun({
     conversationId: queuedRun.conversationId,
     runId: queuedRun.id,
     expectedRevision: queuedRun.revision,
     ownerId: runtimeEpoch
-  }), "run acquisition");
-  let run = findRun(acquired.runs, input.acceptedRun.id);
+  })), "run acquisition");
+  const run = findRun(acquired.runs, input.acceptedRun.id);
   input.onAcquired?.();
-  await publish({
-    type: "run-state",
-    eventId: agentDurableEventId2("run-state", run.id, acquired.version),
-    conversationId: run.conversationId,
-    runId: run.id,
-    snapshotVersion: acquired.version,
-    state: run.state,
-    emittedAt: now().toISOString()
-  });
   const trace = config.observe?.rootTrace();
   const runStartedAt = performance.now();
-  config.observe?.emit({
-    schemaVersion: 1,
-    eventId: generateId(),
-    type: "run-started",
-    conversationId: run.conversationId,
-    runId: run.id,
-    traceId: trace?.traceId ?? generateId(),
-    spanId: trace?.spanId ?? generateId(),
-    ...trace?.parentSpanId && { parentSpanId: trace.parentSpanId },
-    state: run.state,
-    queueWaitMs: Math.max(0, now().getTime() - new Date(run.createdAt).getTime()),
-    emittedAt: now().toISOString()
-  });
   const assistant = AgentMessageSchema2.parse({
     schemaVersion: 1,
     id: run.assistantMessageId,
@@ -2619,19 +2672,10 @@ async function acquireRun(dependencies, input) {
     createdAt: now().toISOString(),
     updatedAt: now().toISOString()
   });
-  const snapshot = appliedSnapshot(await config.store.checkpointRunAssistant({
-    conversationId: run.conversationId,
-    runId: run.id,
-    expectedRevision: run.revision,
-    ownerId: runtimeEpoch,
-    ...run.fencingToken !== undefined && { fencingToken: run.fencingToken },
-    assistant
-  }), "assistant draft");
-  run = findRun(snapshot.runs, run.id);
   const state = {
     run,
-    snapshot,
-    observedVersion: snapshot.version,
+    snapshot: acquired,
+    observedVersion: acquired.version,
     assistant,
     parts: [],
     absorbed: new Map,
@@ -2660,7 +2704,7 @@ async function acquireRun(dependencies, input) {
     serialize,
     currentRun: () => state.run,
     acceptSnapshot: (next) => adoptSnapshot(state, next),
-    publish,
+    publish: dependencies.publish,
     now
   });
   return {
@@ -2678,6 +2722,41 @@ async function acquireRun(dependencies, input) {
     }
   };
 }
+async function beginRunExecution(execution) {
+  const { config, publish, runtimeEpoch, generateId, now } = execution.dependencies;
+  const { state, trace } = execution;
+  const { run, assistant } = state;
+  await publish({
+    type: "run-state",
+    eventId: agentDurableEventId2("run-state", run.id, state.observedVersion),
+    conversationId: run.conversationId,
+    runId: run.id,
+    snapshotVersion: state.observedVersion,
+    state: run.state,
+    emittedAt: now().toISOString()
+  });
+  config.observe?.emit({
+    schemaVersion: 1,
+    eventId: generateId(),
+    type: "run-started",
+    conversationId: run.conversationId,
+    runId: run.id,
+    traceId: trace?.traceId ?? generateId(),
+    spanId: trace?.spanId ?? generateId(),
+    ...trace?.parentSpanId && { parentSpanId: trace.parentSpanId },
+    state: run.state,
+    queueWaitMs: Math.max(0, now().getTime() - new Date(run.createdAt).getTime()),
+    emittedAt: now().toISOString()
+  });
+  adoptSnapshot(state, appliedSnapshot(await config.store.checkpointRunAssistant({
+    conversationId: run.conversationId,
+    runId: run.id,
+    expectedRevision: run.revision,
+    ownerId: runtimeEpoch,
+    ...run.fencingToken !== undefined && { fencingToken: run.fencingToken },
+    assistant
+  }), "assistant draft"));
+}
 
 // src/agent-runtime/run-failure.ts
 function failureThisRuntimeOwns(error) {
@@ -2691,16 +2770,24 @@ function failureThisRuntimeOwns(error) {
     return "runtime_failure";
   return;
 }
+function withLaterFailure(primary, later, what) {
+  return new AggregateError([primary, later], `${primary instanceof Error ? primary.message : String(primary)} (${what} also failed: ${later instanceof Error ? later.message : String(later)})`);
+}
 async function classifyRunFailure(execution, error) {
   const { config, runtimeEpoch } = execution.dependencies;
   const { state, operationLifecycle, executionSignal } = execution;
   operationLifecycle.failStepCheckpoints(error);
   const cleanupFailure = state.providerStreamCleanupFailure;
-  state.internalCause = cleanupFailure === undefined ? error : new AggregateError([error, cleanupFailure], `${error instanceof Error ? error.message : String(error)} (provider stream cleanup also failed: ${cleanupFailure instanceof Error ? cleanupFailure.message : String(cleanupFailure)})`);
-  const latest = await config.store.loadRun({
-    conversationId: state.run.conversationId,
-    runId: state.run.id
-  });
+  state.internalCause = cleanupFailure === undefined ? error : withLaterFailure(error, cleanupFailure, "provider stream cleanup");
+  let latest;
+  try {
+    latest = await config.store.loadRun({
+      conversationId: state.run.conversationId,
+      runId: state.run.id
+    });
+  } catch (readFailure) {
+    state.internalCause = withLaterFailure(state.internalCause, readFailure, "the run read");
+  }
   const latestRun = latest?.run;
   const durableInterrupt = latestRun?.ownerId === runtimeEpoch && latestRun.state === "interrupt_requested";
   if (durableInterrupt && latest && latestRun) {
@@ -2709,7 +2796,11 @@ async function classifyRunFailure(execution, error) {
   }
   const staleOwner = isToolExecutionControlError2(error) && error.reason === "stale_run";
   if (!staleOwner) {
-    await operationLifecycle.finish(executionSignal.aborted || durableInterrupt ? "cancelled" : "failed");
+    try {
+      await operationLifecycle.finish(executionSignal.aborted || durableInterrupt ? "cancelled" : "failed");
+    } catch (finishFailure) {
+      state.internalCause = withLaterFailure(state.internalCause, finishFailure, "closing the operations");
+    }
   }
   if (isToolExecutionControlError2(error) || executionSignal.aborted || durableInterrupt) {
     state.terminalReason = executionSignal.aborted ? abortTerminalReason(executionSignal) : "interrupted";
@@ -3587,7 +3678,9 @@ async function commitRunTerminal(execution) {
       reason: state.terminalReason,
       usage: spent
     });
-    throw error;
+    if (error instanceof AgentRuntimeConflictError2)
+      throw error;
+    throw new AgentRunUnsettledError2(state.run, runtimeEpoch, error);
   }
   state.observedVersion = terminal.snapshotVersion;
   state.run = terminal.run;
@@ -3879,14 +3972,102 @@ async function prepareTurn(execution) {
   return { selectedModel, runtimeContext, prompt, tools, history, projection };
 }
 
+// src/agent-runtime/run-unacquired.ts
+async function closeUnacquiredRun(dependencies, input, failure, executing) {
+  const { config, publish, runtimeEpoch, generateId, now } = dependencies;
+  const { conversationId, id: runId } = input.acceptedRun;
+  const view = await withStoreRetries(() => config.store.loadRun({ conversationId, runId })).catch(() => {
+    throw new AgentRunUnsettledError2(input.acceptedRun, runtimeEpoch, failure);
+  });
+  if (!view || !isActiveRunState(view.run.state))
+    throw failure;
+  const { run } = view;
+  if (run.ownerId !== undefined && run.ownerId !== runtimeEpoch)
+    throw failure;
+  if (executing.has(runId))
+    throw failure;
+  const unsettled = new AgentRunUnsettledError2(run, runtimeEpoch, failure);
+  if (failure instanceof AgentRuntimeConflictError2 && run.ownerId === undefined)
+    throw unsettled;
+  const abandoned = await withStoreRetries(() => config.store.recoverRun({
+    conversationId,
+    runId,
+    expectedRevision: run.revision,
+    action: "abandon"
+  })).catch(() => {
+    return;
+  });
+  const terminal = abandoned?.outcome === "applied" ? terminalOfSnapshot(abandoned.snapshot, runId) : await withStoreRetries(() => config.store.loadRun({ conversationId, runId })).then((latest) => latest ? terminalOfRunView(latest, conversationId) : undefined, () => {
+    return;
+  });
+  if (terminal?.reason !== "abandoned")
+    throw unsettled;
+  input.onAcquired?.();
+  const version = terminal.snapshotVersion;
+  const usage = statedUsage(terminal.run.usage);
+  const emittedAt = now().toISOString();
+  await publish({
+    type: "run-state",
+    eventId: agentDurableEventId2("run-state", runId, version),
+    conversationId,
+    runId,
+    snapshotVersion: version,
+    state: terminal.run.state,
+    emittedAt
+  });
+  config.observe?.emit({
+    schemaVersion: 1,
+    eventId: agentDurableEventId2("terminal", runId, version),
+    type: "run-terminal",
+    conversationId,
+    runId,
+    traceId: generateId(),
+    spanId: generateId(),
+    state: terminal.run.state,
+    terminalReason: terminal.reason,
+    durationMs: 0,
+    usage,
+    internalCause: failure,
+    emittedAt
+  });
+  const metrics = { partial: true, durationMs: 0, usage };
+  await publish({
+    type: "terminal",
+    eventId: agentDurableEventId2("terminal", runId, version),
+    conversationId,
+    runId,
+    snapshotVersion: version,
+    reason: terminal.reason,
+    message: terminal.assistant,
+    metrics,
+    emittedAt
+  });
+  return {
+    run: terminal.run,
+    message: terminal.assistant,
+    reason: terminal.reason,
+    snapshotVersion: version,
+    metrics
+  };
+}
+
 // src/agent-runtime/run-execution.ts
 function createRunExecutor(dependencies) {
+  const executing = new Set;
   return async function executeRun(input) {
-    const acquisition = await acquireRun(dependencies, input);
+    let acquisition;
+    try {
+      acquisition = await acquireRun(dependencies, input);
+    } catch (error) {
+      return closeUnacquiredRun(dependencies, input, error, executing);
+    }
     if (acquisition.kind === "absorbed")
       return acquisition.result;
     const { execution } = acquisition;
+    const runId = execution.state.run.id;
+    executing.add(runId);
     try {
+      await beginRunExecution(execution);
       const invocations = dependencies.config.invocations;
       if (invocations) {
         const trace = await invocations.trace({
@@ -3904,7 +4085,7 @@ function createRunExecutor(dependencies) {
     } finally {
       execution.idleDeadline.dispose();
     }
-    const result = await commitRunTerminal(execution);
+    const result = await commitRunTerminal(execution).finally(() => executing.delete(runId));
     await execution.invocation?.finish(result.reason === "success" ? "succeeded" : ["interrupted", "cancelled", "timeout", "shutdown"].includes(result.reason) ? "cancelled" : "failed");
     return result;
   };
@@ -4283,4 +4464,4 @@ function createAgentRuntime2(config) {
   };
 }
 
-export { AgentRuntimeConflictError2, normalizeSdkUsage, AgentContextOverflowError2, createAgentSessionCoordinator2, registerDeferredToolRepair, registerDeferredToolCatalog, deferredToolCatalog, repairedSearchCall, projectAgentHistoryDetailed2, projectAgentHistory2, ModelInvocationTraceSchema2, CompletionInvocationInputSchema2, ModelInvocationRecordSchema2, currentModelInvocationAttempt2, createModelInvocationLedger2, createAgentToolFenceLifecycle2, AgentTokenCountSchema2, selectAgentHistory2, composeAgentPrompt2, AgentProviderStreamCutError2, classifyProviderFailure2, isToolResultFailure2, recordAgentRetryDecision2, defineStateSlot2, createAgentStateSlotStore2, renderAgentStateSlots2, agentGoalStateSlot2, agentTodoStateSlot2, createAgentRuntime2 };
+export { AgentRuntimeConflictError2, AgentRunUnsettledError2, normalizeSdkUsage, AgentContextOverflowError2, createAgentSessionCoordinator2, registerDeferredToolRepair, registerDeferredToolCatalog, deferredToolCatalog, repairedSearchCall, projectAgentHistoryDetailed2, projectAgentHistory2, ModelInvocationTraceSchema2, CompletionInvocationInputSchema2, ModelInvocationRecordSchema2, currentModelInvocationAttempt2, createModelInvocationLedger2, createAgentToolFenceLifecycle2, AgentTokenCountSchema2, selectAgentHistory2, composeAgentPrompt2, AgentProviderStreamCutError2, classifyProviderFailure2, isToolResultFailure2, recordAgentRetryDecision2, defineStateSlot2, createAgentStateSlotStore2, renderAgentStateSlots2, agentGoalStateSlot2, agentTodoStateSlot2, createAgentRuntime2 };

package/dist/{index-zqkz4af9.js → index-nqxrm176.js}

@@ -1,6 +1,6 @@
 import {
   attemptExclusiveLock
-} from "./index-z3d6smer.js";
+} from "./index-tdpmfks8.js";
 import {
   sleep
 } from "./index-eemvahkc.js";