zeitlich 0.2.37 → 0.2.39

package/dist/workflow.cjs

@@ -113,7 +113,7 @@ function createToolRouter(options) {
       });
     }
   }
-  async function processToolCall(toolCall, turn, sandboxId) {
+  async function processToolCall(toolCall, turn, sandboxId, onRewindRequested) {
     const startTime = Date.now();
     const tool = toolMap.get(toolCall.name);
     const preResult = await runPreHooks(toolCall, tool, turn);
@@ -128,7 +128,7 @@ function createToolRouter(options) {
           reason: "Skipped by PreToolUse hook"
         })
       });
-      return null;
+      return { kind: "skipped" };
     }
     const effectiveArgs = preResult.args;
     workflow.log.debug("tool call dispatched", {
@@ -140,6 +140,7 @@ function createToolRouter(options) {
     let content;
     let resultAppended = false;
     let metadata;
+    let rewindRequested = false;
     try {
       if (tool) {
         const routerContext = {
@@ -157,11 +158,15 @@ function createToolRouter(options) {
         content = response.toolResponse;
         resultAppended = response.resultAppended === true;
         metadata = response.metadata;
+        rewindRequested = response.rewind === true;
       } else {
         result = { error: `Unknown tool: ${toolCall.name}` };
         content = JSON.stringify(result, null, 2);
       }
     } catch (error) {
+      if (workflow.isCancellation(error)) {
+        throw error;
+      }
       workflow.log.warn("tool call failed", {
         toolName: toolCall.name,
         toolCallId: toolCall.id,
@@ -179,6 +184,15 @@ function createToolRouter(options) {
       result = recovery.result;
       content = recovery.content;
     }
+    if (rewindRequested) {
+      const signal = {
+        toolCallId: toolCall.id,
+        toolName: toolCall.name
+      };
+      workflow.log.info("tool requested rewind", { ...signal });
+      onRewindRequested?.(signal);
+      return { kind: "rewind", signal };
+    }
     if (!resultAppended) {
       const config = {
         threadId: options.threadId,
@@ -215,7 +229,7 @@ function createToolRouter(options) {
       turn,
       durationMs
     );
-    return toolResult;
+    return { kind: "result", value: toolResult };
   }
   return {
     hasTools() {
@@ -250,27 +264,59 @@ function createToolRouter(options) {
       }));
     },
     async processToolCalls(toolCalls, context) {
+      const attachRewind = (arr, rewind) => {
+        if (rewind) {
+          arr.rewind = rewind;
+        }
+        return arr;
+      };
       if (toolCalls.length === 0) {
-        return [];
+        return attachRewind([], void 0);
       }
       const turn = context?.turn ?? 0;
       const sandboxId = context?.sandboxId;
+      let rewindSignal;
       if (options.parallel) {
-        const results2 = await Promise.all(
-          toolCalls.map((tc) => processToolCall(tc, turn, sandboxId))
-        );
-        return results2.filter(
-          (r) => r !== null
+        const scope = new workflow.CancellationScope({ cancellable: true });
+        const onRewindRequested = (signal) => {
+          if (!rewindSignal) {
+            rewindSignal = signal;
+            scope.cancel();
+          }
+        };
+        const outcomes = await scope.run(
+          async () => Promise.allSettled(
+            toolCalls.map(
+              (tc) => processToolCall(tc, turn, sandboxId, onRewindRequested)
+            )
+          )
         );
+        const results2 = [];
+        for (const outcome of outcomes) {
+          if (outcome.status === "rejected") {
+            if (workflow.isCancellation(outcome.reason)) {
+              continue;
+            }
+            throw outcome.reason;
+          }
+          if (outcome.value.kind === "result") {
+            results2.push(outcome.value.value);
+          }
+        }
+        return attachRewind(results2, rewindSignal);
       }
       const results = [];
       for (const toolCall of toolCalls) {
-        const result = await processToolCall(toolCall, turn, sandboxId);
-        if (result !== null) {
-          results.push(result);
+        const outcome = await processToolCall(toolCall, turn, sandboxId);
+        if (outcome.kind === "rewind") {
+          rewindSignal = outcome.signal;
+          break;
+        }
+        if (outcome.kind === "result") {
+          results.push(outcome.value);
         }
       }
-      return results;
+      return attachRewind(results, rewindSignal);
     },
     async processToolCallsByName(toolCalls, toolName, handler, context) {
       const matchingCalls = toolCalls.filter((tc) => tc.name === toolName);
@@ -394,6 +440,7 @@ function createSubagentTool(subagents) {
 var childSandboxReadySignal = workflow.defineSignal("childSandboxReady");
 
 // src/lib/subagent/handler.ts
+var DEFAULT_SUBAGENT_WORKFLOW_RUN_TIMEOUT = "1h";
 function resolveSandboxConfig(config) {
   if (!config || config === "none") {
     return { source: "none", init: "per-call", continuation: "fork" };
@@ -425,17 +472,28 @@ function createSubagentHandler(subagents) {
   const threadSandboxes = /* @__PURE__ */ new Map();
   const persistentSandboxes = /* @__PURE__ */ new Map();
   const persistentSandboxCreating = /* @__PURE__ */ new Set();
+  const persistentSandboxCreationError = /* @__PURE__ */ new Map();
   const lazyCreatorAgent = /* @__PURE__ */ new Map();
+  const snapshotBaseCreatorAgent = /* @__PURE__ */ new Map();
   const threadSnapshots = /* @__PURE__ */ new Map();
   const persistentBaseSnapshot = /* @__PURE__ */ new Map();
   const persistentBaseSnapshotCreating = /* @__PURE__ */ new Set();
-  workflow.setHandler(childSandboxReadySignal, ({ childWorkflowId, sandboxId }) => {
-    const agentName = lazyCreatorAgent.get(childWorkflowId);
-    if (agentName && !persistentSandboxes.has(agentName)) {
-      persistentSandboxes.set(agentName, sandboxId);
-      lazyCreatorAgent.delete(childWorkflowId);
+  const persistentBaseSnapshotCreationError = /* @__PURE__ */ new Map();
+  workflow.setHandler(
+    childSandboxReadySignal,
+    ({ childWorkflowId, sandboxId, baseSnapshot }) => {
+      const lazyAgent = lazyCreatorAgent.get(childWorkflowId);
+      if (lazyAgent && !persistentSandboxes.has(lazyAgent)) {
+        persistentSandboxes.set(lazyAgent, sandboxId);
+        lazyCreatorAgent.delete(childWorkflowId);
+      }
+      const snapAgent = snapshotBaseCreatorAgent.get(childWorkflowId);
+      if (snapAgent && baseSnapshot && !persistentBaseSnapshot.has(snapAgent)) {
+        persistentBaseSnapshot.set(snapAgent, baseSnapshot);
+        snapshotBaseCreatorAgent.delete(childWorkflowId);
+      }
     }
-  });
+  );
   const handler = async (args, context) => {
     const config = subagents.find((s) => s.agentName === args.subagent);
     if (!config) {
@@ -491,8 +549,20 @@ function createSubagentHandler(subagents) {
         baseSnap = persistentBaseSnapshot.get(config.agentName);
         if (!baseSnap) {
           if (persistentBaseSnapshotCreating.has(config.agentName)) {
-            await workflow.condition(() => persistentBaseSnapshot.has(config.agentName));
+            await workflow.condition(
+              () => persistentBaseSnapshot.has(config.agentName) || persistentBaseSnapshotCreationError.has(config.agentName) || !persistentBaseSnapshotCreating.has(config.agentName)
+            );
+            const creatorErr = persistentBaseSnapshotCreationError.get(
+              config.agentName
+            );
+            if (creatorErr !== void 0) {
+              throw creatorErr;
+            }
             baseSnap = persistentBaseSnapshot.get(config.agentName);
+            if (!baseSnap) {
+              persistentBaseSnapshotCreating.add(config.agentName);
+              isSnapshotBaseCreator = true;
+            }
           } else {
             persistentBaseSnapshotCreating.add(config.agentName);
             isSnapshotBaseCreator = true;
@@ -510,8 +580,20 @@ function createSubagentHandler(subagents) {
         baseSandboxId = persistentSandboxes.get(config.agentName);
         if (!baseSandboxId) {
           if (persistentSandboxCreating.has(config.agentName)) {
-            await workflow.condition(() => persistentSandboxes.has(config.agentName));
+            await workflow.condition(
+              () => persistentSandboxes.has(config.agentName) || persistentSandboxCreationError.has(config.agentName) || !persistentSandboxCreating.has(config.agentName)
+            );
+            const creatorErr = persistentSandboxCreationError.get(
+              config.agentName
+            );
+            if (creatorErr !== void 0) {
+              throw creatorErr;
+            }
             baseSandboxId = persistentSandboxes.get(config.agentName);
+            if (!baseSandboxId) {
+              persistentSandboxCreating.add(config.agentName);
+              isLazyCreator = true;
+            }
           } else {
             persistentSandboxCreating.add(config.agentName);
             isLazyCreator = true;
@@ -540,6 +622,12 @@ function createSubagentHandler(subagents) {
     };
     const resolvedContext = config.context === void 0 ? void 0 : typeof config.context === "function" ? config.context() : config.context;
     const childOpts = {
+      // Apply a bounded run timeout by default so a child workflow that
+      // fails to initialize or otherwise never reaches a terminal state
+      // cannot hang the parent's `Subagent` tool call forever. Callers can
+      // raise, lower, or disable it via `workflowOptions.workflowRunTimeout`.
+      workflowRunTimeout: DEFAULT_SUBAGENT_WORKFLOW_RUN_TIMEOUT,
+      ...config.workflowOptions ?? {},
       workflowId: childWorkflowId,
       args: resolvedContext === void 0 ? [args.prompt, workflowInput] : [args.prompt, workflowInput, resolvedContext],
       taskQueue: config.taskQueue ?? parentTaskQueue
@@ -547,13 +635,39 @@ function createSubagentHandler(subagents) {
     if (isLazyCreator) {
       lazyCreatorAgent.set(childWorkflowId, config.agentName);
     }
+    if (isSnapshotBaseCreator) {
+      snapshotBaseCreatorAgent.set(childWorkflowId, config.agentName);
+    }
     workflow.log.info("subagent spawned", {
       subagent: config.agentName,
       childWorkflowId,
       threadMode,
       sandboxSource: sandboxCfg.source
     });
-    const childResult = await workflow.executeChild(config.workflow, childOpts);
+    let childResult;
+    try {
+      childResult = await workflow.executeChild(
+        config.workflow,
+        childOpts
+      );
+    } catch (err) {
+      workflow.log.warn("subagent failed", {
+        subagent: config.agentName,
+        childWorkflowId,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      if (isLazyCreator) {
+        persistentSandboxCreating.delete(config.agentName);
+        persistentSandboxCreationError.set(config.agentName, err);
+        lazyCreatorAgent.delete(childWorkflowId);
+      }
+      if (isSnapshotBaseCreator) {
+        persistentBaseSnapshotCreating.delete(config.agentName);
+        persistentBaseSnapshotCreationError.set(config.agentName, err);
+        snapshotBaseCreatorAgent.delete(childWorkflowId);
+      }
+      throw err;
+    }
     const effectiveShutdown = sandboxShutdownOverride ?? sandboxCfg.shutdown ?? "destroy";
     workflow.log.info("subagent completed", {
       subagent: config.agentName,
@@ -597,10 +711,13 @@ function createSubagentHandler(subagents) {
     }
     if (isLazyCreator) {
       persistentSandboxCreating.delete(config.agentName);
+      persistentSandboxCreationError.delete(config.agentName);
       lazyCreatorAgent.delete(childWorkflowId);
     }
     if (isSnapshotBaseCreator) {
       persistentBaseSnapshotCreating.delete(config.agentName);
+      persistentBaseSnapshotCreationError.delete(config.agentName);
+      snapshotBaseCreatorAgent.delete(childWorkflowId);
     }
     if (!toolResponse) {
       return {
@@ -843,6 +960,7 @@ async function createSession({
   sandbox: sandboxInit,
   sandboxShutdown = "destroy",
   onSandboxReady,
+  onSessionExit,
   virtualFs: virtualFsConfig,
   virtualFsOps
 }) {
@@ -867,7 +985,9 @@ async function createSession({
     initializeThread,
     appendSystemMessage,
     appendAgentMessage,
-    forkThread
+    forkThread,
+    loadThreadState,
+    saveThreadState
   } = threadOps;
   const plugins = [];
   let destroySubagentSandboxes;
@@ -961,8 +1081,10 @@ async function createSession({
             nonRetryable: true
           });
         }
+        const forkInit = sandboxInit;
         sandboxId = await sandboxOps.forkSandbox(
-          sandboxInit.sandboxId
+          forkInit.sandboxId,
+          forkInit.options
         );
         sandboxOwned = true;
       } else if (sandboxMode === "from-snapshot") {
@@ -972,8 +1094,11 @@ async function createSession({
             nonRetryable: true
           });
         }
-        const snap = sandboxInit.snapshot;
-        sandboxId = await sandboxOps.restoreSandbox(snap);
+        const restoreInit = sandboxInit;
+        sandboxId = await sandboxOps.restoreSandbox(
+          restoreInit.snapshot,
+          restoreInit.options
+        );
         sandboxOwned = true;
       } else if (sandboxOps) {
         const skillFiles = skills ? collectSkillFiles(skills) : void 0;
@@ -990,7 +1115,10 @@ async function createSession({
         baseSnapshot = await sandboxOps.snapshotSandbox(sandboxId);
       }
       if (sandboxId && sandboxOwned && onSandboxReady) {
-        onSandboxReady(sandboxId);
+        onSandboxReady({
+          sandboxId,
+          ...baseSnapshot && { baseSnapshot }
+        });
       }
       if (virtualFsConfig) {
         if (!virtualFsOps) {
@@ -1033,9 +1161,20 @@ async function createSession({
       });
       const sessionStartMs = Date.now();
       const systemPrompt = stateManager.getSystemPrompt();
+      const rehydrateFromSlice = (slice) => {
+        stateManager.mergeUpdate({
+          tasks: new Map(slice.tasks),
+          ...slice.custom
+        });
+      };
       if (threadMode === "fork" && sourceThreadId) {
         await forkThread(sourceThreadId, threadId, threadKey);
-      } else if (threadMode === "continue") ; else {
+        const forkedSlice = await loadThreadState(threadId, threadKey);
+        if (forkedSlice) rehydrateFromSlice(forkedSlice);
+      } else if (threadMode === "continue") {
+        const continuedSlice = await loadThreadState(threadId, threadKey);
+        if (continuedSlice) rehydrateFromSlice(continuedSlice);
+      } else {
         if (appendSystemPrompt) {
           if (systemPrompt == null || typeof systemPrompt === "string" && systemPrompt.trim() === "") {
             throw workflow.ApplicationFailure.create({
@@ -1057,18 +1196,21 @@ async function createSession({
       let exitReason = "completed";
       let finalMessage = null;
       try {
+        let assistantId;
         while (stateManager.isRunning() && !stateManager.isTerminal() && stateManager.getTurns() < maxTurns) {
           stateManager.incrementTurns();
           const currentTurn = stateManager.getTurns();
           workflow.log.debug("turn started", { agentName, threadId, turn: currentTurn });
           stateManager.setTools(toolRouter.getToolDefinitions());
+          assistantId ??= workflow.uuid4();
           const { message, rawToolCalls, usage } = await runAgent({
             threadId,
             threadKey,
             agentName,
-            metadata
+            metadata,
+            assistantMessageId: assistantId
           });
-          await appendAgentMessage(threadId, workflow.uuid4(), message, threadKey);
+          await appendAgentMessage(threadId, assistantId, message, threadKey);
           if (usage) {
             stateManager.updateUsage(usage);
           }
@@ -1113,6 +1255,18 @@ async function createSession({
               stateManager.updateUsage(result.usage);
             }
           }
+          const rewind = toolCallResults.rewind;
+          if (rewind) {
+            workflow.log.info("rewinding turn", {
+              agentName,
+              threadId,
+              turn: currentTurn,
+              toolCallId: rewind.toolCallId,
+              toolName: rewind.toolName
+            });
+            continue;
+          }
+          assistantId = void 0;
           if (stateManager.getStatus() === "WAITING_FOR_INPUT") {
             const conditionMet = await workflow.condition(
               () => stateManager.getStatus() === "RUNNING",
@@ -1145,6 +1299,19 @@ async function createSession({
         });
         throw workflow.ApplicationFailure.fromError(error);
       } finally {
+        try {
+          await saveThreadState(
+            threadId,
+            stateManager.getPersistedSlice(),
+            threadKey
+          );
+        } catch (persistError) {
+          workflow.log.warn("failed to persist thread state", {
+            agentName,
+            threadId,
+            error: persistError instanceof Error ? persistError.message : String(persistError)
+          });
+        }
         await callSessionEnd(exitReason, stateManager.getTurns());
         if (sandboxOwned && sandboxId && sandboxOps) {
           switch (sandboxShutdown) {
@@ -1181,6 +1348,12 @@ async function createSession({
         ...baseSnapshot && { hasBaseSnapshot: true },
         ...exitSnapshot && { hasExitSnapshot: true }
       });
+      if (onSessionExit) {
+        onSessionExit({
+          ...sandboxId && { sandboxId },
+          ...exitSnapshot && { snapshot: exitSnapshot }
+        });
+      }
       return {
         threadId,
         finalMessage,
@@ -1209,6 +1382,15 @@ function defineWorkflow(config, fn) {
   return workflow;
 }
 
+// src/lib/thread/keys.ts
+var THREAD_TTL_SECONDS = 60 * 60 * 24 * 90;
+function getThreadListKey(threadKey, threadId) {
+  return `${threadKey}:thread:${threadId}`;
+}
+function getThreadMetaKey(threadKey, threadId) {
+  return `${threadKey}:meta:thread:${threadId}`;
+}
+
 // src/lib/types.ts
 function isTerminalStatus(status) {
   return status === "COMPLETED" || status === "FAILED" || status === "CANCELLED";
@@ -1228,11 +1410,19 @@ function createAgentStateManager({
   let systemPrompt = initialState?.systemPrompt;
   const tasks = new Map(initialState?.tasks);
   const {
-    status: _,
-    version: __,
-    turns: ___,
-    tasks: ____,
-    tools: _____,
+    status: _status,
+    version: _version,
+    turns: _turns,
+    tasks: _tasks,
+    tools: _tools,
+    systemPrompt: _systemPrompt,
+    fileTree: _fileTree,
+    inlineFiles: _inlineFiles,
+    virtualFsCtx: _virtualFsCtx,
+    totalInputTokens: _totalInputTokens,
+    totalOutputTokens: _totalOutputTokens,
+    cachedWriteTokens: _cachedWriteTokens,
+    cachedReadTokens: _cachedReadTokens,
     ...custom
   } = initialState ?? {};
   const customState = custom;
@@ -1312,7 +1502,14 @@ function createAgentStateManager({
       version++;
     },
     mergeUpdate(update) {
-      Object.assign(customState, update);
+      const { tasks: nextTasks, ...rest } = update;
+      if (nextTasks) {
+        tasks.clear();
+        for (const [id, task] of nextTasks) {
+          tasks.set(id, task);
+        }
+      }
+      Object.assign(customState, rest);
       version++;
     },
     getCurrentState() {
@@ -1350,6 +1547,12 @@ function createAgentStateManager({
       }
       return deleted;
     },
+    getPersistedSlice() {
+      return {
+        tasks: Array.from(tasks.entries()),
+        custom: { ...customState }
+      };
+    },
     updateUsage(usage) {
       totalInputTokens += usage.inputTokens ?? 0;
       totalOutputTokens += usage.outputTokens ?? 0;
@@ -1393,22 +1596,42 @@ function defineSubagentWorkflow(config, fn) {
       });
     }
     const parentHandle = workflow.getExternalWorkflowHandle(parent.workflowId);
+    let capturedSandboxId;
+    let capturedSnapshot;
+    let capturedBaseSnapshot;
+    let capturedThreadId;
     const sessionInput = {
       agentName: config.name,
       sandboxShutdown: effectiveShutdown,
       ...workflowInput.thread && { thread: workflowInput.thread },
       ...workflowInput.sandbox && { sandbox: workflowInput.sandbox },
-      onSandboxReady: (sandboxId) => {
+      onSandboxReady: ({ sandboxId, baseSnapshot }) => {
+        capturedBaseSnapshot = baseSnapshot;
         const isReuse = workflowInput.sandbox?.mode === "continue";
         if (!isReuse) {
           void parentHandle.signal(childSandboxReadySignal, {
             childWorkflowId: workflow.workflowInfo().workflowId,
-            sandboxId
+            sandboxId,
+            ...baseSnapshot && { baseSnapshot }
           });
         }
+      },
+      onSessionExit: ({ sandboxId, snapshot, threadId }) => {
+        capturedSandboxId = sandboxId;
+        capturedSnapshot = snapshot;
+        capturedThreadId = threadId;
+      }
+    };
+    const result = await fn(prompt, sessionInput, context ?? {});
+    return {
+      ...result,
+      ...capturedThreadId !== void 0 && { threadId: capturedThreadId },
+      ...capturedSandboxId !== void 0 && { sandboxId: capturedSandboxId },
+      ...capturedSnapshot !== void 0 && { snapshot: capturedSnapshot },
+      ...capturedBaseSnapshot !== void 0 && {
+        baseSnapshot: capturedBaseSnapshot
       }
     };
-    return fn(prompt, sessionInput, context ?? {});
   };
   Object.defineProperty(workflow$1, "name", { value: config.name });
   return Object.assign(workflow$1, {
@@ -2067,8 +2290,10 @@ var createAskUserQuestionHandler = () => async (args) => {
   };
 };
 
+exports.DEFAULT_SUBAGENT_WORKFLOW_RUN_TIMEOUT = DEFAULT_SUBAGENT_WORKFLOW_RUN_TIMEOUT;
 exports.SandboxNotFoundError = SandboxNotFoundError;
 exports.SandboxNotSupportedError = SandboxNotSupportedError;
+exports.THREAD_TTL_SECONDS = THREAD_TTL_SECONDS;
 exports.applyVirtualTreeMutations = applyVirtualTreeMutations;
 exports.askUserQuestionTool = askUserQuestionTool;
 exports.bashTool = bashTool;
@@ -2093,6 +2318,8 @@ exports.editTool = editTool;
 exports.filesWithMimeType = filesWithMimeType;
 exports.formatVirtualFileTree = formatVirtualFileTree;
 exports.getShortId = getShortId;
+exports.getThreadListKey = getThreadListKey;
+exports.getThreadMetaKey = getThreadMetaKey;
 exports.globTool = globTool;
 exports.grepTool = grepTool;
 exports.hasDirectory = hasDirectory;