pg-workflows 0.8.3 → 0.10.0

package/dist/index.js

@@ -2,6 +2,7 @@ import {
   DEFAULT_PGBOSS_SCHEMA,
   PAUSE_EVENT_NAME,
   StepType,
+  WORKFLOW_RUN_DLQ_QUEUE_NAME,
   WORKFLOW_RUN_QUEUE_NAME,
   WorkflowClient,
   WorkflowEngineError,
@@ -11,11 +12,16 @@ import {
   getWorkflowRun,
   getWorkflowRuns,
   insertWorkflowRun,
+  invokeChildWorkflowTimelineKey,
+  isInvokeChildWorkflowTimelineEntry,
   runMigrations,
   updateWorkflowRun,
+  validateResourceId,
+  validateWorkflowId,
+  waitForTimelineKey,
   withPostgresTransaction,
   workflow
-} from "./shared/chunk-2xy8z3xp.js";
+} from "./shared/chunk-nygamc7b.js";
 // src/duration.ts
 import parse from "parse-duration";
 var MS_PER_SECOND = 1000;
@@ -91,7 +97,7 @@ function parseWorkflowHandler(handler) {
       const propertyAccess = node.expression;
       const objectName = propertyAccess.expression.getText(sourceFile);
       const methodName = propertyAccess.name.text;
-      if (objectName === "step" && (methodName === "run" || methodName === "waitFor" || methodName === "pause" || methodName === "waitUntil" || methodName === "delay" || methodName === "sleep" || methodName === "poll")) {
+      if (objectName === "step" && (methodName === "run" || methodName === "waitFor" || methodName === "pause" || methodName === "waitUntil" || methodName === "delay" || methodName === "sleep" || methodName === "poll" || methodName === "invokeChildWorkflow")) {
         const firstArg = node.arguments[0];
         if (firstArg) {
           const { id, isDynamic } = extractStepId(firstArg);
@@ -124,13 +130,21 @@ var StepTypeToIcon = {
   ["pause" /* PAUSE */]: "⏸",
   ["waitUntil" /* WAIT_UNTIL */]: "⏲",
   ["delay" /* DELAY */]: "⏱",
-  ["poll" /* POLL */]: "↻"
+  ["poll" /* POLL */]: "↻",
+  ["invokeChildWorkflow" /* INVOKE_CHILD_WORKFLOW */]: "↪"
 };
 var defaultLogger = {
   log: (_message) => console.warn(_message),
   error: (message, error) => console.error(message, error)
 };
 var defaultExpireInSeconds = process.env.WORKFLOW_RUN_EXPIRE_IN_SECONDS ? Number.parseInt(process.env.WORKFLOW_RUN_EXPIRE_IN_SECONDS, 10) : 5 * 60;
+var retrySendOptions = (maxRetries) => ({
+  retryLimit: maxRetries,
+  retryBackoff: true,
+  retryDelay: 1
+});
+var getInvokeChildWorkflowEventName = (childRunId) => `__invoke_child_workflow_completed:${childRunId}`;
+var defaultHeartbeatSeconds = process.env.WORKFLOW_RUN_HEARTBEAT_SECONDS ? Number.parseInt(process.env.WORKFLOW_RUN_HEARTBEAT_SECONDS, 10) : 30;
 
 class WorkflowEngine {
   boss;
@@ -164,7 +178,10 @@ class WorkflowEngine {
     }
     this.db = this.boss.getDb();
   }
-  async start(asEngine = true, { batchSize } = { batchSize: 1 }) {
+  async start(asEngine = true, {
+    batchSize = 1,
+    heartbeatSeconds = defaultHeartbeatSeconds
+  } = {}) {
     if (this._started) {
       return;
     }
@@ -175,13 +192,21 @@ class WorkflowEngine {
         await this.registerWorkflow(workflow2);
       }
     }
-    await this.boss.createQueue(WORKFLOW_RUN_QUEUE_NAME);
+    const mainQueueOptions = {
+      retryLimit: 0,
+      deadLetter: WORKFLOW_RUN_DLQ_QUEUE_NAME,
+      heartbeatSeconds
+    };
+    await this.boss.createQueue(WORKFLOW_RUN_DLQ_QUEUE_NAME, { retryLimit: 0 });
+    await this.boss.createQueue(WORKFLOW_RUN_QUEUE_NAME, mainQueueOptions);
+    await this.boss.updateQueue(WORKFLOW_RUN_QUEUE_NAME, mainQueueOptions);
     const numWorkers = +(process.env.WORKFLOW_RUN_WORKERS ?? 3);
     if (asEngine) {
-      for (let i = 0;i < numWorkers; i++) {
-        await this.boss.work(WORKFLOW_RUN_QUEUE_NAME, { pollingIntervalSeconds: 0.5, batchSize }, (job) => this.handleWorkflowRun(job));
+      await Promise.all(Array.from({ length: numWorkers }, (_, i) => this.boss.work(WORKFLOW_RUN_QUEUE_NAME, { pollingIntervalSeconds: 0.5, batchSize, includeMetadata: true }, (jobs) => this.handleWorkflowRun(jobs)).then(() => {
         this.logger.log(`Worker ${i + 1}/${numWorkers} started for queue ${WORKFLOW_RUN_QUEUE_NAME}`);
-      }
+      })));
+      await this.boss.work(WORKFLOW_RUN_DLQ_QUEUE_NAME, { pollingIntervalSeconds: 0.5, batchSize: 1 }, (jobs) => this.handleWorkflowRunDlq(jobs));
+      this.logger.log(`Worker started for queue ${WORKFLOW_RUN_DLQ_QUEUE_NAME}`);
     }
     this._started = true;
     this.logger.log("Workflow engine started!");
@@ -224,29 +249,57 @@ class WorkflowEngine {
     this.workflows.clear();
     return this;
   }
-  async startWorkflow(refOrParams, inputArg, optionsArg) {
-    let workflowId;
-    let input;
-    let resourceId;
-    let idempotencyKey;
-    let options;
+  resolveWorkflowRunParameters(refOrParams, inputArg, optionsArg) {
     if (typeof refOrParams === "function" && "id" in refOrParams) {
-      workflowId = refOrParams.id;
-      input = inputArg;
-      options = optionsArg;
-      resourceId = optionsArg?.resourceId;
-      idempotencyKey = optionsArg?.idempotencyKey;
-    } else {
-      const params = refOrParams;
-      workflowId = params.workflowId;
-      input = params.input;
-      resourceId = params.resourceId;
-      idempotencyKey = params.idempotencyKey;
-      options = params.options;
+      return {
+        workflowId: refOrParams.id,
+        input: inputArg,
+        options: optionsArg,
+        resourceId: optionsArg?.resourceId,
+        idempotencyKey: optionsArg?.idempotencyKey
+      };
     }
+    const params = refOrParams;
+    return {
+      workflowId: params.workflowId,
+      input: params.input,
+      resourceId: params.resourceId ?? params.options?.resourceId,
+      idempotencyKey: params.idempotencyKey ?? params.options?.idempotencyKey,
+      options: params.options
+    };
+  }
+  async startWorkflow(refOrParams, inputArg, optionsArg) {
+    const { workflowId, input, resourceId, idempotencyKey, options } = this.resolveWorkflowRunParameters(refOrParams, inputArg, optionsArg);
     if (!this._started) {
       await this.start(false, { batchSize: options?.batchSize ?? 1 });
     }
+    const { run } = await this.createWorkflowRun({
+      workflowId,
+      input,
+      resourceId,
+      idempotencyKey,
+      options
+    });
+    this.logger.log("Started workflow run", {
+      runId: run.id,
+      workflowId
+    });
+    return run;
+  }
+  async createWorkflowRun({
+    workflowId,
+    input,
+    resourceId,
+    idempotencyKey,
+    options,
+    parentRunId,
+    parentStepId,
+    parentResourceId,
+    enqueue = true,
+    db
+  }) {
+    validateWorkflowId(workflowId);
+    validateResourceId(resourceId);
     const workflow2 = this.workflows.get(workflowId);
     if (!workflow2) {
       throw new WorkflowEngineError(`Unknown workflow ${workflowId}`);
@@ -263,37 +316,60 @@ class WorkflowEngine {
       }
     }
     const initialStepId = workflow2.steps[0]?.id ?? "__start__";
-    const run = await withPostgresTransaction(this.boss.getDb(), async (_db) => {
-      const timeoutAt = options?.timeout ? new Date(Date.now() + options.timeout) : workflow2.timeout ? new Date(Date.now() + workflow2.timeout) : null;
-      const { run: insertedRun, created } = await insertWorkflowRun({
-        resourceId,
-        workflowId,
-        currentStepId: initialStepId,
-        status: "running" /* RUNNING */,
-        input,
-        maxRetries: options?.retries ?? workflow2.retries ?? 0,
-        timeoutAt,
-        idempotencyKey
-      }, _db);
-      if (created) {
-        const job = {
-          runId: insertedRun.id,
-          resourceId,
-          workflowId,
-          input
-        };
-        await this.boss.send(WORKFLOW_RUN_QUEUE_NAME, job, {
-          startAfter: new Date,
-          expireInSeconds: options?.expireInSeconds ?? defaultExpireInSeconds
-        });
+    const timeoutAt = options?.timeout ? new Date(Date.now() + options.timeout) : workflow2.timeout ? new Date(Date.now() + workflow2.timeout) : null;
+    const insertRun = async (targetDb) => await insertWorkflowRun({
+      resourceId,
+      workflowId,
+      currentStepId: initialStepId,
+      status: "running" /* RUNNING */,
+      input,
+      maxRetries: options?.retries ?? workflow2.retries ?? 0,
+      timeoutAt,
+      idempotencyKey,
+      parentRunId,
+      parentStepId,
+      parentResourceId
+    }, targetDb);
+    const insertAndEnqueue = async (targetDb) => {
+      const result = await insertRun(targetDb);
+      if (enqueue && result.created) {
+        await this.enqueueWorkflowRun(result.run, options, targetDb);
       }
-      return insertedRun;
-    }, this.pool);
-    this.logger.log("Started workflow run", {
+      return result;
+    };
+    const { run, created } = db ? await insertAndEnqueue(db) : await withPostgresTransaction(this.boss.getDb(), insertAndEnqueue, this.pool);
+    return { run, created };
+  }
+  async enqueueWorkflowRun(run, options, db) {
+    const job = {
       runId: run.id,
-      workflowId
+      resourceId: run.resourceId ?? undefined,
+      workflowId: run.workflowId,
+      input: run.input
+    };
+    await this.boss.send(WORKFLOW_RUN_QUEUE_NAME, job, {
+      startAfter: new Date,
+      expireInSeconds: options?.expireInSeconds ?? defaultExpireInSeconds,
+      ...retrySendOptions(run.maxRetries),
+      ...db ? { db } : {}
+    });
+  }
+  async notifyParentOfChildTerminalRun(childRun) {
+    if (!childRun.parentRunId || !childRun.parentStepId) {
+      return;
+    }
+    const parentRun = await getWorkflowRun({
+      runId: childRun.parentRunId,
+      resourceId: childRun.parentResourceId ?? undefined
+    }, { db: this.db });
+    if (!parentRun || parentRun.status === "completed" /* COMPLETED */ || parentRun.status === "failed" /* FAILED */ || parentRun.status === "cancelled" /* CANCELLED */) {
+      return;
+    }
+    await this.triggerEvent({
+      runId: parentRun.id,
+      resourceId: parentRun.resourceId ?? undefined,
+      eventName: getInvokeChildWorkflowEventName(childRun.id)
     });
-    return run;
   }
   async pauseWorkflow({
     runId,
@@ -325,6 +401,9 @@ class WorkflowEngine {
     if (current.status !== "paused" /* PAUSED */) {
       throw new WorkflowEngineError(`Cannot resume workflow run in '${current.status}' status, must be 'paused'`, current.workflowId, runId);
     }
+    if (this.getInvokeChildWorkflowStepEntry(current.timeline, current.currentStepId)) {
+      return current;
+    }
     return this.triggerEvent({
       runId,
       resourceId,
@@ -344,6 +423,9 @@ class WorkflowEngine {
       return run;
     }
     const stepId = run.currentStepId;
+    if (this.getInvokeChildWorkflowStepEntry(run.timeline, stepId)) {
+      return run;
+    }
     const waitForStep = this.getWaitForStepEntry(run.timeline, stepId);
     if (!waitForStep) {
       return run;
@@ -392,6 +474,7 @@ class WorkflowEngine {
       expectedStatuses: ["pending" /* PENDING */, "running" /* RUNNING */, "paused" /* PAUSED */]
     });
     this.logger.log(`cancelled workflow run with id ${runId}`);
+    await this.notifyParentOfChildTerminalRun(run);
     return run;
   }
   async triggerEvent({
@@ -415,7 +498,8 @@ class WorkflowEngine {
       }
     };
     await this.boss.send(WORKFLOW_RUN_QUEUE_NAME, job, {
-      expireInSeconds: options?.expireInSeconds ?? defaultExpireInSeconds
+      expireInSeconds: options?.expireInSeconds ?? defaultExpireInSeconds,
+      ...retrySendOptions(run.maxRetries)
     });
     this.logger.log(`event ${eventName} sent for workflow run with id ${runId}`);
     return run;
@@ -494,7 +578,7 @@ class WorkflowEngine {
     return run.resourceId ?? undefined;
   }
   async handleWorkflowRun([job]) {
-    const { runId = "", resourceId, workflowId = "", input, event } = job?.data ?? {};
+    const { runId = "", resourceId, workflowId = "", event } = job?.data ?? {};
     let run;
     let scopedResourceId;
     try {
@@ -517,6 +601,14 @@ class WorkflowEngine {
         throw new WorkflowEngineError(`Workflow run ${runId} does not match job workflowId ${workflowId}`, workflowId, runId);
       }
       scopedResourceId = this.resolveScopedResourceId(resourceId, run);
+      if (job?.retryCount !== undefined && run.retryCount !== job.retryCount) {
+        await this.updateRun({
+          runId,
+          resourceId: scopedResourceId,
+          data: { retryCount: job.retryCount }
+        });
+        run = { ...run, retryCount: job.retryCount };
+      }
       if (run.status === "cancelled" /* CANCELLED */) {
         this.logger.log(`Workflow run ${runId} is cancelled, skipping`);
         return;
@@ -620,6 +712,22 @@ class WorkflowEngine {
           }
           const timeoutMs = options?.timeout ? parseDuration(options.timeout) : undefined;
           return this.pollStep({ run, stepId, conditionFn, intervalMs, timeoutMs });
+        },
+        invokeChildWorkflow: async (stepId, refOrParams, inputArg, optionsArg) => {
+          if (!run) {
+            throw new WorkflowEngineError("Missing workflow run", workflowId, runId);
+          }
+          const resolvedChildCall = this.resolveWorkflowRunParameters(refOrParams, inputArg, optionsArg);
+          const childWorkflowInvocation = {
+            run,
+            stepId,
+            workflowId: resolvedChildCall.workflowId,
+            input: resolvedChildCall.input,
+            options: resolvedChildCall.options,
+            resourceId: resolvedChildCall.resourceId,
+            idempotencyKey: resolvedChildCall.idempotencyKey
+          };
+          return this.invokeChildWorkflowStep(childWorkflowInvocation);
         }
       };
       let step = { ...baseStep };
@@ -644,7 +752,7 @@ class WorkflowEngine {
       const shouldComplete = run.status === "running" /* RUNNING */ && (noParsedSteps || isLastParsedStep || hasPluginSteps && result !== undefined);
       if (shouldComplete) {
         const normalizedResult = result === undefined ? {} : result;
-        await this.updateRun({
+        const completedRun = await this.updateRun({
           runId,
           resourceId: scopedResourceId,
           data: {
@@ -654,56 +762,244 @@ class WorkflowEngine {
             jobId: job?.id
           }
         });
+        await this.notifyParentOfChildTerminalRun(completedRun);
         this.logger.log("Workflow run completed.", {
           runId,
           workflowId
         });
       }
     } catch (error) {
-      if (run && run.retryCount < run.maxRetries) {
-        await this.updateRun({
-          runId,
-          resourceId: scopedResourceId,
-          data: {
-            retryCount: run.retryCount + 1,
-            jobId: job?.id
-          }
-        });
-        const retryDelay = 2 ** run.retryCount * 1000;
-        const pgBossJob = {
-          runId,
-          resourceId: scopedResourceId,
-          workflowId,
-          input
-        };
-        await this.boss?.send("workflow-run", pgBossJob, {
-          startAfter: new Date(Date.now() + retryDelay),
-          expireInSeconds: defaultExpireInSeconds
-        });
-        return;
-      }
       if (runId) {
-        await this.updateRun({
+        const updatedRun = await this.updateRun({
           runId,
           resourceId: scopedResourceId,
           data: {
-            status: "failed" /* FAILED */,
             error: error instanceof Error ? error.message : String(error),
             jobId: job?.id
           }
         });
+        if (updatedRun.status === "completed" /* COMPLETED */ || updatedRun.status === "failed" /* FAILED */ || updatedRun.status === "cancelled" /* CANCELLED */) {
+          await this.notifyParentOfChildTerminalRun(updatedRun);
+        }
       }
       throw error;
     }
   }
+  async handleWorkflowRunDlq([job]) {
+    const { runId } = job?.data ?? {};
+    if (!runId)
+      return;
+    const run = await getWorkflowRun({ runId }, { db: this.db });
+    if (!run || run.status !== "running" /* RUNNING */)
+      return;
+    const failedRun = await this.updateRun({
+      runId,
+      resourceId: run.resourceId ?? undefined,
+      data: {
+        status: "failed" /* FAILED */,
+        error: run.error ?? "Workflow run worker died or job expired before completion"
+      }
+    });
+    await this.notifyParentOfChildTerminalRun(failedRun);
+    this.logger.log("Marked stuck workflow run as failed", {
+      runId,
+      workflowId: run.workflowId
+    });
+  }
   getCachedStepEntry(timeline, stepId) {
     const stepEntry = timeline[stepId];
     return stepEntry && typeof stepEntry === "object" && "output" in stepEntry ? stepEntry : null;
   }
   getWaitForStepEntry(timeline, stepId) {
-    const entry = timeline[`${stepId}-wait-for`];
+    const entry = timeline[waitForTimelineKey(stepId)];
     return entry && typeof entry === "object" && "waitFor" in entry ? entry : null;
   }
+  getInvokeChildWorkflowStepEntry(timeline, stepId) {
+    const entry = timeline[invokeChildWorkflowTimelineKey(stepId)];
+    return isInvokeChildWorkflowTimelineEntry(entry) ? entry : null;
+  }
+  getCompletedChildOutput(childRun) {
+    return childRun.output === undefined ? {} : childRun.output;
+  }
+  throwForNonCompletedChild(childRun) {
+    throw new WorkflowEngineError(`Child workflow ${childRun.workflowId} ${childRun.status}${childRun.error ? `: ${childRun.error}` : ""}`, childRun.workflowId, childRun.id);
+  }
+  assertInvokeChildWorkflowStepOwnership({
+    childRun,
+    parentRun,
+    stepId,
+    workflowId
+  }) {
+    const expectedParentResourceId = parentRun.resourceId ?? null;
+    const matches = childRun.workflowId === workflowId && childRun.parentRunId === parentRun.id && childRun.parentStepId === stepId && childRun.parentResourceId === expectedParentResourceId;
+    if (!matches) {
+      throw new WorkflowEngineError(`Idempotency key resolved to workflow run ${childRun.id}, which does not belong to invokeChildWorkflow step '${stepId}'`, workflowId, parentRun.id);
+    }
+  }
+  async invokeChildWorkflowStep({
+    run,
+    stepId,
+    workflowId,
+    input,
+    resourceId,
+    idempotencyKey,
+    options
+  }) {
+    let invokeOutput;
+    let hasInvokeOutput = false;
+    const childResourceId = resourceId ?? run.resourceId ?? undefined;
+    const childIdempotencyKey = idempotencyKey;
+    await withPostgresTransaction(this.db, async (db) => {
+      const lockedRun = await this.getRun({ runId: run.id, resourceId: run.resourceId ?? undefined }, { exclusiveLock: true, db });
+      if (lockedRun.status === "cancelled" /* CANCELLED */ || lockedRun.status === "paused" /* PAUSED */ || lockedRun.status === "failed" /* FAILED */) {
+        return;
+      }
+      const lockedCached = this.getCachedStepEntry(lockedRun.timeline, stepId);
+      if (lockedCached?.output !== undefined) {
+        invokeOutput = lockedCached.output;
+        hasInvokeOutput = true;
+        return;
+      }
+      const lockedInvoke = this.getInvokeChildWorkflowStepEntry(lockedRun.timeline, stepId);
+      if (lockedInvoke) {
+        const existingChildResourceId = "childResourceId" in lockedInvoke.invokeChildWorkflow ? lockedInvoke.invokeChildWorkflow.childResourceId ?? undefined : childResourceId;
+        const existingChildRun = await this.getRun({
+          runId: lockedInvoke.invokeChildWorkflow.childRunId,
+          resourceId: existingChildResourceId
+        });
+        if (existingChildRun.status === "completed" /* COMPLETED */) {
+          invokeOutput = this.getCompletedChildOutput(existingChildRun);
+          hasInvokeOutput = true;
+          await this.updateRun({
+            runId: run.id,
+            resourceId: run.resourceId ?? undefined,
+            data: {
+              timeline: merge(lockedRun.timeline, {
+                [stepId]: {
+                  output: invokeOutput,
+                  timestamp: new Date
+                }
+              })
+            }
+          }, { db });
+          return;
+        }
+        if (existingChildRun.status === "failed" /* FAILED */ || existingChildRun.status === "cancelled" /* CANCELLED */) {
+          this.throwForNonCompletedChild(existingChildRun);
+        }
+        await this.pauseRunForWait({
+          run: lockedRun,
+          stepId,
+          eventName: getInvokeChildWorkflowEventName(existingChildRun.id),
+          skipOutput: true,
+          db
+        });
+        return;
+      }
+      const result = await this.createWorkflowRun({
+        workflowId,
+        input,
+        resourceId: childResourceId,
+        idempotencyKey: childIdempotencyKey,
+        options,
+        parentRunId: run.id,
+        parentStepId: stepId,
+        parentResourceId: run.resourceId ?? undefined,
+        enqueue: true,
+        db
+      });
+      const childRun = result.run;
+      if (!result.created) {
+        this.assertInvokeChildWorkflowStepOwnership({
+          childRun,
+          parentRun: lockedRun,
+          stepId,
+          workflowId
+        });
+        if (childRun.status === "completed" /* COMPLETED */) {
+          invokeOutput = this.getCompletedChildOutput(childRun);
+          hasInvokeOutput = true;
+          await this.updateRun({
+            runId: run.id,
+            resourceId: run.resourceId ?? undefined,
+            data: {
+              timeline: merge(lockedRun.timeline, {
+                [invokeChildWorkflowTimelineKey(stepId)]: {
+                  invokeChildWorkflow: {
+                    childRunId: childRun.id,
+                    childWorkflowId: childRun.workflowId,
+                    childResourceId: childRun.resourceId
+                  },
+                  timestamp: new Date
+                },
+                [stepId]: {
+                  output: invokeOutput,
+                  timestamp: new Date
+                }
+              })
+            }
+          }, { db });
+          return;
+        }
+        if (childRun.status === "failed" /* FAILED */ || childRun.status === "cancelled" /* CANCELLED */) {
+          this.throwForNonCompletedChild(childRun);
+        }
+      }
+      await this.pauseRunForWait({
+        run: lockedRun,
+        stepId,
+        eventName: getInvokeChildWorkflowEventName(childRun.id),
+        skipOutput: true,
+        db,
+        timeline: merge(lockedRun.timeline, {
+          [invokeChildWorkflowTimelineKey(stepId)]: {
+            invokeChildWorkflow: {
+              childRunId: childRun.id,
+              childWorkflowId: childRun.workflowId,
+              childResourceId: childRun.resourceId
+            },
+            timestamp: new Date
+          }
+        })
+      });
+    }, this.pool);
+    if (hasInvokeOutput) {
+      return invokeOutput;
+    }
+  }
+  async pauseRunForWait({
+    run,
+    stepId,
+    eventName,
+    timeoutEvent,
+    skipOutput,
+    db,
+    timeline
+  }) {
+    const baseTimeline = timeline ?? run.timeline;
+    const waitFor = {};
+    if (eventName)
+      waitFor.eventName = eventName;
+    if (timeoutEvent)
+      waitFor.timeoutEvent = timeoutEvent;
+    if (skipOutput)
+      waitFor.skipOutput = true;
+    await this.updateRun({
+      runId: run.id,
+      resourceId: run.resourceId ?? undefined,
+      data: {
+        status: "paused" /* PAUSED */,
+        currentStepId: stepId,
+        pausedAt: new Date,
+        timeline: merge(baseTimeline, {
+          [waitForTimelineKey(stepId)]: {
+            waitFor,
+            timestamp: new Date
+          }
+        })
+      }
+    }, { db });
+  }
   async runStep({
     stepId,
     run,
@@ -792,21 +1088,7 @@ ${error.stack}` : String(error)
     const timeoutEvent = timeoutDate ? `__timeout_${stepId}` : undefined;
     await withPostgresTransaction(this.db, async (db) => {
       const freshRun = await this.getRun({ runId: run.id, resourceId: run.resourceId ?? undefined }, { exclusiveLock: true, db });
-      return this.updateRun({
-        runId: run.id,
-        resourceId: run.resourceId ?? undefined,
-        data: {
-          status: "paused" /* PAUSED */,
-          currentStepId: stepId,
-          pausedAt: new Date,
-          timeline: merge(freshRun.timeline, {
-            [`${stepId}-wait-for`]: {
-              waitFor: { eventName, timeoutEvent },
-              timestamp: new Date
-            }
-          })
-        }
-      }, { db });
+      return this.pauseRunForWait({ run: freshRun, stepId, eventName, timeoutEvent, db });
     }, this.pool);
     if (timeoutDate && timeoutEvent) {
       try {
@@ -819,7 +1101,8 @@ ${error.stack}` : String(error)
         };
         await this.boss.send(WORKFLOW_RUN_QUEUE_NAME, job, {
           startAfter: timeoutDate.getTime() <= Date.now() ? new Date : timeoutDate,
-          expireInSeconds: defaultExpireInSeconds
+          expireInSeconds: defaultExpireInSeconds,
+          ...retrySendOptions(run.maxRetries)
         });
       } catch (error) {
         await this.updateRun({
@@ -919,7 +1202,7 @@ ${error.stack}` : String(error)
           pausedAt: new Date,
           timeline: merge(freshRun.timeline, {
             [`${stepId}-poll`]: { startedAt: startedAt.toISOString() },
-            [`${stepId}-wait-for`]: {
+            [waitForTimelineKey(stepId)]: {
               waitFor: { timeoutEvent: pollEvent, skipOutput: true },
               timestamp: new Date
             }
@@ -936,7 +1219,8 @@ ${error.stack}` : String(error)
         event: { name: pollEvent, data: {} }
       }, {
         startAfter: new Date(Date.now() + intervalMs),
-        expireInSeconds: defaultExpireInSeconds
+        expireInSeconds: defaultExpireInSeconds,
+        ...retrySendOptions(run.maxRetries)
       });
     } catch (error) {
       await this.updateRun({
@@ -979,6 +1263,9 @@ ${error.stack}` : String(error)
     statuses,
     workflowId
   }) {
+    if (workflowId)
+      validateWorkflowId(workflowId);
+    validateResourceId(resourceId);
     return getWorkflowRuns({
       resourceId,
       startingAfter,
@@ -991,6 +1278,8 @@ ${error.stack}` : String(error)
 }
 export {
   workflow,
+  validateWorkflowId,
+  validateResourceId,
   parseDuration,
   createWorkflowRef,
   WorkflowStatus,
@@ -1001,5 +1290,5 @@ export {
   StepType
 };
 
-//# debugId=9A3E4733F40A491264756E2164756E21
+//# debugId=F3273B34BE68C60E64756E2164756E21
 //# sourceMappingURL=index.js.map