trigger.dev 3.0.0-beta.44 → 3.0.0-beta.46

package/dist/workers/prod/entry-point.js

@@ -3,7 +3,8 @@ import {
   CoordinatorToProdWorkerMessages,
   PostStartCauses,
   PreStopCauses,
-  ProdWorkerToCoordinatorMessages
+  ProdWorkerToCoordinatorMessages,
+  TaskRunErrorCodes as TaskRunErrorCodes2
 } from "@trigger.dev/core/v3";
 import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 
@@ -65,9 +66,280 @@ var SimpleLogger = class {
   }
 };
 
-// src/workers/prod/entry-point.ts
-import { readFile } from "node:fs/promises";
-import { createServer } from "node:http";
+// ../core-apps/src/process.ts
+var EXIT_CODE_ALREADY_HANDLED = 111;
+var EXIT_CODE_CHILD_NONZERO = 112;
+
+// ../core-apps/src/backoff.ts
+import { setTimeout as timeout } from "node:timers/promises";
+var StopRetrying = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "StopRetrying";
+  }
+};
+var AttemptTimeout = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "AttemptTimeout";
+  }
+};
+var RetryLimitExceeded = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RetryLimitExceeded";
+  }
+};
+var ExponentialBackoff = class _ExponentialBackoff {
+  #retries = 0;
+  #type;
+  #base;
+  #factor;
+  #min;
+  #max;
+  #maxRetries;
+  #maxElapsed;
+  constructor(type, opts = {}) {
+    this.#type = type ?? "NoJitter";
+    this.#base = opts.base ?? 2;
+    this.#factor = opts.factor ?? 1;
+    this.#min = opts.min ?? -Infinity;
+    this.#max = opts.max ?? Infinity;
+    this.#maxRetries = opts.maxRetries ?? Infinity;
+    this.#maxElapsed = opts.maxElapsed ?? Infinity;
+  }
+  #clone(type, opts = {}) {
+    return new _ExponentialBackoff(type ?? this.#type, {
+      base: opts.base ?? this.#base,
+      factor: opts.factor ?? this.#factor,
+      min: opts.min ?? this.#min,
+      max: opts.max ?? this.#max,
+      maxRetries: opts.maxRetries ?? this.#maxRetries,
+      maxElapsed: opts.maxElapsed ?? this.#maxElapsed
+    });
+  }
+  type(type) {
+    return this.#clone(type);
+  }
+  base(base) {
+    return this.#clone(void 0, { base });
+  }
+  factor(factor) {
+    return this.#clone(void 0, { factor });
+  }
+  min(min) {
+    return this.#clone(void 0, { min });
+  }
+  max(max) {
+    return this.#clone(void 0, { max });
+  }
+  maxRetries(maxRetries) {
+    return this.#clone(void 0, { maxRetries });
+  }
+  // TODO: With .execute(), should this also include the time it takes to execute the callback?
+  maxElapsed(maxElapsed) {
+    return this.#clone(void 0, { maxElapsed });
+  }
+  retries(retries) {
+    if (typeof retries !== "undefined") {
+      if (retries > this.#maxRetries) {
+        console.error(
+          `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
+        );
+        this.#retries = this.#maxRetries;
+      } else {
+        this.#retries = retries;
+      }
+    }
+    return this.#clone();
+  }
+  async *retryAsync(maxRetries = this.#maxRetries ?? Infinity) {
+    let elapsed = 0;
+    let retry = 0;
+    while (retry <= maxRetries) {
+      const delay = this.delay(retry);
+      elapsed += delay;
+      if (elapsed > this.#maxElapsed) {
+        break;
+      }
+      yield {
+        delay: {
+          seconds: delay,
+          milliseconds: delay * 1e3
+        },
+        retry
+      };
+      retry++;
+    }
+  }
+  async *[Symbol.asyncIterator]() {
+    yield* this.retryAsync();
+  }
+  /** Returns the delay for the current retry in seconds. */
+  delay(retries = this.#retries, jitter = true) {
+    if (retries > this.#maxRetries) {
+      console.error(
+        `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
+      );
+      retries = this.#maxRetries;
+    }
+    let delay = this.#factor * this.#base ** retries;
+    switch (this.#type) {
+      case "NoJitter": {
+        break;
+      }
+      case "FullJitter": {
+        if (!jitter) {
+          delay = 0;
+          break;
+        }
+        delay *= Math.random();
+        break;
+      }
+      case "EqualJitter": {
+        if (!jitter) {
+          delay *= 0.5;
+          break;
+        }
+        delay *= 0.5 * (1 + Math.random());
+        break;
+      }
+      default: {
+        throw new Error(`Unknown backoff type: ${this.#type}`);
+      }
+    }
+    if (delay < this.#min) {
+      delay = this.#min + Math.random() * (this.#min * 0.2);
+    }
+    if (delay > this.#max) {
+      delay = this.#max - Math.random() * (this.#max * 0.2);
+    }
+    delay = Math.round(delay);
+    return delay;
+  }
+  /** Waits with the appropriate delay for the current retry. */
+  async wait(retries = this.#retries, jitter = true) {
+    if (retries > this.#maxRetries) {
+      console.error(`Retry limit exceeded: ${retries} > ${this.#maxRetries}`);
+      throw new RetryLimitExceeded();
+    }
+    const delay = this.delay(retries, jitter);
+    return await timeout(delay * 1e3);
+  }
+  elapsed(retries = this.#retries, jitter = true) {
+    let elapsed = 0;
+    for (let i = 0; i <= retries; i++) {
+      elapsed += this.delay(i, jitter);
+    }
+    const total = elapsed;
+    let days = 0;
+    if (elapsed > 3600 * 24) {
+      days = Math.floor(elapsed / 3600 / 24);
+      elapsed -= days * 3600 * 24;
+    }
+    let hours = 0;
+    if (elapsed > 3600) {
+      hours = Math.floor(elapsed / 3600);
+      elapsed -= hours * 3600;
+    }
+    let minutes = 0;
+    if (elapsed > 60) {
+      minutes = Math.floor(elapsed / 60);
+      elapsed -= minutes * 60;
+    }
+    const seconds = elapsed;
+    return {
+      seconds,
+      minutes,
+      hours,
+      days,
+      total
+    };
+  }
+  reset() {
+    this.#retries = 0;
+    return this;
+  }
+  next() {
+    this.#retries++;
+    return this.delay();
+  }
+  stop() {
+    throw new StopRetrying();
+  }
+  get state() {
+    return {
+      retries: this.#retries,
+      type: this.#type,
+      base: this.#base,
+      factor: this.#factor,
+      min: this.#min,
+      max: this.#max,
+      maxRetries: this.#maxRetries,
+      maxElapsed: this.#maxElapsed
+    };
+  }
+  async execute(callback, { attemptTimeoutMs = 0 } = {}) {
+    let elapsedMs = 0;
+    let finalError = void 0;
+    for await (const { delay, retry } of this) {
+      const start = Date.now();
+      if (retry > 0) {
+        console.log(`Retrying in ${delay.milliseconds}ms`);
+        await timeout(delay.milliseconds);
+      }
+      let attemptTimeout = void 0;
+      try {
+        const result = await new Promise(async (resolve, reject) => {
+          if (attemptTimeoutMs > 0) {
+            attemptTimeout = setTimeout(() => {
+              reject(new AttemptTimeout());
+            }, attemptTimeoutMs);
+          }
+          try {
+            const callbackResult = await callback({ delay, retry, elapsedMs });
+            resolve(callbackResult);
+          } catch (error) {
+            reject(error);
+          }
+        });
+        return {
+          success: true,
+          result
+        };
+      } catch (error) {
+        finalError = error;
+        if (error instanceof StopRetrying) {
+          return {
+            success: false,
+            cause: "StopRetrying",
+            error: error.message
+          };
+        }
+        if (error instanceof AttemptTimeout) {
+          continue;
+        }
+      } finally {
+        elapsedMs += Date.now() - start;
+        clearTimeout(attemptTimeout);
+      }
+    }
+    if (finalError instanceof AttemptTimeout) {
+      return {
+        success: false,
+        cause: "Timeout"
+      };
+    } else {
+      return {
+        success: false,
+        cause: "MaxRetries",
+        error: finalError
+      };
+    }
+  }
+  static RetryLimitExceeded = RetryLimitExceeded;
+  static StopRetrying = StopRetrying;
+};
 
 // src/workers/prod/backgroundWorker.ts
 import {
@@ -99,9 +371,11 @@ var TaskMetadataParseError = class extends Error {
   }
 };
 var UnexpectedExitError = class extends Error {
-  constructor(code) {
+  constructor(code, signal, stderr) {
     super(`Unexpected exit with code ${code}`);
     this.code = code;
+    this.signal = signal;
+    this.stderr = stderr;
     this.name = "UnexpectedExitError";
   }
 };
@@ -129,6 +403,32 @@ var GracefulExitTimeoutError = class extends Error {
     this.name = "GracefulExitTimeoutError";
   }
 };
+function getFriendlyErrorMessage(code, signal, stderr, dockerMode = true) {
+  const message = (text) => {
+    if (signal) {
+      return `[${signal}] ${text}`;
+    } else {
+      return text;
+    }
+  };
+  if (code === 137) {
+    if (dockerMode) {
+      return message(
+        "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+      );
+    } else {
+      return message(
+        "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
+      );
+    }
+  }
+  if (stderr?.includes("OOMErrorHandler")) {
+    return message(
+      "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+    );
+  }
+  return message(`Process exited with code ${code}.`);
+}
 
 // src/workers/prod/backgroundWorker.ts
 var ProdBackgroundWorker = class {
@@ -142,13 +442,9 @@ var ProdBackgroundWorker = class {
    */
   onTaskHeartbeat = new Evt();
   onTaskRunHeartbeat = new Evt();
-  onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
-  preCheckpointNotification = Evt.create();
-  checkpointCanceledNotification = Evt.create();
-  onReadyForCheckpoint = Evt.create();
-  onCancelCheckpoint = Evt.create();
+  onWaitForBatch = new Evt();
   onCreateTaskRunAttempt = Evt.create();
   attemptCreatedNotification = Evt.create();
   _onClose = new Evt();
@@ -186,7 +482,10 @@ var ProdBackgroundWorker = class {
     this._closed = true;
   }
   async flushTelemetry() {
+    console.log("Flushing telemetry");
+    const start = performance.now();
     await this._taskRunProcess?.cleanup(false);
+    console.log("Flushed telemetry", { duration: performance.now() - start });
   }
   async initialize(options) {
     if (this._initialized) {
@@ -209,7 +508,7 @@ var ProdBackgroundWorker = class {
           ...options?.env
         }
       });
-      const timeout = setTimeout(() => {
+      const timeout3 = setTimeout(() => {
         if (resolved) {
           return;
         }
@@ -226,7 +525,7 @@ var ProdBackgroundWorker = class {
       });
       child.on("exit", (code) => {
         if (!resolved) {
-          clearTimeout(timeout);
+          clearTimeout(timeout3);
           resolved = true;
           reject(new Error(`Worker exited with code ${code}`));
         }
@@ -238,7 +537,7 @@ var ProdBackgroundWorker = class {
         handlers: {
           TASKS_READY: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               resolve(message.tasks);
               child.kill();
@@ -246,7 +545,7 @@ var ProdBackgroundWorker = class {
           },
           UNCAUGHT_EXCEPTION: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               reject(new UncaughtExceptionError(message.error, message.origin));
               child.kill();
@@ -254,7 +553,7 @@ var ProdBackgroundWorker = class {
           },
           TASKS_FAILED_TO_PARSE: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               reject(new TaskMetadataParseError(message.zodIssues, message.tasks));
               child.kill();
@@ -331,18 +630,6 @@ var ProdBackgroundWorker = class {
     taskRunProcess.onWaitForTask.attach((message) => {
       this.onWaitForTask.post(message);
     });
-    taskRunProcess.onReadyForCheckpoint.attach((message) => {
-      this.onReadyForCheckpoint.post(message);
-    });
-    taskRunProcess.onCancelCheckpoint.attach((message) => {
-      this.onCancelCheckpoint.post(message);
-    });
-    this.preCheckpointNotification.attach((message) => {
-      taskRunProcess.preCheckpointNotification.post(message);
-    });
-    this.checkpointCanceledNotification.attach((message) => {
-      taskRunProcess.checkpointCanceledNotification.post(message);
-    });
     await taskRunProcess.initialize();
     this._taskRunProcess = taskRunProcess;
     return this._taskRunProcess;
@@ -383,6 +670,7 @@ var ProdBackgroundWorker = class {
     }
   }
   async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
+    console.log("Trying graceful exit", { kill, initialSignal });
     try {
       const initialExit = taskRunProcess.onExit.waitFor(5e3);
       if (kill) {
@@ -395,6 +683,7 @@ var ProdBackgroundWorker = class {
     }
   }
   async #tryForcefulExit(taskRunProcess) {
+    console.log("Trying forceful exit");
     try {
       const forcedKill = taskRunProcess.onExit.waitFor(5e3);
       taskRunProcess.kill("SIGKILL");
@@ -455,7 +744,9 @@ var ProdBackgroundWorker = class {
           retry: void 0,
           error: {
             type: "INTERNAL_ERROR",
-            code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE
+            code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
+            message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
+            stackTrace: e.stderr
           }
         };
       }
@@ -506,16 +797,19 @@ var ProdBackgroundWorker = class {
     this.onCreateTaskRunAttempt.post({ runId: payload.runId });
     let execution;
     try {
-      const attemptCreated = await this.attemptCreatedNotification.waitFor(3e4);
+      const start = performance.now();
+      const attemptCreated = await this.attemptCreatedNotification.waitFor(12e4);
       if (!attemptCreated.success) {
-        throw new Error(
-          `Failed to create attempt${attemptCreated.reason ? `: ${attemptCreated.reason}` : ""}`
-        );
+        throw new Error(`${attemptCreated.reason ?? "Unknown error"}`);
       }
+      console.log("Attempt created", {
+        number: attemptCreated.execution.attempt.number,
+        duration: performance.now() - start
+      });
       execution = attemptCreated.execution;
     } catch (error) {
       console.error("Error while creating attempt", error);
-      throw new Error(`Failed to create task run attempt: ${error}`);
+      throw new Error(`Failed to create attempt: ${error}`);
     }
     const completion = await this.executeTaskRun(
       {
@@ -553,6 +847,7 @@ var TaskRunProcess = class {
   _isBeingKilled = false;
   _isBeingCancelled = false;
   _gracefulExitTimeoutElapsed = false;
+  _stderr = [];
   /**
    * @deprecated use onTaskRunHeartbeat instead
    */
@@ -560,13 +855,10 @@ var TaskRunProcess = class {
   onTaskRunHeartbeat = new Evt();
   onExit = new Evt();
   onIsBeingKilled = new Evt();
-  onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
+  onWaitForBatch = new Evt();
   preCheckpointNotification = Evt.create();
-  checkpointCanceledNotification = Evt.create();
-  onReadyForCheckpoint = Evt.create();
-  onCancelCheckpoint = Evt.create();
   async initialize() {
     this._child = fork(this.path, {
       stdio: [
@@ -614,6 +906,10 @@ var TaskRunProcess = class {
           if (this.messageId) {
             this.onTaskRunHeartbeat.post(this.messageId);
           } else {
+            console.error(
+              "No message id for task heartbeat, falling back to (deprecated) attempt heartbeat",
+              { id: message.id }
+            );
             this.onTaskHeartbeat.post(message.id);
           }
         },
@@ -627,41 +923,6 @@ var TaskRunProcess = class {
         },
         WAIT_FOR_DURATION: async (message) => {
           this.onWaitForDuration.post(message);
-          try {
-            const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor(
-              3e4
-            );
-            return {
-              willCheckpointAndRestore
-            };
-          } catch (error) {
-            console.error("Error while waiting for pre-checkpoint notification", error);
-            return {
-              willCheckpointAndRestore: false
-            };
-          }
-        },
-        READY_FOR_CHECKPOINT: async (message) => {
-          this.onReadyForCheckpoint.post(message);
-        },
-        CANCEL_CHECKPOINT: async (message) => {
-          const version = "v2";
-          this.onCancelCheckpoint.post(message);
-          try {
-            const { checkpointCanceled } = await this.checkpointCanceledNotification.waitFor(
-              3e4
-            );
-            return {
-              version,
-              checkpointCanceled
-            };
-          } catch (error) {
-            console.error("Error while waiting for checkpoint cancellation", error);
-            return {
-              version,
-              checkpointCanceled: true
-            };
-          }
         }
       }
     });
@@ -691,14 +952,21 @@ var TaskRunProcess = class {
       childPid: this._childPid,
       realChildPid: this._child?.pid
     });
-    await this._ipc?.sendWithAck(
-      "CLEANUP",
-      {
-        flush: true,
-        kill: killParentProcess
-      },
-      3e4
-    );
+    try {
+      await this._ipc?.sendWithAck(
+        "CLEANUP",
+        {
+          flush: true,
+          kill: killParentProcess
+        },
+        3e4
+      );
+    } catch (error) {
+      console.error("Error while cleaning up task run process", error);
+      if (killParentProcess) {
+        process.exit(0);
+      }
+    }
     if (killChildProcess) {
       this._gracefulExitTimeoutElapsed = true;
       await this.kill("SIGKILL");
@@ -728,19 +996,30 @@ var TaskRunProcess = class {
   }
   taskRunCompletedNotification(completion) {
     if (!completion.ok && typeof completion.retry !== "undefined") {
+      console.error(
+        "Task run completed with error and wants to retry, won't send task run completed notification"
+      );
       return;
     }
-    if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
-      this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
-        version: "v2",
-        completion
-      });
+    if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
+      console.error(
+        "Child process not connected or being killed, can't send task run completed notification"
+      );
+      return;
     }
+    this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
+      version: "v2",
+      completion
+    });
   }
   waitCompletedNotification() {
-    if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
-      this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
+    if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
+      console.error(
+        "Child process not connected or being killed, can't send wait completed notification"
+      );
+      return;
     }
+    this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
   }
   async #handleExit(code, signal) {
     console.log("handling child exit", { code, signal });
@@ -760,7 +1039,13 @@ var TaskRunProcess = class {
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code ?? -1));
+          rejecter(
+            new UnexpectedExitError(
+              code ?? -1,
+              signal,
+              this._stderr.length ? this._stderr.join("\n") : void 0
+            )
+          );
         }
       }
     }
@@ -770,7 +1055,12 @@ var TaskRunProcess = class {
     console.log(data.toString());
   }
   #handleStdErr(data) {
-    console.error(data.toString());
+    const text = data.toString();
+    console.error(text);
+    if (this._stderr.length > 100) {
+      this._stderr.shift();
+    }
+    this._stderr.push(text);
   }
   async kill(signal, timeoutInMs) {
     this._isBeingKilled = true;
@@ -790,7 +1080,11 @@ var TaskRunProcess = class {
 };
 
 // src/workers/prod/entry-point.ts
-import { setTimeout as setTimeout2 } from "node:timers/promises";
+import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers";
+import { randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { setTimeout as timeout2 } from "node:timers/promises";
 var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
 var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
 var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
@@ -798,6 +1092,9 @@ var MACHINE_NAME = process.env.MACHINE_NAME || "local";
 var POD_NAME = process.env.POD_NAME || "some-pod";
 var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
 var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
+var defaultBackoff = new ExponentialBackoff("FullJitter", {
+  maxRetries: 5
+});
 var ProdWorker = class {
   constructor(port, host = "0.0.0.0") {
     this.host = host;
@@ -822,6 +1119,12 @@ var ProdWorker = class {
   attemptFriendlyId;
   nextResumeAfter;
   waitForPostStart = false;
+  connectionCount = 0;
+  waitForTaskReplay;
+  waitForBatchReplay;
+  readyForLazyAttemptReplay;
+  submitAttemptCompletionReplay;
+  durationResumeFallback;
   #httpPort;
   #backgroundWorker;
   #httpServer;
@@ -835,7 +1138,7 @@ var ProdWorker = class {
         logger.log("Waiting for attempt to complete before exiting", {
           terminationGracePeriodSeconds
         });
-        await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
+        await timeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
         gracefulExitTimeoutElapsed = true;
         logger.log("Termination timeout reached, exiting gracefully.");
       } else {
@@ -846,20 +1149,16 @@ var ProdWorker = class {
     }
     logger.log("Unhandled signal", { signal });
   }
-  async #exitGracefully(gracefulExitTimeoutElapsed = false) {
+  async #exitGracefully(gracefulExitTimeoutElapsed = false, exitCode = 0) {
     await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
     if (!gracefulExitTimeoutElapsed) {
-      process.exit(0);
+      process.exit(exitCode);
     }
   }
-  async #reconnect(isPostStart = false, reconnectImmediately = false) {
-    if (isPostStart) {
-      this.waitForPostStart = false;
-    }
+  async #reconnectAfterPostStart() {
+    this.waitForPostStart = false;
     this.#coordinatorSocket.close();
-    if (!reconnectImmediately) {
-      await setTimeout2(1e3);
-    }
+    this.connectionCount = 0;
     let coordinatorHost = COORDINATOR_HOST;
     try {
       if (this.runningInKubernetes) {
@@ -883,6 +1182,98 @@ var ProdWorker = class {
       this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
     }
   }
+  // MARK: TASK WAIT
+  async #waitForTaskHandler(message, replayIdempotencyKey) {
+    const waitForTask = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Wait for task with backoff", { retry });
+      if (!this.attemptFriendlyId) {
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        throw new ExponentialBackoff.StopRetrying("No attempt ID");
+      }
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_TASK", {
+        version: "v2",
+        friendlyId: message.friendlyId,
+        attemptFriendlyId: this.attemptFriendlyId
+      });
+    });
+    if (!waitForTask.success) {
+      logger.error("Failed to wait for task with backoff", {
+        cause: waitForTask.cause,
+        error: waitForTask.error
+      });
+      this.#emitUnrecoverableError(
+        "WaitForTaskFailed",
+        `${waitForTask.cause}: ${waitForTask.error}`
+      );
+      return;
+    }
+    const { willCheckpointAndRestore } = waitForTask.result;
+    await this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
+    if (willCheckpointAndRestore) {
+      if (!this.waitForTaskReplay) {
+        this.waitForTaskReplay = {
+          message,
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
+          logger.error(
+            "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.waitForTaskReplay.attempt++;
+      }
+    }
+  }
+  // MARK: BATCH WAIT
+  async #waitForBatchHandler(message, replayIdempotencyKey) {
+    const waitForBatch = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Wait for batch with backoff", { retry });
+      if (!this.attemptFriendlyId) {
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        throw new ExponentialBackoff.StopRetrying("No attempt ID");
+      }
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_BATCH", {
+        version: "v2",
+        batchFriendlyId: message.batchFriendlyId,
+        runFriendlyIds: message.runFriendlyIds,
+        attemptFriendlyId: this.attemptFriendlyId
+      });
+    });
+    if (!waitForBatch.success) {
+      logger.error("Failed to wait for batch with backoff", {
+        cause: waitForBatch.cause,
+        error: waitForBatch.error
+      });
+      this.#emitUnrecoverableError(
+        "WaitForBatchFailed",
+        `${waitForBatch.cause}: ${waitForBatch.error}`
+      );
+      return;
+    }
+    const { willCheckpointAndRestore } = waitForBatch.result;
+    await this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
+    if (willCheckpointAndRestore) {
+      if (!this.waitForBatchReplay) {
+        this.waitForBatchReplay = {
+          message,
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
+          logger.error(
+            "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.waitForBatchReplay.attempt++;
+      }
+    }
+  }
+  // MARK: WORKER CREATION
   #createBackgroundWorker() {
     const backgroundWorker = new ProdBackgroundWorker("worker.js", {
       projectConfig: __PROJECT_CONFIG__,
@@ -895,148 +1286,187 @@ var ProdWorker = class {
       contentHash: this.contentHash
     });
     backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
-      this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
+      logger.log("onTaskHeartbeat", { attemptFriendlyId });
+      this.#coordinatorSocket.socket.volatile.emit("TASK_HEARTBEAT", {
+        version: "v1",
+        attemptFriendlyId
+      });
     });
     backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
-      this.#coordinatorSocket.socket.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
-    });
-    backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
-      await this.#prepareForCheckpoint();
-      this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
-    });
-    backgroundWorker.onCancelCheckpoint.attach(async (message) => {
-      logger.log("onCancelCheckpoint", { message });
-      const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
-        "CANCEL_CHECKPOINT",
-        {
-          version: "v2",
-          reason: message.reason
-        }
-      );
-      logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
-      if (checkpointCanceled) {
-        if (message.reason === "WAIT_FOR_DURATION") {
-          this.paused = false;
-          this.nextResumeAfter = void 0;
-          this.waitForPostStart = false;
-        }
-      }
-      backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
+      logger.log("onTaskRunHeartbeat", { runId });
+      this.#coordinatorSocket.socket.volatile.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
     });
     backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
       logger.log("onCreateTaskRunAttempt()", { message });
-      const createAttempt = await this.#coordinatorSocket.socket.emitWithAck(
-        "CREATE_TASK_RUN_ATTEMPT",
-        {
+      const createAttempt = await defaultBackoff.execute(async ({ retry }) => {
+        logger.log("Create task run attempt with backoff", { retry });
+        return await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CREATE_TASK_RUN_ATTEMPT", {
           version: "v1",
           runId: message.runId
-        }
-      );
+        });
+      });
       if (!createAttempt.success) {
         backgroundWorker.attemptCreatedNotification.post({
           success: false,
-          reason: createAttempt.reason
+          reason: `Failed to create attempt with backoff due to ${createAttempt.cause}. ${createAttempt.error}`
+        });
+        return;
+      }
+      if (!createAttempt.result.success) {
+        backgroundWorker.attemptCreatedNotification.post({
+          success: false,
+          reason: createAttempt.result.reason
         });
         return;
       }
       backgroundWorker.attemptCreatedNotification.post({
         success: true,
-        execution: createAttempt.executionPayload.execution
+        execution: createAttempt.result.executionPayload.execution
       });
     });
     backgroundWorker.attemptCreatedNotification.attach((message) => {
+      logger.log("attemptCreatedNotification", {
+        success: message.success,
+        ...message.success ? {
+          attempt: message.execution.attempt,
+          queue: message.execution.queue,
+          worker: message.execution.worker,
+          machine: message.execution.machine
+        } : {
+          reason: message.reason
+        }
+      });
       if (!message.success) {
         return;
       }
       this.attemptFriendlyId = message.execution.attempt.id;
     });
     backgroundWorker.onWaitForDuration.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
-        this.#emitUnrecoverableError(
-          "NoAttemptId",
-          "Attempt ID not set before waiting for duration"
-        );
-        return;
-      }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_DURATION",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
+      logger.log("onWaitForDuration", { ...message, drift: Date.now() - message.now });
+      noResume: {
+        const { ms, waitThresholdInMs } = message;
+        const internalTimeout = unboundedTimeout(ms, "internal");
+        const checkpointSafeInternalTimeout = checkpointSafeTimeout(ms);
+        if (ms < waitThresholdInMs) {
+          await internalTimeout;
+          break noResume;
         }
-      );
-      this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
-    });
-    backgroundWorker.onWaitForTask.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
-        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for task");
-        return;
-      }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_TASK",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
+        const waitForDuration = await defaultBackoff.execute(async ({ retry }) => {
+          logger.log("Wait for duration with backoff", { retry });
+          if (!this.attemptFriendlyId) {
+            logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+            throw new ExponentialBackoff.StopRetrying("No attempt ID");
+          }
+          return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_DURATION", {
+            ...message,
+            attemptFriendlyId: this.attemptFriendlyId
+          });
+        });
+        if (!waitForDuration.success) {
+          logger.error("Failed to wait for duration with backoff", {
+            cause: waitForDuration.cause,
+            error: waitForDuration.error
+          });
+          this.#emitUnrecoverableError(
+            "WaitForDurationFailed",
+            `${waitForDuration.cause}: ${waitForDuration.error}`
+          );
+          return;
+        }
+        const { willCheckpointAndRestore } = waitForDuration.result;
+        if (!willCheckpointAndRestore) {
+          await internalTimeout;
+          break noResume;
+        }
+        await this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
+        await Promise.race([internalTimeout, checkpointSafeInternalTimeout]);
+        try {
+          const { checkpointCanceled } = await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CANCEL_CHECKPOINT", {
+            version: "v2",
+            reason: "WAIT_FOR_DURATION"
+          });
+          logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
+          if (checkpointCanceled) {
+            break noResume;
+          }
+          logger.log("Waiting for external duration resume as we may have been restored");
+          const idempotencyKey = randomUUID();
+          this.durationResumeFallback = { idempotencyKey };
+          setTimeout(() => {
+            if (!this.durationResumeFallback) {
+              logger.error("Already resumed after duration, skipping fallback");
+              return;
+            }
+            if (this.durationResumeFallback.idempotencyKey !== idempotencyKey) {
+              logger.error("Duration resume idempotency key mismatch, skipping fallback");
+              return;
+            }
+            logger.log("Resuming after duration with fallback");
+            this.#resumeAfterDuration();
+          }, 15e3);
+        } catch (error) {
+          logger.debug("Checkpoint cancellation timed out", { error });
+          break noResume;
         }
-      );
-      this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
-    });
-    backgroundWorker.onWaitForBatch.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
-        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for batch");
         return;
       }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_BATCH",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
-        }
-      );
-      this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
+      this.#resumeAfterDuration();
     });
+    backgroundWorker.onWaitForTask.attach(this.#waitForTaskHandler.bind(this));
+    backgroundWorker.onWaitForBatch.attach(this.#waitForBatchHandler.bind(this));
     return backgroundWorker;
   }
   async #prepareForWait(reason, willCheckpointAndRestore) {
     logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
-    this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
-    if (willCheckpointAndRestore) {
-      this.paused = true;
-      this.nextResumeAfter = reason;
-      this.waitForPostStart = true;
-      if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
-        await this.#prepareForCheckpoint();
-      }
+    if (!willCheckpointAndRestore) {
+      return;
     }
+    this.paused = true;
+    this.nextResumeAfter = reason;
+    this.waitForPostStart = true;
+    await this.#prepareForCheckpoint();
   }
-  async #prepareForRetry(willCheckpointAndRestore, shouldExit) {
-    logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
+  // MARK: RETRY PREP
+  async #prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode) {
+    logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit, exitCode });
     if (shouldExit) {
       if (willCheckpointAndRestore) {
-        logger.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
+        logger.error("WARNING: Will checkpoint but also requested exit. This won't end well.");
       }
-      await this.#exitGracefully();
+      await this.#exitGracefully(false, exitCode);
       return;
     }
     this.paused = false;
     this.waitForPostStart = false;
     this.executing = false;
     this.attemptFriendlyId = void 0;
-    if (willCheckpointAndRestore) {
-      this.waitForPostStart = true;
-      this.#prepareForCheckpoint(false);
-      this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
+    if (!willCheckpointAndRestore) {
       return;
     }
+    this.waitForPostStart = true;
+    await this.#prepareForCheckpoint(false);
   }
+  // MARK: CHECKPOINT PREP
   async #prepareForCheckpoint(flush = true) {
     if (flush) {
-      await this.#backgroundWorker.flushTelemetry();
+      try {
+        await this.#backgroundWorker.flushTelemetry();
+      } catch (error) {
+        logger.error(
+          "Failed to flush telemetry while preparing for checkpoint, will proceed anyway",
+          { error }
+        );
+      }
+    }
+    try {
+      await this.#backgroundWorker.forceKillOldTaskRunProcesses();
+    } catch (error) {
+      logger.error(
+        "Failed to kill previous worker while preparing for checkpoint, will proceed anyway",
+        { error }
+      );
     }
-    await this.#backgroundWorker.forceKillOldTaskRunProcesses();
+    this.#readyForCheckpoint();
   }
   #resumeAfterDuration() {
     this.paused = false;
@@ -1044,6 +1474,106 @@ var ProdWorker = class {
     this.waitForPostStart = false;
     this.#backgroundWorker.waitCompletedNotification();
   }
+  async #readyForLazyAttempt() {
+    const idempotencyKey = randomUUID();
+    this.readyForLazyAttemptReplay = {
+      idempotencyKey
+    };
+    for await (const { delay, retry } of defaultBackoff.min(10).maxRetries(3)) {
+      if (retry > 0) {
+        logger.log("retrying ready for lazy attempt", { retry });
+      }
+      this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
+        version: "v1",
+        runId: this.runId,
+        totalCompletions: this.completed.size
+      });
+      await timeout2(delay.milliseconds);
+      if (!this.readyForLazyAttemptReplay) {
+        logger.error("replay ready for lazy attempt cancelled, discarding", {
+          idempotencyKey
+        });
+        return;
+      }
+      if (idempotencyKey !== this.readyForLazyAttemptReplay.idempotencyKey) {
+        logger.error("replay ready for lazy attempt idempotency key mismatch, discarding", {
+          idempotencyKey,
+          newIdempotencyKey: this.readyForLazyAttemptReplay.idempotencyKey
+        });
+        return;
+      }
+    }
+    this.#failRun(this.runId, "Failed to receive execute request in a reasonable time");
+  }
+  #readyForCheckpoint() {
+    this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
+  }
+  #failRun(anyRunId, error) {
+    logger.error("Failing run", { anyRunId, error });
+    const completion = {
+      ok: false,
+      id: anyRunId,
+      retry: void 0,
+      error: error instanceof Error ? {
+        type: "BUILT_IN_ERROR",
+        name: error.name,
+        message: error.message,
+        stackTrace: error.stack ?? ""
+      } : {
+        type: "BUILT_IN_ERROR",
+        name: "UnknownError",
+        message: String(error),
+        stackTrace: ""
+      }
+    };
+    this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
+      version: "v1",
+      completion
+    });
+  }
+  // MARK: ATTEMPT COMPLETION
+  async #submitAttemptCompletion(execution, completion, replayIdempotencyKey) {
+    const taskRunCompleted = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Submit attempt completion with backoff", { retry });
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("TASK_RUN_COMPLETED", {
+        version: "v1",
+        execution,
+        completion
+      });
+    });
+    if (!taskRunCompleted.success) {
+      logger.error("Failed to complete lazy attempt with backoff", {
+        cause: taskRunCompleted.cause,
+        error: taskRunCompleted.error
+      });
+      this.#failRun(execution.run.id, taskRunCompleted.error);
+      return;
+    }
+    const { willCheckpointAndRestore, shouldExit } = taskRunCompleted.result;
+    logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+    const exitCode = !completion.ok && completion.error.type === "INTERNAL_ERROR" && completion.error.code === TaskRunErrorCodes2.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE ? EXIT_CODE_CHILD_NONZERO : 0;
+    await this.#prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode);
+    if (willCheckpointAndRestore) {
+      if (!this.submitAttemptCompletionReplay) {
+        this.submitAttemptCompletionReplay = {
+          message: {
+            execution,
+            completion
+          },
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
+          logger.error(
+            "attempt completion handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.submitAttemptCompletionReplay.attempt++;
+      }
+    }
+  }
   #returnValidatedExtraHeaders(headers) {
     for (const [key, value] of Object.entries(headers)) {
       if (value === void 0) {
@@ -1052,7 +1582,7 @@ var ProdWorker = class {
     }
     return headers;
   }
-  // FIXME: If the the worker can't connect for a while, this runs MANY times - it should only run once
+  // MARK: COORDINATOR SOCKET
   #createCoordinatorSocket(host) {
     const extraHeaders = this.#returnValidatedExtraHeaders({
       "x-machine-name": MACHINE_NAME,
@@ -1076,6 +1606,10 @@ var ProdWorker = class {
       clientMessages: ProdWorkerToCoordinatorMessages,
       serverMessages: CoordinatorToProdWorkerMessages,
       extraHeaders,
+      ioOptions: {
+        reconnectionDelay: 1e3,
+        reconnectionDelayMax: 3e3
+      },
       handlers: {
         RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
           if (!this.paused) {
@@ -1101,6 +1635,16 @@ var ProdWorker = class {
             );
             return;
           }
+          switch (this.nextResumeAfter) {
+            case "WAIT_FOR_TASK": {
+              this.waitForTaskReplay = void 0;
+              break;
+            }
+            case "WAIT_FOR_BATCH": {
+              this.waitForBatchReplay = void 0;
+              break;
+            }
+          }
           this.paused = false;
           this.nextResumeAfter = void 0;
           this.waitForPostStart = false;
@@ -1124,8 +1668,10 @@ var ProdWorker = class {
             });
             return;
           }
+          this.durationResumeFallback = void 0;
           this.#resumeAfterDuration();
         },
+        // Deprecated: This will never get called as this worker supports lazy attempts. It's only here for a quick view of the flow old workers use.
         EXECUTE_TASK_RUN: async ({ executionPayload }) => {
           if (this.executing) {
             logger.error("dropping execute request, already executing");
@@ -1146,46 +1692,31 @@ var ProdWorker = class {
             completion
           });
           logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
-          this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+          await this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
         },
         EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
+          this.readyForLazyAttemptReplay = void 0;
           if (this.executing) {
             logger.error("dropping execute request, already executing");
             return;
           }
+          const attemptCount = message.lazyPayload.attemptCount ?? 0;
+          logger.log("execute attempt counts", { attemptCount, completed: this.completed.size });
+          if (this.completed.size > 0 && this.completed.size >= attemptCount + 1) {
+            logger.error("dropping execute request, already completed");
+            return;
+          }
           this.executing = true;
           try {
             const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
             logger.log("completed", completion);
             this.completed.add(execution.attempt.id);
-            const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
-              version: "v1",
-              execution,
-              completion
-            });
-            logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
-            this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+            await this.#submitAttemptCompletion(execution, completion);
           } catch (error) {
-            const completion = {
-              ok: false,
-              id: message.lazyPayload.runId,
-              retry: void 0,
-              error: error instanceof Error ? {
-                type: "BUILT_IN_ERROR",
-                name: error.name,
-                message: error.message,
-                stackTrace: error.stack ?? ""
-              } : {
-                type: "BUILT_IN_ERROR",
-                name: "UnknownError",
-                message: String(error),
-                stackTrace: ""
-              }
-            };
-            this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
-              version: "v1",
-              completion
+            logger.error("Failed to complete lazy attempt", {
+              error
             });
+            this.#failRun(message.lazyPayload.runId, error);
           }
         },
         REQUEST_ATTEMPT_CANCELLATION: async (message) => {
@@ -1199,146 +1730,153 @@ var ProdWorker = class {
         REQUEST_EXIT: async (message) => {
           if (message.version === "v2" && message.delayInMs) {
             logger.log("exit requested with delay", { delayInMs: message.delayInMs });
-            await setTimeout2(message.delayInMs);
+            await timeout2(message.delayInMs);
           }
           this.#coordinatorSocket.close();
           process.exit(0);
         },
         READY_FOR_RETRY: async (message) => {
           if (this.completed.size < 1) {
+            logger.error("Received READY_FOR_RETRY but no completions yet. This is a bug.");
             return;
           }
-          this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
-            version: "v1",
-            runId: this.runId,
-            totalCompletions: this.completed.size
-          });
+          this.submitAttemptCompletionReplay = void 0;
+          await this.#readyForLazyAttempt();
         }
       },
+      // MARK: ON CONNECTION
       onConnection: async (socket, handler, sender, logger2) => {
-        logger2.log("connected to coordinator", { status: this.#status });
-        if (this.waitForPostStart) {
-          logger2.log("skip connection handler, waiting for post start hook");
-          return;
-        }
-        if (this.paused) {
-          if (!this.nextResumeAfter) {
-            logger2.error("Missing next resume reason", { status: this.#status });
-            this.#emitUnrecoverableError(
-              "NoNextResume",
-              "Next resume reason not set while resuming from paused state"
-            );
+        logger2.log("connected to coordinator", {
+          status: this.#status,
+          connectionCount: ++this.connectionCount
+        });
+        socket.emit("SET_STATE", { version: "v1", attemptFriendlyId: this.attemptFriendlyId });
+        try {
+          if (this.waitForPostStart) {
+            logger2.log("skip connection handler, waiting for post start hook");
             return;
           }
-          if (!this.attemptFriendlyId) {
-            logger2.error("Missing friendly ID", { status: this.#status });
-            this.#emitUnrecoverableError(
-              "NoAttemptId",
-              "Attempt ID not set while resuming from paused state"
-            );
+          if (this.paused) {
+            if (!this.nextResumeAfter) {
+              logger2.error("Missing next resume reason", { status: this.#status });
+              this.#emitUnrecoverableError(
+                "NoNextResume",
+                "Next resume reason not set while resuming from paused state"
+              );
+              return;
+            }
+            if (!this.attemptFriendlyId) {
+              logger2.error("Missing friendly ID", { status: this.#status });
+              this.#emitUnrecoverableError(
+                "NoAttemptId",
+                "Attempt ID not set while resuming from paused state"
+              );
+              return;
+            }
+            socket.emit("READY_FOR_RESUME", {
+              version: "v1",
+              attemptFriendlyId: this.attemptFriendlyId,
+              type: this.nextResumeAfter
+            });
             return;
           }
-          socket.emit("READY_FOR_RESUME", {
-            version: "v1",
-            attemptFriendlyId: this.attemptFriendlyId,
-            type: this.nextResumeAfter
-          });
-          return;
-        }
-        if (process.env.INDEX_TASKS === "true") {
-          try {
-            const taskResources = await this.#initializeWorker();
-            const { success } = await socket.emitWithAck("INDEX_TASKS", {
-              version: "v2",
-              deploymentId: this.deploymentId,
-              ...taskResources,
-              supportsLazyAttempts: true
-            });
-            if (success) {
-              logger2.info("indexing done, shutting down..");
-              process.exit(0);
-            } else {
-              logger2.info("indexing failure, shutting down..");
-              process.exit(1);
-            }
-          } catch (e) {
-            const stderr = this.#backgroundWorker.stderr.join("\n");
-            if (e instanceof TaskMetadataParseError) {
-              logger2.error("tasks metadata parse error", {
-                zodIssues: e.zodIssues,
-                tasks: e.tasks
-              });
+          if (process.env.INDEX_TASKS === "true") {
+            const failIndex = (error) => {
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
+                error
+              });
+            };
+            process.removeAllListeners("uncaughtException");
+            process.on("uncaughtException", (error) => {
+              console.error("Uncaught exception while indexing", error);
+              failIndex(error);
+            });
+            try {
+              const taskResources = await this.#initializeWorker();
+              const indexTasks = await defaultBackoff.maxRetries(3).execute(async () => {
+                return await socket.timeout(2e4).emitWithAck("INDEX_TASKS", {
+                  version: "v2",
+                  deploymentId: this.deploymentId,
+                  ...taskResources,
+                  supportsLazyAttempts: true
+                });
+              });
+              if (!indexTasks.success || !indexTasks.result.success) {
+                logger2.error("indexing failure, shutting down..", { indexTasks });
+                process.exit(1);
+              } else {
+                logger2.info("indexing done, shutting down..");
+                process.exit(0);
+              }
+            } catch (e) {
+              const stderr = this.#backgroundWorker.stderr.join("\n");
+              if (e instanceof TaskMetadataParseError) {
+                logger2.error("tasks metadata parse error", {
+                  zodIssues: e.zodIssues,
+                  tasks: e.tasks
+                });
+                failIndex({
                   name: "TaskMetadataParseError",
                   message: "There was an error parsing the task metadata",
                   stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
                   stderr
-                }
-              });
-            } else if (e instanceof UncaughtExceptionError) {
-              const error = {
-                name: e.originalError.name,
-                message: e.originalError.message,
-                stack: e.originalError.stack,
-                stderr
-              };
-              logger2.error("uncaught exception", { originalError: error });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error
-              });
-            } else if (e instanceof Error) {
-              const error = {
-                name: e.name,
-                message: e.message,
-                stack: e.stack,
-                stderr
-              };
-              logger2.error("error", { error });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error
-              });
-            } else if (typeof e === "string") {
-              logger2.error("string error", { error: { message: e } });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                });
+              } else if (e instanceof UncaughtExceptionError) {
+                const error = {
+                  name: e.originalError.name,
+                  message: e.originalError.message,
+                  stack: e.originalError.stack,
+                  stderr
+                };
+                logger2.error("uncaught exception", { originalError: error });
+                failIndex(error);
+              } else if (e instanceof Error) {
+                const error = {
+                  name: e.name,
+                  message: e.message,
+                  stack: e.stack,
+                  stderr
+                };
+                logger2.error("error", { error });
+                failIndex(error);
+              } else if (typeof e === "string") {
+                logger2.error("string error", { error: { message: e } });
+                failIndex({
                   name: "Error",
                   message: e,
                   stderr
-                }
-              });
-            } else {
-              logger2.error("unknown error", { error: e });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                });
+              } else {
+                logger2.error("unknown error", { error: e });
+                failIndex({
                   name: "Error",
                   message: "Unknown error",
                   stderr
-                }
-              });
+                });
+              }
+              await timeout2(1e3);
+              process.exit(EXIT_CODE_ALREADY_HANDLED);
             }
-            await setTimeout2(200);
-            process.exit(111);
           }
+          if (this.executing) {
+            return;
+          }
+          process.removeAllListeners("uncaughtException");
+          process.on("uncaughtException", (error) => {
+            console.error("Uncaught exception during run", error);
+            this.#failRun(this.runId, error);
+          });
+          await this.#readyForLazyAttempt();
+        } catch (error) {
+          logger2.error("connection handler error", { error });
+        } finally {
+          if (this.connectionCount === 1) {
+            return;
+          }
+          this.#handleReplays();
         }
-        if (this.executing) {
-          return;
-        }
-        socket.emit("READY_FOR_LAZY_ATTEMPT", {
-          version: "v1",
-          runId: this.runId,
-          totalCompletions: this.completed.size
-        });
       },
       onError: async (socket, err, logger2) => {
         logger2.error("onError", {
@@ -1347,13 +1885,109 @@ var ProdWorker = class {
             message: err.message
           }
         });
-        await this.#reconnect();
-      },
-      onDisconnect: async (socket, reason, description, logger2) => {
       }
     });
     return coordinatorConnection;
   }
+  // MARK: REPLAYS
+  async #handleReplays() {
+    const backoff = new ExponentialBackoff().type("FullJitter").maxRetries(3);
+    const replayCancellationDelay = 2e4;
+    if (this.waitForTaskReplay) {
+      logger.log("replaying wait for task", { ...this.waitForTaskReplay });
+      const { idempotencyKey, message, attempt } = this.waitForTaskReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.waitForTaskReplay) {
+        logger.error("wait for task replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
+        logger.error("wait for task replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.waitForTaskReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#waitForTaskHandler(message);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("wait for task replay retry limit exceeded", { error });
+        } else {
+          logger.error("wait for task replay error", { error });
+        }
+      }
+      return;
+    }
+    if (this.waitForBatchReplay) {
+      logger.log("replaying wait for batch", {
+        ...this.waitForBatchReplay,
+        cancellationDelay: replayCancellationDelay
+      });
+      const { idempotencyKey, message, attempt } = this.waitForBatchReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.waitForBatchReplay) {
+        logger.error("wait for batch replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
+        logger.error("wait for batch replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.waitForBatchReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#waitForBatchHandler(message);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("wait for batch replay retry limit exceeded", { error });
+        } else {
+          logger.error("wait for batch replay error", { error });
+        }
+      }
+      return;
+    }
+    if (this.submitAttemptCompletionReplay) {
+      logger.log("replaying attempt completion", {
+        ...this.submitAttemptCompletionReplay,
+        cancellationDelay: replayCancellationDelay
+      });
+      const { idempotencyKey, message, attempt } = this.submitAttemptCompletionReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.submitAttemptCompletionReplay) {
+        logger.error("attempt completion replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
+        logger.error("attempt completion replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.submitAttemptCompletionReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#submitAttemptCompletion(message.execution, message.completion, idempotencyKey);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("attempt completion replay retry limit exceeded", { error });
+        } else {
+          logger.error("attempt completion replay error", { error });
+        }
+      }
+      return;
+    }
+  }
+  // MARK: HTTP SERVER
   #createHttpServer() {
     const httpServer = createServer(async (req, res) => {
       logger.log(`[${req.method}]`, req.url);
@@ -1372,17 +2006,13 @@ var ProdWorker = class {
             return reply.text("Connected to coordinator");
           }
           case "/close": {
-            await this.#coordinatorSocket.sendWithAck("LOG", {
-              version: "v1",
-              text: `[${req.method}] ${req.url}`
-            });
             this.#coordinatorSocket.close();
+            this.connectionCount = 0;
             return reply.text("Disconnected from coordinator");
           }
           case "/test": {
-            await this.#coordinatorSocket.sendWithAck("LOG", {
-              version: "v1",
-              text: `[${req.method}] ${req.url}`
+            await this.#coordinatorSocket.socket.timeout(1e4).emitWithAck("TEST", {
+              version: "v1"
             });
             return reply.text("Received ACK from coordinator");
           }
@@ -1417,7 +2047,7 @@ var ProdWorker = class {
                 break;
               }
               case "restore": {
-                await this.#reconnect(true, true);
+                await this.#reconnectAfterPostStart();
                 break;
               }
               default: {
@@ -1448,7 +2078,7 @@ var ProdWorker = class {
       }
       logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
       this.#httpPort = getRandomPortNumber();
-      await setTimeout2(100);
+      await timeout2(100);
       this.start();
     });
     return httpServer;
@@ -1458,8 +2088,12 @@ var ProdWorker = class {
     await this.#backgroundWorker.initialize({ env: envVars });
     let packageVersion;
     const taskResources = [];
-    if (!this.#backgroundWorker.tasks) {
-      throw new Error(`Background Worker started without tasks`);
+    if (!this.#backgroundWorker.tasks || this.#backgroundWorker.tasks.length === 0) {
+      throw new Error(
+        `Background Worker started without tasks. Searched in: ${__PROJECT_CONFIG__.triggerDirectories?.join(
+          ", "
+        )}`
+      );
     }
     for (const task of this.#backgroundWorker.tasks) {
       taskResources.push(task);
@@ -1493,7 +2127,9 @@ var ProdWorker = class {
       completed: this.completed.size,
       nextResumeAfter: this.nextResumeAfter,
       waitForPostStart: this.waitForPostStart,
-      attemptFriendlyId: this.attemptFriendlyId
+      attemptFriendlyId: this.attemptFriendlyId,
+      waitForTaskReplay: this.waitForTaskReplay,
+      waitForBatchReplay: this.waitForBatchReplay
     };
   }
   #emitUnrecoverableError(name, message) {