trigger.dev 3.0.0-beta.5 → 3.0.0-beta.50

package/dist/workers/prod/entry-point.js

@@ -4,8 +4,9 @@ import {
   PostStartCauses,
   PreStopCauses,
   ProdWorkerToCoordinatorMessages,
-  ZodSocketConnection as ZodSocketConnection2
+  TaskRunErrorCodes as TaskRunErrorCodes2
 } from "@trigger.dev/core/v3";
+import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 
 // ../core-apps/src/http.ts
 var HttpReply = class {
@@ -65,27 +66,280 @@ var SimpleLogger = class {
   }
 };
 
-// ../core-apps/src/provider.ts
-import {
-  ClientToSharedQueueMessages,
-  clientWebsocketMessages,
-  PlatformToProviderMessages,
-  ProviderToPlatformMessages,
-  SharedQueueToClientMessages,
-  ZodMessageSender,
-  ZodSocketConnection
-} from "@trigger.dev/core/v3";
-var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
-var MACHINE_NAME = process.env.MACHINE_NAME || "local";
-var PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
-var PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
-var PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
-var SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
-var logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+// ../core-apps/src/process.ts
+var EXIT_CODE_ALREADY_HANDLED = 111;
+var EXIT_CODE_CHILD_NONZERO = 112;
 
-// src/workers/prod/entry-point.ts
-import { readFile } from "node:fs/promises";
-import { createServer } from "node:http";
+// ../core-apps/src/backoff.ts
+import { setTimeout as timeout } from "node:timers/promises";
+var StopRetrying = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "StopRetrying";
+  }
+};
+var AttemptTimeout = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "AttemptTimeout";
+  }
+};
+var RetryLimitExceeded = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RetryLimitExceeded";
+  }
+};
+var ExponentialBackoff = class _ExponentialBackoff {
+  #retries = 0;
+  #type;
+  #base;
+  #factor;
+  #min;
+  #max;
+  #maxRetries;
+  #maxElapsed;
+  constructor(type, opts = {}) {
+    this.#type = type ?? "NoJitter";
+    this.#base = opts.base ?? 2;
+    this.#factor = opts.factor ?? 1;
+    this.#min = opts.min ?? -Infinity;
+    this.#max = opts.max ?? Infinity;
+    this.#maxRetries = opts.maxRetries ?? Infinity;
+    this.#maxElapsed = opts.maxElapsed ?? Infinity;
+  }
+  #clone(type, opts = {}) {
+    return new _ExponentialBackoff(type ?? this.#type, {
+      base: opts.base ?? this.#base,
+      factor: opts.factor ?? this.#factor,
+      min: opts.min ?? this.#min,
+      max: opts.max ?? this.#max,
+      maxRetries: opts.maxRetries ?? this.#maxRetries,
+      maxElapsed: opts.maxElapsed ?? this.#maxElapsed
+    });
+  }
+  type(type) {
+    return this.#clone(type);
+  }
+  base(base) {
+    return this.#clone(void 0, { base });
+  }
+  factor(factor) {
+    return this.#clone(void 0, { factor });
+  }
+  min(min) {
+    return this.#clone(void 0, { min });
+  }
+  max(max) {
+    return this.#clone(void 0, { max });
+  }
+  maxRetries(maxRetries) {
+    return this.#clone(void 0, { maxRetries });
+  }
+  // TODO: With .execute(), should this also include the time it takes to execute the callback?
+  maxElapsed(maxElapsed) {
+    return this.#clone(void 0, { maxElapsed });
+  }
+  retries(retries) {
+    if (typeof retries !== "undefined") {
+      if (retries > this.#maxRetries) {
+        console.error(
+          `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
+        );
+        this.#retries = this.#maxRetries;
+      } else {
+        this.#retries = retries;
+      }
+    }
+    return this.#clone();
+  }
+  async *retryAsync(maxRetries = this.#maxRetries ?? Infinity) {
+    let elapsed = 0;
+    let retry = 0;
+    while (retry <= maxRetries) {
+      const delay = this.delay(retry);
+      elapsed += delay;
+      if (elapsed > this.#maxElapsed) {
+        break;
+      }
+      yield {
+        delay: {
+          seconds: delay,
+          milliseconds: delay * 1e3
+        },
+        retry
+      };
+      retry++;
+    }
+  }
+  async *[Symbol.asyncIterator]() {
+    yield* this.retryAsync();
+  }
+  /** Returns the delay for the current retry in seconds. */
+  delay(retries = this.#retries, jitter = true) {
+    if (retries > this.#maxRetries) {
+      console.error(
+        `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
+      );
+      retries = this.#maxRetries;
+    }
+    let delay = this.#factor * this.#base ** retries;
+    switch (this.#type) {
+      case "NoJitter": {
+        break;
+      }
+      case "FullJitter": {
+        if (!jitter) {
+          delay = 0;
+          break;
+        }
+        delay *= Math.random();
+        break;
+      }
+      case "EqualJitter": {
+        if (!jitter) {
+          delay *= 0.5;
+          break;
+        }
+        delay *= 0.5 * (1 + Math.random());
+        break;
+      }
+      default: {
+        throw new Error(`Unknown backoff type: ${this.#type}`);
+      }
+    }
+    if (delay < this.#min) {
+      delay = this.#min + Math.random() * (this.#min * 0.2);
+    }
+    if (delay > this.#max) {
+      delay = this.#max - Math.random() * (this.#max * 0.2);
+    }
+    delay = Math.round(delay);
+    return delay;
+  }
+  /** Waits with the appropriate delay for the current retry. */
+  async wait(retries = this.#retries, jitter = true) {
+    if (retries > this.#maxRetries) {
+      console.error(`Retry limit exceeded: ${retries} > ${this.#maxRetries}`);
+      throw new RetryLimitExceeded();
+    }
+    const delay = this.delay(retries, jitter);
+    return await timeout(delay * 1e3);
+  }
+  elapsed(retries = this.#retries, jitter = true) {
+    let elapsed = 0;
+    for (let i = 0; i <= retries; i++) {
+      elapsed += this.delay(i, jitter);
+    }
+    const total = elapsed;
+    let days = 0;
+    if (elapsed > 3600 * 24) {
+      days = Math.floor(elapsed / 3600 / 24);
+      elapsed -= days * 3600 * 24;
+    }
+    let hours = 0;
+    if (elapsed > 3600) {
+      hours = Math.floor(elapsed / 3600);
+      elapsed -= hours * 3600;
+    }
+    let minutes = 0;
+    if (elapsed > 60) {
+      minutes = Math.floor(elapsed / 60);
+      elapsed -= minutes * 60;
+    }
+    const seconds = elapsed;
+    return {
+      seconds,
+      minutes,
+      hours,
+      days,
+      total
+    };
+  }
+  reset() {
+    this.#retries = 0;
+    return this;
+  }
+  next() {
+    this.#retries++;
+    return this.delay();
+  }
+  stop() {
+    throw new StopRetrying();
+  }
+  get state() {
+    return {
+      retries: this.#retries,
+      type: this.#type,
+      base: this.#base,
+      factor: this.#factor,
+      min: this.#min,
+      max: this.#max,
+      maxRetries: this.#maxRetries,
+      maxElapsed: this.#maxElapsed
+    };
+  }
+  async execute(callback, { attemptTimeoutMs = 0 } = {}) {
+    let elapsedMs = 0;
+    let finalError = void 0;
+    for await (const { delay, retry } of this) {
+      const start = Date.now();
+      if (retry > 0) {
+        console.log(`Retrying in ${delay.milliseconds}ms`);
+        await timeout(delay.milliseconds);
+      }
+      let attemptTimeout = void 0;
+      try {
+        const result = await new Promise(async (resolve, reject) => {
+          if (attemptTimeoutMs > 0) {
+            attemptTimeout = setTimeout(() => {
+              reject(new AttemptTimeout());
+            }, attemptTimeoutMs);
+          }
+          try {
+            const callbackResult = await callback({ delay, retry, elapsedMs });
+            resolve(callbackResult);
+          } catch (error) {
+            reject(error);
+          }
+        });
+        return {
+          success: true,
+          result
+        };
+      } catch (error) {
+        finalError = error;
+        if (error instanceof StopRetrying) {
+          return {
+            success: false,
+            cause: "StopRetrying",
+            error: error.message
+          };
+        }
+        if (error instanceof AttemptTimeout) {
+          continue;
+        }
+      } finally {
+        elapsedMs += Date.now() - start;
+        clearTimeout(attemptTimeout);
+      }
+    }
+    if (finalError instanceof AttemptTimeout) {
+      return {
+        success: false,
+        cause: "Timeout"
+      };
+    } else {
+      return {
+        success: false,
+        cause: "MaxRetries",
+        error: finalError
+      };
+    }
+  }
+  static RetryLimitExceeded = RetryLimitExceeded;
+  static StopRetrying = StopRetrying;
+};
 
 // src/workers/prod/backgroundWorker.ts
 import {
@@ -93,9 +347,9 @@ import {
   ProdWorkerToChildMessages,
   SemanticInternalAttributes,
   TaskRunErrorCodes,
-  ZodIpcConnection,
   correctErrorStackTrace
 } from "@trigger.dev/core/v3";
+import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { Evt } from "evt";
 import { fork } from "node:child_process";
 
@@ -116,12 +370,12 @@ var TaskMetadataParseError = class extends Error {
     this.name = "TaskMetadataParseError";
   }
 };
-
-// src/workers/prod/backgroundWorker.ts
 var UnexpectedExitError = class extends Error {
-  constructor(code) {
+  constructor(code, signal, stderr) {
     super(`Unexpected exit with code ${code}`);
     this.code = code;
+    this.signal = signal;
+    this.stderr = stderr;
     this.name = "UnexpectedExitError";
   }
 };
@@ -137,33 +391,101 @@ var CancelledProcessError = class extends Error {
     this.name = "CancelledProcessError";
   }
 };
+var SigKillTimeoutProcessError = class extends Error {
+  constructor() {
+    super("Process kill timeout");
+    this.name = "SigKillTimeoutProcessError";
+  }
+};
+var GracefulExitTimeoutError = class extends Error {
+  constructor() {
+    super("Graceful exit timeout");
+    this.name = "GracefulExitTimeoutError";
+  }
+};
+function getFriendlyErrorMessage(code, signal, stderr, dockerMode = true) {
+  const message = (text) => {
+    if (signal) {
+      return `[${signal}] ${text}`;
+    } else {
+      return text;
+    }
+  };
+  if (code === 137) {
+    if (dockerMode) {
+      return message(
+        "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+      );
+    } else {
+      return message(
+        "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
+      );
+    }
+  }
+  if (stderr?.includes("OOMErrorHandler")) {
+    return message(
+      "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+    );
+  }
+  return message(`Process exited with code ${code}.`);
+}
+
+// src/workers/prod/backgroundWorker.ts
 var ProdBackgroundWorker = class {
   constructor(path, params) {
     this.path = path;
     this.params = params;
   }
   _initialized = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
-  onWaitForBatch = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
-  preCheckpointNotification = Evt.create();
-  onReadyForCheckpoint = Evt.create();
-  onCancelCheckpoint = Evt.create();
+  onWaitForBatch = new Evt();
+  onCreateTaskRunAttempt = Evt.create();
+  attemptCreatedNotification = Evt.create();
   _onClose = new Evt();
   tasks = [];
+  stderr = [];
   _taskRunProcess;
+  _taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
   _closed = false;
-  async close() {
+  async close(gracefulExitTimeoutElapsed = false) {
+    console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
     if (this._closed) {
       return;
     }
     this._closed = true;
     this.onTaskHeartbeat.detach();
-    await this._taskRunProcess?.cleanup(true);
+    this.onTaskRunHeartbeat.detach();
+    await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
+  }
+  async #killTaskRunProcess(flush = true, initialSignal = "SIGTERM") {
+    console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
+    if (this._closed || !this._taskRunProcess) {
+      return;
+    }
+    if (flush) {
+      await this.flushTelemetry();
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
+      console.error("Error while trying graceful exit", error);
+    });
+    console.log("Killed task run process, setting closed to true", {
+      closed: this._closed,
+      pid: currentTaskRunProcess.pid
+    });
+    this._closed = true;
   }
   async flushTelemetry() {
+    console.log("Flushing telemetry");
+    const start = performance.now();
     await this._taskRunProcess?.cleanup(false);
+    console.log("Flushed telemetry", { duration: performance.now() - start });
   }
   async initialize(options) {
     if (this._initialized) {
@@ -186,7 +508,7 @@ var ProdBackgroundWorker = class {
           ...options?.env
         }
       });
-      const timeout = setTimeout(() => {
+      const timeout3 = setTimeout(() => {
         if (resolved) {
           return;
         }
@@ -194,6 +516,20 @@ var ProdBackgroundWorker = class {
         child.kill();
         reject(new Error("Worker timed out"));
       }, 1e4);
+      child.stdout?.on("data", (data) => {
+        console.log(data.toString());
+      });
+      child.stderr?.on("data", (data) => {
+        console.error(data.toString());
+        this.stderr.push(data.toString());
+      });
+      child.on("exit", (code) => {
+        if (!resolved) {
+          clearTimeout(timeout3);
+          resolved = true;
+          reject(new Error(`Worker exited with code ${code}`));
+        }
+      });
       new ZodIpcConnection({
         listenSchema: ProdChildToWorkerMessages,
         emitSchema: ProdWorkerToChildMessages,
@@ -201,7 +537,7 @@ var ProdBackgroundWorker = class {
         handlers: {
           TASKS_READY: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               resolve(message.tasks);
               child.kill();
@@ -209,7 +545,7 @@ var ProdBackgroundWorker = class {
           },
           UNCAUGHT_EXCEPTION: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               reject(new UncaughtExceptionError(message.error, message.origin));
               child.kill();
@@ -217,7 +553,7 @@ var ProdBackgroundWorker = class {
           },
           TASKS_FAILED_TO_PARSE: async (message) => {
             if (!resolved) {
-              clearTimeout(timeout);
+              clearTimeout(timeout3);
               resolved = true;
               reject(new TaskMetadataParseError(message.zodIssues, message.tasks));
               child.kill();
@@ -225,19 +561,6 @@ var ProdBackgroundWorker = class {
           }
         }
       });
-      child.stdout?.on("data", (data) => {
-        console.log(data.toString());
-      });
-      child.stderr?.on("data", (data) => {
-        console.error(data.toString());
-      });
-      child.on("exit", (code) => {
-        if (!resolved) {
-          clearTimeout(timeout);
-          resolved = true;
-          reject(new Error(`Worker exited with code ${code}`));
-        }
-      });
     });
     this._initialized = true;
   }
@@ -250,63 +573,135 @@ var ProdBackgroundWorker = class {
   }
   // We need to notify all the task run processes that a task run has completed,
   // in case they are waiting for it through triggerAndWait
-  async taskRunCompletedNotification(completion, execution) {
-    this._taskRunProcess?.taskRunCompletedNotification(completion, execution);
+  async taskRunCompletedNotification(completion) {
+    this._taskRunProcess?.taskRunCompletedNotification(completion);
   }
   async waitCompletedNotification() {
     this._taskRunProcess?.waitCompletedNotification();
   }
-  async #initializeTaskRunProcess(payload) {
+  async #getFreshTaskRunProcess(payload, messageId) {
     const metadata = this.getMetadata(
       payload.execution.worker.id,
       payload.execution.worker.version
     );
-    if (!this._taskRunProcess) {
-      const taskRunProcess = new TaskRunProcess(
-        payload.execution,
-        this.path,
-        {
-          ...this.params.env,
-          ...payload.environment ?? {}
-        },
-        metadata,
-        this.params
-      );
-      taskRunProcess.onExit.attach(() => {
+    console.log("Getting fresh task run process, setting closed to false", {
+      closed: this._closed
+    });
+    this._closed = false;
+    await this.#killCurrentTaskRunProcessBeforeAttempt();
+    const taskRunProcess = new TaskRunProcess(
+      payload.execution.run.id,
+      payload.execution.run.isTest,
+      this.path,
+      {
+        ...this.params.env,
+        ...payload.environment ?? {}
+      },
+      metadata,
+      this.params,
+      messageId
+    );
+    taskRunProcess.onExit.attach(({ pid }) => {
+      console.log("Task run process exited", { pid });
+      if (this._taskRunProcess?.pid === pid) {
         this._taskRunProcess = void 0;
-      });
-      taskRunProcess.onTaskHeartbeat.attach((id) => {
-        this.onTaskHeartbeat.post(id);
-      });
-      taskRunProcess.onWaitForBatch.attach((message) => {
-        this.onWaitForBatch.post(message);
-      });
-      taskRunProcess.onWaitForDuration.attach((message) => {
-        this.onWaitForDuration.post(message);
-      });
-      taskRunProcess.onWaitForTask.attach((message) => {
-        this.onWaitForTask.post(message);
-      });
-      taskRunProcess.onReadyForCheckpoint.attach((message) => {
-        this.onReadyForCheckpoint.post(message);
-      });
-      taskRunProcess.onCancelCheckpoint.attach((message) => {
-        this.onCancelCheckpoint.post(message);
-      });
-      this.preCheckpointNotification.attach((message) => {
-        taskRunProcess.preCheckpointNotification.post(message);
-      });
-      await taskRunProcess.initialize();
-      this._taskRunProcess = taskRunProcess;
-    }
+      }
+      if (pid) {
+        this._taskRunProcessesBeingKilled.delete(pid);
+      }
+    });
+    taskRunProcess.onIsBeingKilled.attach((taskRunProcess2) => {
+      if (taskRunProcess2?.pid) {
+        this._taskRunProcessesBeingKilled.set(taskRunProcess2.pid, taskRunProcess2);
+      }
+    });
+    taskRunProcess.onTaskHeartbeat.attach((id) => {
+      this.onTaskHeartbeat.post(id);
+    });
+    taskRunProcess.onTaskRunHeartbeat.attach((id) => {
+      this.onTaskRunHeartbeat.post(id);
+    });
+    taskRunProcess.onWaitForBatch.attach((message) => {
+      this.onWaitForBatch.post(message);
+    });
+    taskRunProcess.onWaitForDuration.attach((message) => {
+      this.onWaitForDuration.post(message);
+    });
+    taskRunProcess.onWaitForTask.attach((message) => {
+      this.onWaitForTask.post(message);
+    });
+    await taskRunProcess.initialize();
+    this._taskRunProcess = taskRunProcess;
     return this._taskRunProcess;
   }
-  // We need to fork the process before we can execute any tasks
-  async executeTaskRun(payload) {
+  async forceKillOldTaskRunProcesses() {
+    for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
+      try {
+        await taskRunProcess.kill("SIGKILL");
+      } catch (error) {
+        console.error("Error while force killing old task run processes", error);
+      }
+    }
+  }
+  async #killCurrentTaskRunProcessBeforeAttempt() {
+    console.log("killCurrentTaskRunProcessBeforeAttempt()", {
+      hasTaskRunProcess: !!this._taskRunProcess
+    });
+    if (!this._taskRunProcess) {
+      return;
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    console.log("Killing current task run process", {
+      isBeingKilled: currentTaskRunProcess?.isBeingKilled,
+      totalBeingKilled: this._taskRunProcessesBeingKilled.size
+    });
+    if (currentTaskRunProcess.isBeingKilled) {
+      if (this._taskRunProcessesBeingKilled.size > 1) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+      }
+    } else {
+      if (this._taskRunProcessesBeingKilled.size > 0) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+        currentTaskRunProcess.kill("SIGTERM", 5e3).catch(() => {
+        });
+      }
+    }
+  }
+  async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
+    console.log("Trying graceful exit", { kill, initialSignal });
     try {
-      const taskRunProcess = await this.#initializeTaskRunProcess(payload);
+      const initialExit = taskRunProcess.onExit.waitFor(5e3);
+      if (kill) {
+        taskRunProcess.kill(initialSignal);
+      }
+      await initialExit;
+    } catch (error) {
+      console.error("TaskRunProcess graceful kill timeout exceeded", error);
+      this.#tryForcefulExit(taskRunProcess);
+    }
+  }
+  async #tryForcefulExit(taskRunProcess) {
+    console.log("Trying forceful exit");
+    try {
+      const forcedKill = taskRunProcess.onExit.waitFor(5e3);
+      taskRunProcess.kill("SIGKILL");
+      await forcedKill;
+    } catch (error) {
+      console.error("TaskRunProcess forced kill timeout exceeded", error);
+      throw new SigKillTimeoutProcessError();
+    }
+  }
+  // We need to fork the process before we can execute any tasks, use a fresh process for each execution
+  async executeTaskRun(payload, messageId) {
+    try {
+      const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
+      console.log("executing task run", {
+        attempt: payload.execution.attempt.id,
+        taskRunPid: taskRunProcess.pid
+      });
       const result = await taskRunProcess.executeTaskRun(payload);
-      await taskRunProcess.cleanup(result.ok || result.retry === void 0);
       if (result.ok) {
         return result;
       }
@@ -349,7 +744,32 @@ var ProdBackgroundWorker = class {
           retry: void 0,
           error: {
             type: "INTERNAL_ERROR",
-            code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE
+            code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
+            message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
+            stackTrace: e.stderr
+          }
+        };
+      }
+      if (e instanceof SigKillTimeoutProcessError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT
+          }
+        };
+      }
+      if (e instanceof GracefulExitTimeoutError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
+            message: "Worker process killed while attempt in progress."
           }
         };
       }
@@ -362,10 +782,44 @@ var ProdBackgroundWorker = class {
           code: TaskRunErrorCodes.TASK_EXECUTION_FAILED
         }
       };
+    } finally {
+      await this.#killTaskRunProcess();
     }
   }
   async cancelAttempt(attemptId) {
-    await this._taskRunProcess?.cancel();
+    if (!this._taskRunProcess) {
+      console.error("No task run process to cancel attempt", { attemptId });
+      return;
+    }
+    await this._taskRunProcess.cancel();
+  }
+  async executeTaskRunLazyAttempt(payload) {
+    this.onCreateTaskRunAttempt.post({ runId: payload.runId });
+    let execution;
+    try {
+      const start = performance.now();
+      const attemptCreated = await this.attemptCreatedNotification.waitFor(12e4);
+      if (!attemptCreated.success) {
+        throw new Error(`${attemptCreated.reason ?? "Unknown error"}`);
+      }
+      console.log("Attempt created", {
+        number: attemptCreated.execution.attempt.number,
+        duration: performance.now() - start
+      });
+      execution = attemptCreated.execution;
+    } catch (error) {
+      console.error("Error while creating attempt", error);
+      throw new Error(`Failed to create attempt: ${error}`);
+    }
+    const completion = await this.executeTaskRun(
+      {
+        execution,
+        traceContext: payload.traceContext,
+        environment: payload.environment
+      },
+      payload.messageId
+    );
+    return { execution, completion };
   }
   async #correctError(error, execution) {
     return {
@@ -375,28 +829,36 @@ var ProdBackgroundWorker = class {
   }
 };
 var TaskRunProcess = class {
-  constructor(execution, path, env, metadata, worker) {
-    this.execution = execution;
+  constructor(runId, isTest, path, env, metadata, worker, messageId) {
+    this.runId = runId;
+    this.isTest = isTest;
     this.path = path;
     this.env = env;
     this.metadata = metadata;
     this.worker = worker;
+    this.messageId = messageId;
   }
   _ipc;
   _child;
+  _childPid;
   _attemptPromises = /* @__PURE__ */ new Map();
   _attemptStatuses = /* @__PURE__ */ new Map();
   _currentExecution;
   _isBeingKilled = false;
   _isBeingCancelled = false;
+  _gracefulExitTimeoutElapsed = false;
+  _stderr = [];
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onExit = new Evt();
-  onWaitForBatch = new Evt();
+  onIsBeingKilled = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
+  onWaitForBatch = new Evt();
   preCheckpointNotification = Evt.create();
-  onReadyForCheckpoint = Evt.create();
-  onCancelCheckpoint = Evt.create();
   async initialize() {
     this._child = fork(this.path, {
       stdio: [
@@ -409,7 +871,7 @@ var TaskRunProcess = class {
         "ipc"
       ],
       env: {
-        ...this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
+        ...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
         ...this.env,
         OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
           [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
@@ -417,6 +879,7 @@ var TaskRunProcess = class {
         ...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
       }
     });
+    this._childPid = this._child?.pid;
     this._ipc = new ZodIpcConnection({
       listenSchema: ProdChildToWorkerMessages,
       emitSchema: ProdWorkerToChildMessages,
@@ -437,28 +900,29 @@ var TaskRunProcess = class {
           resolver(result);
         },
         READY_TO_DISPOSE: async (message) => {
+          process.exit(0);
         },
         TASK_HEARTBEAT: async (message) => {
-          this.onTaskHeartbeat.post(message.id);
+          if (this.messageId) {
+            this.onTaskRunHeartbeat.post(this.messageId);
+          } else {
+            console.error(
+              "No message id for task heartbeat, falling back to (deprecated) attempt heartbeat",
+              { id: message.id }
+            );
+            this.onTaskHeartbeat.post(message.id);
+          }
         },
         TASKS_READY: async (message) => {
         },
+        WAIT_FOR_TASK: async (message) => {
+          this.onWaitForTask.post(message);
+        },
         WAIT_FOR_BATCH: async (message) => {
           this.onWaitForBatch.post(message);
         },
         WAIT_FOR_DURATION: async (message) => {
           this.onWaitForDuration.post(message);
-          const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor();
-          return { willCheckpointAndRestore };
-        },
-        WAIT_FOR_TASK: async (message) => {
-          this.onWaitForTask.post(message);
-        },
-        READY_FOR_CHECKPOINT: async (message) => {
-          this.onReadyForCheckpoint.post(message);
-        },
-        CANCEL_CHECKPOINT: async (message) => {
-          this.onCancelCheckpoint.post(message);
         }
       }
     });
@@ -470,15 +934,43 @@ var TaskRunProcess = class {
     this._isBeingCancelled = true;
     await this.cleanup(true);
   }
-  async cleanup(kill = false) {
+  async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
+    console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
     if (kill && this._isBeingKilled) {
       return;
     }
-    this._isBeingKilled = kill;
-    await this._ipc?.sendWithAck("CLEANUP", {
-      flush: true,
-      kill
+    if (kill) {
+      this._isBeingKilled = true;
+      this.onIsBeingKilled.post(this);
+    }
+    const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
+    const killParentProcess = kill && !killChildProcess;
+    console.log("Cleaning up task run process", {
+      killChildProcess,
+      killParentProcess,
+      ipc: this._ipc,
+      childPid: this._childPid,
+      realChildPid: this._child?.pid
     });
+    try {
+      await this._ipc?.sendWithAck(
+        "CLEANUP",
+        {
+          flush: true,
+          kill: killParentProcess
+        },
+        3e4
+      );
+    } catch (error) {
+      console.error("Error while cleaning up task run process", error);
+      if (killParentProcess) {
+        process.exit(0);
+      }
+    }
+    if (killChildProcess) {
+      this._gracefulExitTimeoutElapsed = true;
+      await this.kill("SIGKILL");
+    }
   }
   async executeTaskRun(payload) {
     let resolver;
@@ -502,25 +994,38 @@ var TaskRunProcess = class {
     this._currentExecution = void 0;
     return result;
   }
-  taskRunCompletedNotification(completion, execution) {
+  taskRunCompletedNotification(completion) {
     if (!completion.ok && typeof completion.retry !== "undefined") {
+      console.error(
+        "Task run completed with error and wants to retry, won't send task run completed notification"
+      );
       return;
     }
-    if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
-      this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
-        completion,
-        execution
-      });
+    if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
+      console.error(
+        "Child process not connected or being killed, can't send task run completed notification"
+      );
+      return;
     }
+    this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
+      version: "v2",
+      completion
+    });
   }
   waitCompletedNotification() {
-    if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
-      this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
+    if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
+      console.error(
+        "Child process not connected or being killed, can't send wait completed notification"
+      );
+      return;
     }
+    this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
   }
-  async #handleExit(code) {
+  async #handleExit(code, signal) {
+    console.log("handling child exit", { code, signal });
     for (const [id, status] of this._attemptStatuses.entries()) {
       if (status === "PENDING") {
+        console.log("found pending attempt", { id });
         this._attemptStatuses.set(id, "REJECTED");
         const attemptPromise = this._attemptPromises.get(id);
         if (!attemptPromise) {
@@ -529,124 +1034,73 @@ var TaskRunProcess = class {
         const { rejecter } = attemptPromise;
         if (this._isBeingCancelled) {
           rejecter(new CancelledProcessError());
+        } else if (this._gracefulExitTimeoutElapsed) {
+          rejecter(new GracefulExitTimeoutError());
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code));
+          rejecter(
+            new UnexpectedExitError(
+              code ?? -1,
+              signal,
+              this._stderr.length ? this._stderr.join("\n") : void 0
+            )
+          );
         }
       }
     }
-    this.onExit.post(code);
+    this.onExit.post({ code, signal, pid: this.pid });
   }
   #handleLog(data) {
-    if (!this._currentExecution) {
-      return;
-    }
-    console.log(
-      `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number}] ${data.toString()}`
-    );
+    console.log(data.toString());
   }
   #handleStdErr(data) {
-    if (this._isBeingKilled) {
-      return;
-    }
-    if (!this._currentExecution) {
-      console.error(`[${this.metadata.version}] ${data.toString()}`);
-      return;
+    const text = data.toString();
+    console.error(text);
+    if (this._stderr.length > 100) {
+      this._stderr.shift();
     }
-    console.error(
-      `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number}] ${data.toString()}`
-    );
+    this._stderr.push(text);
   }
-  #kill() {
-    if (this._child && !this._child.killed) {
-      this._child?.kill();
+  async kill(signal, timeoutInMs) {
+    this._isBeingKilled = true;
+    const killTimeout = this.onExit.waitFor(timeoutInMs);
+    this.onIsBeingKilled.post(this);
+    this._child?.kill(signal);
+    if (timeoutInMs) {
+      await killTimeout;
     }
   }
+  get isBeingKilled() {
+    return this._isBeingKilled || this._child?.killed;
+  }
+  get pid() {
+    return this._childPid;
+  }
 };
 
 // src/workers/prod/entry-point.ts
-import { setTimeout as setTimeout2 } from "node:timers/promises";
-var HTTP_SERVER_PORT2 = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
-var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers";
+import { randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { setTimeout as timeout2 } from "node:timers/promises";
+var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
+var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
 var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
-var MACHINE_NAME2 = process.env.MACHINE_NAME || "local";
+var MACHINE_NAME = process.env.MACHINE_NAME || "local";
 var POD_NAME = process.env.POD_NAME || "some-pod";
 var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
-var logger2 = new SimpleLogger(`[${MACHINE_NAME2}][${SHORT_HASH}]`);
+var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
+var defaultBackoff = new ExponentialBackoff("FullJitter", {
+  maxRetries: 5
+});
 var ProdWorker = class {
   constructor(port, host = "0.0.0.0") {
     this.host = host;
+    process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
     this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
-    this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
-      projectConfig: __PROJECT_CONFIG__,
-      env: {
-        ...gatherProcessEnv(),
-        TRIGGER_API_URL: this.apiUrl,
-        TRIGGER_SECRET_KEY: this.apiKey,
-        OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"
-      },
-      contentHash: this.contentHash
-    });
-    this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
-      this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
-    });
-    this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
-      this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
-    });
-    this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
-      logger2.log("onCancelCheckpoint() clearing paused state, don't wait for post start hook", {
-        paused: this.paused,
-        nextResumeAfter: this.nextResumeAfter,
-        waitForPostStart: this.waitForPostStart
-      });
-      this.paused = false;
-      this.nextResumeAfter = void 0;
-      this.waitForPostStart = false;
-      this.#coordinatorSocket.socket.emit("CANCEL_CHECKPOINT", { version: "v1" });
-    });
-    this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
-        return;
-      }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_DURATION",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
-        }
-      );
-      this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
-    });
-    this.#backgroundWorker.onWaitForTask.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
-        return;
-      }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_TASK",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
-        }
-      );
-      this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
-    });
-    this.#backgroundWorker.onWaitForBatch.attach(async (message) => {
-      if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
-        return;
-      }
-      const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
-        "WAIT_FOR_BATCH",
-        {
-          ...message,
-          attemptFriendlyId: this.attemptFriendlyId
-        }
-      );
-      this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
-    });
+    this.#backgroundWorker = this.#createBackgroundWorker();
     this.#httpPort = port;
     this.#httpServer = this.#createHttpServer();
   }
@@ -665,68 +1119,461 @@ var ProdWorker = class {
   attemptFriendlyId;
   nextResumeAfter;
   waitForPostStart = false;
+  connectionCount = 0;
+  waitForTaskReplay;
+  waitForBatchReplay;
+  readyForLazyAttemptReplay;
+  submitAttemptCompletionReplay;
+  durationResumeFallback;
   #httpPort;
   #backgroundWorker;
   #httpServer;
   #coordinatorSocket;
-  async #reconnect(isPostStart = false) {
-    if (isPostStart) {
-      this.waitForPostStart = false;
-    }
-    this.#coordinatorSocket.close();
-    if (!this.runningInKubernetes) {
-      this.#coordinatorSocket.connect();
+  async #handleSignal(signal) {
+    logger.log("Received signal", { signal });
+    if (signal === "SIGTERM") {
+      let gracefulExitTimeoutElapsed = false;
+      if (this.executing) {
+        const terminationGracePeriodSeconds = 60 * 60;
+        logger.log("Waiting for attempt to complete before exiting", {
+          terminationGracePeriodSeconds
+        });
+        await timeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
+        gracefulExitTimeoutElapsed = true;
+        logger.log("Termination timeout reached, exiting gracefully.");
+      } else {
+        logger.log("Not executing, exiting immediately.");
+      }
+      await this.#exitGracefully(gracefulExitTimeoutElapsed);
       return;
     }
+    logger.log("Unhandled signal", { signal });
+  }
+  async #exitGracefully(gracefulExitTimeoutElapsed = false, exitCode = 0) {
+    await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
+    if (!gracefulExitTimeoutElapsed) {
+      process.exit(exitCode);
+    }
+  }
+  async #reconnectAfterPostStart() {
+    this.waitForPostStart = false;
+    this.#coordinatorSocket.close();
+    this.connectionCount = 0;
+    let coordinatorHost = COORDINATOR_HOST;
     try {
-      const coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
-        "\n",
-        ""
-      );
-      logger2.log("reconnecting", {
-        coordinatorHost: {
-          fromEnv: COORDINATOR_HOST,
-          fromVolume: coordinatorHost,
-          current: this.#coordinatorSocket.socket.io.opts.hostname
-        }
+      if (this.runningInKubernetes) {
+        coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
+          "\n",
+          ""
+        );
+        logger.log("reconnecting", {
+          coordinatorHost: {
+            fromEnv: COORDINATOR_HOST,
+            fromVolume: coordinatorHost,
+            current: this.#coordinatorSocket.socket.io.opts.hostname
+          }
+        });
+      }
+    } catch (error) {
+      logger.error("taskinfo read error during reconnect", {
+        error: error instanceof Error ? error.message : error
       });
+    } finally {
       this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
-    } catch (error) {
-      logger2.error("taskinfo read error during reconnect", { error });
-      this.#coordinatorSocket.connect();
     }
   }
-  #prepareForWait(reason, willCheckpointAndRestore) {
-    logger2.log(`prepare for ${reason}`, { willCheckpointAndRestore });
-    this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
+  // MARK: TASK WAIT
+  async #waitForTaskHandler(message, replayIdempotencyKey) {
+    const waitForTask = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Wait for task with backoff", { retry });
+      if (!this.attemptFriendlyId) {
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        throw new ExponentialBackoff.StopRetrying("No attempt ID");
+      }
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_TASK", {
+        version: "v2",
+        friendlyId: message.friendlyId,
+        attemptFriendlyId: this.attemptFriendlyId
+      });
+    });
+    if (!waitForTask.success) {
+      logger.error("Failed to wait for task with backoff", {
+        cause: waitForTask.cause,
+        error: waitForTask.error
+      });
+      this.#emitUnrecoverableError(
+        "WaitForTaskFailed",
+        `${waitForTask.cause}: ${waitForTask.error}`
+      );
+      return;
+    }
+    const { willCheckpointAndRestore } = waitForTask.result;
+    await this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
+    if (willCheckpointAndRestore) {
+      if (!this.waitForTaskReplay) {
+        this.waitForTaskReplay = {
+          message,
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
+          logger.error(
+            "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.waitForTaskReplay.attempt++;
+      }
+    }
+  }
+  // MARK: BATCH WAIT
+  async #waitForBatchHandler(message, replayIdempotencyKey) {
+    const waitForBatch = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Wait for batch with backoff", { retry });
+      if (!this.attemptFriendlyId) {
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        throw new ExponentialBackoff.StopRetrying("No attempt ID");
+      }
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_BATCH", {
+        version: "v2",
+        batchFriendlyId: message.batchFriendlyId,
+        runFriendlyIds: message.runFriendlyIds,
+        attemptFriendlyId: this.attemptFriendlyId
+      });
+    });
+    if (!waitForBatch.success) {
+      logger.error("Failed to wait for batch with backoff", {
+        cause: waitForBatch.cause,
+        error: waitForBatch.error
+      });
+      this.#emitUnrecoverableError(
+        "WaitForBatchFailed",
+        `${waitForBatch.cause}: ${waitForBatch.error}`
+      );
+      return;
+    }
+    const { willCheckpointAndRestore } = waitForBatch.result;
+    await this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
     if (willCheckpointAndRestore) {
-      this.paused = true;
-      this.nextResumeAfter = reason;
-      this.waitForPostStart = true;
+      if (!this.waitForBatchReplay) {
+        this.waitForBatchReplay = {
+          message,
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
+          logger.error(
+            "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.waitForBatchReplay.attempt++;
+      }
+    }
+  }
+  // MARK: WORKER CREATION
+  #createBackgroundWorker() {
+    const backgroundWorker = new ProdBackgroundWorker("worker.js", {
+      projectConfig: __PROJECT_CONFIG__,
+      env: {
+        ...gatherProcessEnv(),
+        TRIGGER_API_URL: this.apiUrl,
+        TRIGGER_SECRET_KEY: this.apiKey,
+        OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"
+      },
+      contentHash: this.contentHash
+    });
+    backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
+      logger.log("onTaskHeartbeat", { attemptFriendlyId });
+      this.#coordinatorSocket.socket.volatile.emit("TASK_HEARTBEAT", {
+        version: "v1",
+        attemptFriendlyId
+      });
+    });
+    backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
+      logger.log("onTaskRunHeartbeat", { runId });
+      this.#coordinatorSocket.socket.volatile.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
+    });
+    backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
+      logger.log("onCreateTaskRunAttempt()", { message });
+      const createAttempt = await defaultBackoff.execute(async ({ retry }) => {
+        logger.log("Create task run attempt with backoff", { retry });
+        return await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CREATE_TASK_RUN_ATTEMPT", {
+          version: "v1",
+          runId: message.runId
+        });
+      });
+      if (!createAttempt.success) {
+        backgroundWorker.attemptCreatedNotification.post({
+          success: false,
+          reason: `Failed to create attempt with backoff due to ${createAttempt.cause}. ${createAttempt.error}`
+        });
+        return;
+      }
+      if (!createAttempt.result.success) {
+        backgroundWorker.attemptCreatedNotification.post({
+          success: false,
+          reason: createAttempt.result.reason
+        });
+        return;
+      }
+      backgroundWorker.attemptCreatedNotification.post({
+        success: true,
+        execution: createAttempt.result.executionPayload.execution
+      });
+    });
+    backgroundWorker.attemptCreatedNotification.attach((message) => {
+      logger.log("attemptCreatedNotification", {
+        success: message.success,
+        ...message.success ? {
+          attempt: message.execution.attempt,
+          queue: message.execution.queue,
+          worker: message.execution.worker,
+          machine: message.execution.machine
+        } : {
+          reason: message.reason
+        }
+      });
+      if (!message.success) {
+        return;
+      }
+      this.attemptFriendlyId = message.execution.attempt.id;
+    });
+    backgroundWorker.onWaitForDuration.attach(async (message) => {
+      logger.log("onWaitForDuration", { ...message, drift: Date.now() - message.now });
+      noResume: {
+        const { ms, waitThresholdInMs } = message;
+        const internalTimeout = unboundedTimeout(ms, "internal");
+        const checkpointSafeInternalTimeout = checkpointSafeTimeout(ms);
+        if (ms < waitThresholdInMs) {
+          await internalTimeout;
+          break noResume;
+        }
+        const waitForDuration = await defaultBackoff.execute(async ({ retry }) => {
+          logger.log("Wait for duration with backoff", { retry });
+          if (!this.attemptFriendlyId) {
+            logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+            throw new ExponentialBackoff.StopRetrying("No attempt ID");
+          }
+          return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_DURATION", {
+            ...message,
+            attemptFriendlyId: this.attemptFriendlyId
+          });
+        });
+        if (!waitForDuration.success) {
+          logger.error("Failed to wait for duration with backoff", {
+            cause: waitForDuration.cause,
+            error: waitForDuration.error
+          });
+          this.#emitUnrecoverableError(
+            "WaitForDurationFailed",
+            `${waitForDuration.cause}: ${waitForDuration.error}`
+          );
+          return;
+        }
+        const { willCheckpointAndRestore } = waitForDuration.result;
+        if (!willCheckpointAndRestore) {
+          await internalTimeout;
+          break noResume;
+        }
+        await this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
+        await Promise.race([internalTimeout, checkpointSafeInternalTimeout]);
+        try {
+          const { checkpointCanceled } = await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CANCEL_CHECKPOINT", {
+            version: "v2",
+            reason: "WAIT_FOR_DURATION"
+          });
+          logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
+          if (checkpointCanceled) {
+            break noResume;
+          }
+          logger.log("Waiting for external duration resume as we may have been restored");
+          const idempotencyKey = randomUUID();
+          this.durationResumeFallback = { idempotencyKey };
+          setTimeout(() => {
+            if (!this.durationResumeFallback) {
+              logger.error("Already resumed after duration, skipping fallback");
+              return;
+            }
+            if (this.durationResumeFallback.idempotencyKey !== idempotencyKey) {
+              logger.error("Duration resume idempotency key mismatch, skipping fallback");
+              return;
+            }
+            logger.log("Resuming after duration with fallback");
+            this.#resumeAfterDuration();
+          }, 15e3);
+        } catch (error) {
+          logger.debug("Checkpoint cancellation timed out", { error });
+          break noResume;
+        }
+        return;
+      }
+      this.#resumeAfterDuration();
+    });
+    backgroundWorker.onWaitForTask.attach(this.#waitForTaskHandler.bind(this));
+    backgroundWorker.onWaitForBatch.attach(this.#waitForBatchHandler.bind(this));
+    return backgroundWorker;
+  }
+  async #prepareForWait(reason, willCheckpointAndRestore) {
+    logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
+    if (!willCheckpointAndRestore) {
+      return;
     }
+    this.paused = true;
+    this.nextResumeAfter = reason;
+    this.waitForPostStart = true;
+    await this.#prepareForCheckpoint();
   }
-  async #prepareForRetry(willCheckpointAndRestore, shouldExit) {
-    logger2.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
+  // MARK: RETRY PREP
+  async #prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode) {
+    logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit, exitCode });
     if (shouldExit) {
       if (willCheckpointAndRestore) {
-        logger2.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
+        logger.error("WARNING: Will checkpoint but also requested exit. This won't end well.");
       }
-      await this.#backgroundWorker.close();
-      process.exit(0);
+      await this.#exitGracefully(false, exitCode);
+      return;
     }
+    this.paused = false;
+    this.waitForPostStart = false;
     this.executing = false;
     this.attemptFriendlyId = void 0;
-    if (willCheckpointAndRestore) {
-      this.waitForPostStart = true;
-      this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
+    if (!willCheckpointAndRestore) {
       return;
     }
+    this.waitForPostStart = true;
+    await this.#prepareForCheckpoint(false);
+  }
+  // MARK: CHECKPOINT PREP
+  async #prepareForCheckpoint(flush = true) {
+    if (flush) {
+      try {
+        await this.#backgroundWorker.flushTelemetry();
+      } catch (error) {
+        logger.error(
+          "Failed to flush telemetry while preparing for checkpoint, will proceed anyway",
+          { error }
+        );
+      }
+    }
+    try {
+      await this.#backgroundWorker.forceKillOldTaskRunProcesses();
+    } catch (error) {
+      logger.error(
+        "Failed to kill previous worker while preparing for checkpoint, will proceed anyway",
+        { error }
+      );
+    }
+    this.#readyForCheckpoint();
   }
   #resumeAfterDuration() {
     this.paused = false;
     this.nextResumeAfter = void 0;
+    this.waitForPostStart = false;
     this.#backgroundWorker.waitCompletedNotification();
   }
+  async #readyForLazyAttempt() {
+    const idempotencyKey = randomUUID();
+    this.readyForLazyAttemptReplay = {
+      idempotencyKey
+    };
+    for await (const { delay, retry } of defaultBackoff.min(10).maxRetries(3)) {
+      if (retry > 0) {
+        logger.log("retrying ready for lazy attempt", { retry });
+      }
+      this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
+        version: "v1",
+        runId: this.runId,
+        totalCompletions: this.completed.size
+      });
+      await timeout2(delay.milliseconds);
+      if (!this.readyForLazyAttemptReplay) {
+        logger.error("replay ready for lazy attempt cancelled, discarding", {
+          idempotencyKey
+        });
+        return;
+      }
+      if (idempotencyKey !== this.readyForLazyAttemptReplay.idempotencyKey) {
+        logger.error("replay ready for lazy attempt idempotency key mismatch, discarding", {
+          idempotencyKey,
+          newIdempotencyKey: this.readyForLazyAttemptReplay.idempotencyKey
+        });
+        return;
+      }
+    }
+    this.#failRun(this.runId, "Failed to receive execute request in a reasonable time");
+  }
+  #readyForCheckpoint() {
+    this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
+  }
+  #failRun(anyRunId, error) {
+    logger.error("Failing run", { anyRunId, error });
+    const completion = {
+      ok: false,
+      id: anyRunId,
+      retry: void 0,
+      error: error instanceof Error ? {
+        type: "BUILT_IN_ERROR",
+        name: error.name,
+        message: error.message,
+        stackTrace: error.stack ?? ""
+      } : {
+        type: "BUILT_IN_ERROR",
+        name: "UnknownError",
+        message: String(error),
+        stackTrace: ""
+      }
+    };
+    this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
+      version: "v1",
+      completion
+    });
+  }
+  // MARK: ATTEMPT COMPLETION
+  async #submitAttemptCompletion(execution, completion, replayIdempotencyKey) {
+    const taskRunCompleted = await defaultBackoff.execute(async ({ retry }) => {
+      logger.log("Submit attempt completion with backoff", { retry });
+      return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("TASK_RUN_COMPLETED", {
+        version: "v1",
+        execution,
+        completion
+      });
+    });
+    if (!taskRunCompleted.success) {
+      logger.error("Failed to complete lazy attempt with backoff", {
+        cause: taskRunCompleted.cause,
+        error: taskRunCompleted.error
+      });
+      this.#failRun(execution.run.id, taskRunCompleted.error);
+      return;
+    }
+    const { willCheckpointAndRestore, shouldExit } = taskRunCompleted.result;
+    logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+    const exitCode = !completion.ok && completion.error.type === "INTERNAL_ERROR" && completion.error.code === TaskRunErrorCodes2.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE ? EXIT_CODE_CHILD_NONZERO : 0;
+    await this.#prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode);
+    if (willCheckpointAndRestore) {
+      if (!this.submitAttemptCompletionReplay) {
+        this.submitAttemptCompletionReplay = {
+          message: {
+            execution,
+            completion
+          },
+          attempt: 1,
+          idempotencyKey: randomUUID()
+        };
+      } else {
+        if (replayIdempotencyKey && replayIdempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
+          logger.error(
+            "attempt completion handler called with mismatched idempotency key, won't overwrite replay request"
+          );
+          return;
+        }
+        this.submitAttemptCompletionReplay.attempt++;
+      }
+    }
+  }
   #returnValidatedExtraHeaders(headers) {
     for (const [key, value] of Object.entries(headers)) {
       if (value === void 0) {
@@ -735,9 +1582,10 @@ var ProdWorker = class {
     }
     return headers;
   }
+  // MARK: COORDINATOR SOCKET
   #createCoordinatorSocket(host) {
     const extraHeaders = this.#returnValidatedExtraHeaders({
-      "x-machine-name": MACHINE_NAME2,
+      "x-machine-name": MACHINE_NAME,
       "x-pod-name": POD_NAME,
       "x-trigger-content-hash": this.contentHash,
       "x-trigger-project-ref": this.projectRef,
@@ -749,247 +1597,400 @@ var ProdWorker = class {
     if (this.attemptFriendlyId) {
       extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
     }
-    logger2.log("connecting to coordinator", {
-      host,
-      port: COORDINATOR_PORT,
-      extraHeaders
-    });
-    const coordinatorConnection = new ZodSocketConnection2({
+    logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
+    logger.debug(`connecting with extra headers`, { extraHeaders });
+    const coordinatorConnection = new ZodSocketConnection({
       namespace: "prod-worker",
       host,
       port: COORDINATOR_PORT,
       clientMessages: ProdWorkerToCoordinatorMessages,
       serverMessages: CoordinatorToProdWorkerMessages,
       extraHeaders,
+      ioOptions: {
+        reconnectionDelay: 1e3,
+        reconnectionDelayMax: 3e3
+      },
       handlers: {
-        RESUME_AFTER_DEPENDENCY: async (message) => {
+        RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
           if (!this.paused) {
-            logger2.error("worker not paused", {
-              completions: message.completions,
-              executions: message.executions
-            });
+            logger.error("Failed to resume after dependency: Worker not paused");
             return;
           }
-          if (message.completions.length !== message.executions.length) {
-            logger2.error("did not receive the same number of completions and executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
-            return;
-          }
-          if (message.completions.length === 0 || message.executions.length === 0) {
-            logger2.error("no completions or executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (completions.length === 0) {
+            logger.error("Failed to resume after dependency: No completions");
             return;
           }
           if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
-            logger2.error("not waiting to resume after dependency", {
+            logger.error("Failed to resume after dependency: Invalid next resume", {
               nextResumeAfter: this.nextResumeAfter
             });
             return;
           }
-          if (this.nextResumeAfter === "WAIT_FOR_TASK" && message.completions.length > 1) {
-            logger2.error("waiting for single task but got multiple completions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
+            logger.error(
+              "Failed to resume after dependency: Waiting for single task but got multiple completions",
+              {
+                completions
+              }
+            );
             return;
           }
+          switch (this.nextResumeAfter) {
+            case "WAIT_FOR_TASK": {
+              this.waitForTaskReplay = void 0;
+              break;
+            }
+            case "WAIT_FOR_BATCH": {
+              this.waitForBatchReplay = void 0;
+              break;
+            }
+          }
           this.paused = false;
           this.nextResumeAfter = void 0;
-          for (let i = 0; i < message.completions.length; i++) {
-            const completion = message.completions[i];
-            const execution = message.executions[i];
-            if (!completion || !execution)
+          this.waitForPostStart = false;
+          for (let i = 0; i < completions.length; i++) {
+            const completion = completions[i];
+            if (!completion)
               continue;
-            this.#backgroundWorker.taskRunCompletedNotification(completion, execution);
+            this.#backgroundWorker.taskRunCompletedNotification(completion);
           }
         },
         RESUME_AFTER_DURATION: async (message) => {
           if (!this.paused) {
-            logger2.error("worker not paused", {
+            logger.error("worker not paused", {
               attemptId: message.attemptId
             });
             return;
           }
           if (this.nextResumeAfter !== "WAIT_FOR_DURATION") {
-            logger2.error("not waiting to resume after duration", {
+            logger.error("not waiting to resume after duration", {
               nextResumeAfter: this.nextResumeAfter
             });
             return;
           }
+          this.durationResumeFallback = void 0;
           this.#resumeAfterDuration();
         },
+        // Deprecated: This will never get called as this worker supports lazy attempts. It's only here for a quick view of the flow old workers use.
         EXECUTE_TASK_RUN: async ({ executionPayload }) => {
           if (this.executing) {
-            logger2.error("dropping execute request, already executing");
+            logger.error("dropping execute request, already executing");
             return;
           }
           if (this.completed.has(executionPayload.execution.attempt.id)) {
-            logger2.error("dropping execute request, already completed");
+            logger.error("dropping execute request, already completed");
             return;
           }
           this.executing = true;
           this.attemptFriendlyId = executionPayload.execution.attempt.id;
           const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
-          logger2.log("completed", completion);
+          logger.log("completed", completion);
           this.completed.add(executionPayload.execution.attempt.id);
-          await this.#backgroundWorker.flushTelemetry();
           const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
             version: "v1",
             execution: executionPayload.execution,
             completion
           });
-          logger2.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
-          this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+          logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+          await this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+        },
+        EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
+          this.readyForLazyAttemptReplay = void 0;
+          if (this.executing) {
+            logger.error("dropping execute request, already executing");
+            return;
+          }
+          const attemptCount = message.lazyPayload.attemptCount ?? 0;
+          logger.log("execute attempt counts", { attemptCount, completed: this.completed.size });
+          if (this.completed.size > 0 && this.completed.size >= attemptCount + 1) {
+            logger.error("dropping execute request, already completed");
+            return;
+          }
+          this.executing = true;
+          try {
+            const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
+            logger.log("completed", completion);
+            this.completed.add(execution.attempt.id);
+            await this.#submitAttemptCompletion(execution, completion);
+          } catch (error) {
+            logger.error("Failed to complete lazy attempt", {
+              error
+            });
+            this.#failRun(message.lazyPayload.runId, error);
+          }
         },
         REQUEST_ATTEMPT_CANCELLATION: async (message) => {
           if (!this.executing) {
+            logger.log("dropping cancel request, not executing", { status: this.#status });
             return;
           }
+          logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
           await this.#backgroundWorker.cancelAttempt(message.attemptId);
         },
-        REQUEST_EXIT: async () => {
+        REQUEST_EXIT: async (message) => {
+          if (message.version === "v2" && message.delayInMs) {
+            logger.log("exit requested with delay", { delayInMs: message.delayInMs });
+            await timeout2(message.delayInMs);
+          }
           this.#coordinatorSocket.close();
           process.exit(0);
         },
         READY_FOR_RETRY: async (message) => {
           if (this.completed.size < 1) {
+            logger.error("Received READY_FOR_RETRY but no completions yet. This is a bug.");
             return;
           }
-          this.#coordinatorSocket.socket.emit("READY_FOR_EXECUTION", {
-            version: "v1",
-            runId: this.runId,
-            totalCompletions: this.completed.size
-          });
+          this.submitAttemptCompletionReplay = void 0;
+          await this.#readyForLazyAttempt();
         }
       },
-      onConnection: async (socket, handler, sender, logger3) => {
-        if (this.waitForPostStart) {
-          logger3.log("skip connection handler, waiting for post start hook");
-          return;
-        }
-        if (process.env.INDEX_TASKS === "true") {
-          try {
-            const taskResources = await this.#initializeWorker();
-            const { success } = await socket.emitWithAck("INDEX_TASKS", {
+      // MARK: ON CONNECTION
+      onConnection: async (socket, handler, sender, logger2) => {
+        logger2.log("connected to coordinator", {
+          status: this.#status,
+          connectionCount: ++this.connectionCount
+        });
+        socket.emit("SET_STATE", { version: "v1", attemptFriendlyId: this.attemptFriendlyId });
+        try {
+          if (this.waitForPostStart) {
+            logger2.log("skip connection handler, waiting for post start hook");
+            return;
+          }
+          if (this.paused) {
+            if (!this.nextResumeAfter) {
+              logger2.error("Missing next resume reason", { status: this.#status });
+              this.#emitUnrecoverableError(
+                "NoNextResume",
+                "Next resume reason not set while resuming from paused state"
+              );
+              return;
+            }
+            if (!this.attemptFriendlyId) {
+              logger2.error("Missing friendly ID", { status: this.#status });
+              this.#emitUnrecoverableError(
+                "NoAttemptId",
+                "Attempt ID not set while resuming from paused state"
+              );
+              return;
+            }
+            socket.emit("READY_FOR_RESUME", {
               version: "v1",
-              deploymentId: this.deploymentId,
-              ...taskResources
+              attemptFriendlyId: this.attemptFriendlyId,
+              type: this.nextResumeAfter
             });
-            if (success) {
-              logger3.info("indexing done, shutting down..");
-              process.exit(0);
-            } else {
-              logger3.info("indexing failure, shutting down..");
-              process.exit(1);
-            }
-          } catch (e) {
-            if (e instanceof TaskMetadataParseError) {
-              logger3.error("tasks metadata parse error", { message: e.zodIssues, tasks: e.tasks });
+            return;
+          }
+          if (process.env.INDEX_TASKS === "true") {
+            const failIndex = (error) => {
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
+                error
+              });
+            };
+            process.removeAllListeners("uncaughtException");
+            process.on("uncaughtException", (error) => {
+              console.error("Uncaught exception while indexing", error);
+              failIndex(error);
+            });
+            try {
+              const taskResources = await this.#initializeWorker();
+              const indexTasks = await defaultBackoff.maxRetries(3).execute(async () => {
+                return await socket.timeout(2e4).emitWithAck("INDEX_TASKS", {
+                  version: "v2",
+                  deploymentId: this.deploymentId,
+                  ...taskResources,
+                  supportsLazyAttempts: true
+                });
+              });
+              if (!indexTasks.success || !indexTasks.result.success) {
+                logger2.error("indexing failure, shutting down..", { indexTasks });
+                process.exit(1);
+              } else {
+                logger2.info("indexing done, shutting down..");
+                process.exit(0);
+              }
+            } catch (e) {
+              const stderr = this.#backgroundWorker.stderr.join("\n");
+              if (e instanceof TaskMetadataParseError) {
+                logger2.error("tasks metadata parse error", {
+                  zodIssues: e.zodIssues,
+                  tasks: e.tasks
+                });
+                failIndex({
                   name: "TaskMetadataParseError",
                   message: "There was an error parsing the task metadata",
-                  stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks })
-                }
-              });
-            } else if (e instanceof UncaughtExceptionError) {
-              logger3.error("uncaught exception", { message: e.originalError.message });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                  stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
+                  stderr
+                });
+              } else if (e instanceof UncaughtExceptionError) {
+                const error = {
                   name: e.originalError.name,
                   message: e.originalError.message,
-                  stack: e.originalError.stack
-                }
-              });
-            } else if (e instanceof Error) {
-              logger3.error("error", { message: e.message });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                  stack: e.originalError.stack,
+                  stderr
+                };
+                logger2.error("uncaught exception", { originalError: error });
+                failIndex(error);
+              } else if (e instanceof Error) {
+                const error = {
                   name: e.name,
                   message: e.message,
-                  stack: e.stack
-                }
-              });
-            } else if (typeof e === "string") {
-              logger3.error("string error", { message: e });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                  stack: e.stack,
+                  stderr
+                };
+                logger2.error("error", { error });
+                failIndex(error);
+              } else if (typeof e === "string") {
+                logger2.error("string error", { error: { message: e } });
+                failIndex({
                   name: "Error",
-                  message: e
-                }
-              });
-            } else {
-              logger3.error("unknown error", { error: e });
-              socket.emit("INDEXING_FAILED", {
-                version: "v1",
-                deploymentId: this.deploymentId,
-                error: {
+                  message: e,
+                  stderr
+                });
+              } else {
+                logger2.error("unknown error", { error: e });
+                failIndex({
                   name: "Error",
-                  message: "Unknown error"
-                }
-              });
+                  message: "Unknown error",
+                  stderr
+                });
+              }
+              await timeout2(1e3);
+              process.exit(EXIT_CODE_ALREADY_HANDLED);
             }
-            await setTimeout2(200);
-            process.exit(1);
           }
-        }
-        if (this.paused) {
-          if (!this.nextResumeAfter) {
-            return;
-          }
-          if (!this.attemptFriendlyId) {
-            logger3.error("Missing friendly ID");
+          if (this.executing) {
             return;
           }
-          if (this.nextResumeAfter === "WAIT_FOR_DURATION") {
-            this.#resumeAfterDuration();
+          process.removeAllListeners("uncaughtException");
+          process.on("uncaughtException", (error) => {
+            console.error("Uncaught exception during run", error);
+            this.#failRun(this.runId, error);
+          });
+          await this.#readyForLazyAttempt();
+        } catch (error) {
+          logger2.error("connection handler error", { error });
+        } finally {
+          if (this.connectionCount === 1) {
             return;
           }
-          socket.emit("READY_FOR_RESUME", {
-            version: "v1",
-            attemptFriendlyId: this.attemptFriendlyId,
-            type: this.nextResumeAfter
-          });
-          return;
-        }
-        if (this.executing) {
-          return;
+          this.#handleReplays();
         }
-        socket.emit("READY_FOR_EXECUTION", {
-          version: "v1",
-          runId: this.runId,
-          totalCompletions: this.completed.size
-        });
       },
-      onError: async (socket, err, logger3) => {
-        logger3.error("onError", {
+      onError: async (socket, err, logger2) => {
+        logger2.error("onError", {
           error: {
             name: err.name,
             message: err.message
           }
         });
-        await this.#reconnect();
-      },
-      onDisconnect: async (socket, reason, description, logger3) => {
       }
     });
     return coordinatorConnection;
   }
+  // MARK: REPLAYS
+  async #handleReplays() {
+    const backoff = new ExponentialBackoff().type("FullJitter").maxRetries(3);
+    const replayCancellationDelay = 2e4;
+    if (this.waitForTaskReplay) {
+      logger.log("replaying wait for task", { ...this.waitForTaskReplay });
+      const { idempotencyKey, message, attempt } = this.waitForTaskReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.waitForTaskReplay) {
+        logger.error("wait for task replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
+        logger.error("wait for task replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.waitForTaskReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#waitForTaskHandler(message);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("wait for task replay retry limit exceeded", { error });
+        } else {
+          logger.error("wait for task replay error", { error });
+        }
+      }
+      return;
+    }
+    if (this.waitForBatchReplay) {
+      logger.log("replaying wait for batch", {
+        ...this.waitForBatchReplay,
+        cancellationDelay: replayCancellationDelay
+      });
+      const { idempotencyKey, message, attempt } = this.waitForBatchReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.waitForBatchReplay) {
+        logger.error("wait for batch replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
+        logger.error("wait for batch replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.waitForBatchReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#waitForBatchHandler(message);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("wait for batch replay retry limit exceeded", { error });
+        } else {
+          logger.error("wait for batch replay error", { error });
+        }
+      }
+      return;
+    }
+    if (this.submitAttemptCompletionReplay) {
+      logger.log("replaying attempt completion", {
+        ...this.submitAttemptCompletionReplay,
+        cancellationDelay: replayCancellationDelay
+      });
+      const { idempotencyKey, message, attempt } = this.submitAttemptCompletionReplay;
+      await timeout2(replayCancellationDelay);
+      if (!this.submitAttemptCompletionReplay) {
+        logger.error("attempt completion replay cancelled, discarding", {
+          originalMessage: { idempotencyKey, message, attempt }
+        });
+        return;
+      }
+      if (idempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
+        logger.error("attempt completion replay idempotency key mismatch, discarding", {
+          originalMessage: { idempotencyKey, message, attempt },
+          newMessage: this.submitAttemptCompletionReplay
+        });
+        return;
+      }
+      try {
+        await backoff.wait(attempt + 1);
+        await this.#submitAttemptCompletion(message.execution, message.completion, idempotencyKey);
+      } catch (error) {
+        if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
+          logger.error("attempt completion replay retry limit exceeded", { error });
+        } else {
+          logger.error("attempt completion replay error", { error });
+        }
+      }
+      return;
+    }
+  }
+  // MARK: HTTP SERVER
   #createHttpServer() {
     const httpServer = createServer(async (req, res) => {
-      logger2.log(`[${req.method}]`, req.url);
+      logger.log(`[${req.method}]`, req.url);
       const reply = new HttpReply(res);
       try {
         const url = new URL(req.url ?? "", `http://${req.headers.host}`);
@@ -998,35 +1999,27 @@ var ProdWorker = class {
             return reply.text("ok");
           }
           case "/status": {
-            return reply.json({
-              executing: this.executing,
-              pause: this.paused,
-              nextResumeAfter: this.nextResumeAfter
-            });
+            return reply.json(this.#status);
           }
           case "/connect": {
             this.#coordinatorSocket.connect();
             return reply.text("Connected to coordinator");
           }
           case "/close": {
-            await this.#coordinatorSocket.sendWithAck("LOG", {
-              version: "v1",
-              text: `[${req.method}] ${req.url}`
-            });
             this.#coordinatorSocket.close();
+            this.connectionCount = 0;
             return reply.text("Disconnected from coordinator");
           }
           case "/test": {
-            await this.#coordinatorSocket.sendWithAck("LOG", {
-              version: "v1",
-              text: `[${req.method}] ${req.url}`
+            await this.#coordinatorSocket.socket.timeout(1e4).emitWithAck("TEST", {
+              version: "v1"
             });
             return reply.text("Received ACK from coordinator");
           }
           case "/preStop": {
             const cause = PreStopCauses.safeParse(url.searchParams.get("cause"));
             if (!cause.success) {
-              logger2.error("Failed to parse cause", { cause });
+              logger.error("Failed to parse cause", { cause });
               return reply.text("Failed to parse cause", 400);
             }
             switch (cause.data) {
@@ -1034,17 +2027,16 @@ var ProdWorker = class {
                 break;
               }
               default: {
-                logger2.error("Unhandled cause", { cause: cause.data });
+                logger.error("Unhandled cause", { cause: cause.data });
                 break;
               }
             }
-            logger2.log("preStop", { url: req.url });
             return reply.text("preStop ok");
           }
           case "/postStart": {
             const cause = PostStartCauses.safeParse(url.searchParams.get("cause"));
             if (!cause.success) {
-              logger2.error("Failed to parse cause", { cause });
+              logger.error("Failed to parse cause", { cause });
               return reply.text("Failed to parse cause", 400);
             }
             switch (cause.data) {
@@ -1055,11 +2047,11 @@ var ProdWorker = class {
                 break;
               }
               case "restore": {
-                await this.#reconnect(true);
+                await this.#reconnectAfterPostStart();
                 break;
               }
               default: {
-                logger2.error("Unhandled cause", { cause: cause.data });
+                logger.error("Unhandled cause", { cause: cause.data });
                 break;
               }
             }
@@ -1070,7 +2062,7 @@ var ProdWorker = class {
           }
         }
       } catch (error) {
-        logger2.error("HTTP server error", { error });
+        logger.error("HTTP server error", { error });
         reply.empty(500);
       }
     });
@@ -1078,15 +2070,15 @@ var ProdWorker = class {
       socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
     });
     httpServer.on("listening", () => {
-      logger2.log("http server listening on port", this.#httpPort);
+      logger.log("http server listening on port", this.#httpPort);
     });
     httpServer.on("error", async (error) => {
       if (error.code != "EADDRINUSE") {
         return;
       }
-      logger2.error(`port ${this.#httpPort} already in use, retrying with random port..`);
+      logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
       this.#httpPort = getRandomPortNumber();
-      await setTimeout2(100);
+      await timeout2(100);
       this.start();
     });
     return httpServer;
@@ -1096,8 +2088,12 @@ var ProdWorker = class {
     await this.#backgroundWorker.initialize({ env: envVars });
     let packageVersion;
     const taskResources = [];
-    if (!this.#backgroundWorker.tasks) {
-      throw new Error(`Background Worker started without tasks`);
+    if (!this.#backgroundWorker.tasks || this.#backgroundWorker.tasks.length === 0) {
+      throw new Error(
+        `Background Worker started without tasks. Searched in: ${__PROJECT_CONFIG__.triggerDirectories?.join(
+          ", "
+        )}`
+      );
     }
     for (const task of this.#backgroundWorker.tasks) {
       taskResources.push(task);
@@ -1124,11 +2120,32 @@ var ProdWorker = class {
     const data = await response.json();
     return data?.variables ?? {};
   }
+  get #status() {
+    return {
+      executing: this.executing,
+      paused: this.paused,
+      completed: this.completed.size,
+      nextResumeAfter: this.nextResumeAfter,
+      waitForPostStart: this.waitForPostStart,
+      attemptFriendlyId: this.attemptFriendlyId,
+      waitForTaskReplay: this.waitForTaskReplay,
+      waitForBatchReplay: this.waitForBatchReplay
+    };
+  }
+  #emitUnrecoverableError(name, message) {
+    this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
+      version: "v1",
+      error: {
+        name,
+        message
+      }
+    });
+  }
   start() {
     this.#httpServer.listen(this.#httpPort, this.host);
   }
 };
-var prodWorker = new ProdWorker(HTTP_SERVER_PORT2);
+var prodWorker = new ProdWorker(HTTP_SERVER_PORT);
 prodWorker.start();
 function gatherProcessEnv() {
   const env = {
@@ -1139,7 +2156,8 @@ function gatherProcessEnv() {
     LANG: process.env.LANG,
     TERM: process.env.TERM,
     NODE_PATH: process.env.NODE_PATH,
-    HOME: process.env.HOME
+    HOME: process.env.HOME,
+    NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS
   };
   return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== void 0));
 }