trigger.dev 3.0.0-beta.4 → 3.0.0-beta.40

package/dist/workers/prod/entry-point.js

@@ -3,9 +3,9 @@ import {
   CoordinatorToProdWorkerMessages,
   PostStartCauses,
   PreStopCauses,
-  ProdWorkerToCoordinatorMessages,
-  ZodSocketConnection as ZodSocketConnection2
+  ProdWorkerToCoordinatorMessages
 } from "@trigger.dev/core/v3";
+import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 
 // ../core-apps/src/http.ts
 var HttpReply = class {
@@ -65,24 +65,6 @@ var SimpleLogger = class {
   }
 };
 
-// ../core-apps/src/provider.ts
-import {
-  ClientToSharedQueueMessages,
-  clientWebsocketMessages,
-  PlatformToProviderMessages,
-  ProviderToPlatformMessages,
-  SharedQueueToClientMessages,
-  ZodMessageSender,
-  ZodSocketConnection
-} from "@trigger.dev/core/v3";
-var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
-var MACHINE_NAME = process.env.MACHINE_NAME || "local";
-var PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
-var PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
-var PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
-var SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
-var logger = new SimpleLogger(`[${MACHINE_NAME}]`);
-
 // src/workers/prod/entry-point.ts
 import { readFile } from "node:fs/promises";
 import { createServer } from "node:http";
@@ -93,9 +75,9 @@ import {
   ProdWorkerToChildMessages,
   SemanticInternalAttributes,
   TaskRunErrorCodes,
-  ZodIpcConnection,
   correctErrorStackTrace
 } from "@trigger.dev/core/v3";
+import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { Evt } from "evt";
 import { fork } from "node:child_process";
 
@@ -116,8 +98,6 @@ var TaskMetadataParseError = class extends Error {
     this.name = "TaskMetadataParseError";
   }
 };
-
-// src/workers/prod/backgroundWorker.ts
 var UnexpectedExitError = class extends Error {
   constructor(code) {
     super(`Unexpected exit with code ${code}`);
@@ -137,30 +117,73 @@ var CancelledProcessError = class extends Error {
     this.name = "CancelledProcessError";
   }
 };
+var SigKillTimeoutProcessError = class extends Error {
+  constructor() {
+    super("Process kill timeout");
+    this.name = "SigKillTimeoutProcessError";
+  }
+};
+var GracefulExitTimeoutError = class extends Error {
+  constructor() {
+    super("Graceful exit timeout");
+    this.name = "GracefulExitTimeoutError";
+  }
+};
+
+// src/workers/prod/backgroundWorker.ts
 var ProdBackgroundWorker = class {
   constructor(path, params) {
     this.path = path;
     this.params = params;
   }
   _initialized = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
   preCheckpointNotification = Evt.create();
+  checkpointCanceledNotification = Evt.create();
   onReadyForCheckpoint = Evt.create();
   onCancelCheckpoint = Evt.create();
+  onCreateTaskRunAttempt = Evt.create();
+  attemptCreatedNotification = Evt.create();
   _onClose = new Evt();
   tasks = [];
+  stderr = [];
   _taskRunProcess;
+  _taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
   _closed = false;
-  async close() {
+  async close(gracefulExitTimeoutElapsed = false) {
+    console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
     if (this._closed) {
       return;
     }
     this._closed = true;
     this.onTaskHeartbeat.detach();
-    await this._taskRunProcess?.cleanup(true);
+    this.onTaskRunHeartbeat.detach();
+    await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
+  }
+  async #killTaskRunProcess(flush = true, initialSignal = "SIGTERM") {
+    console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
+    if (this._closed || !this._taskRunProcess) {
+      return;
+    }
+    if (flush) {
+      await this.flushTelemetry();
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
+      console.error("Error while trying graceful exit", error);
+    });
+    console.log("Killed task run process, setting closed to true", {
+      closed: this._closed,
+      pid: currentTaskRunProcess.pid
+    });
+    this._closed = true;
   }
   async flushTelemetry() {
     await this._taskRunProcess?.cleanup(false);
@@ -194,6 +217,20 @@ var ProdBackgroundWorker = class {
         child.kill();
         reject(new Error("Worker timed out"));
       }, 1e4);
+      child.stdout?.on("data", (data) => {
+        console.log(data.toString());
+      });
+      child.stderr?.on("data", (data) => {
+        console.error(data.toString());
+        this.stderr.push(data.toString());
+      });
+      child.on("exit", (code) => {
+        if (!resolved) {
+          clearTimeout(timeout);
+          resolved = true;
+          reject(new Error(`Worker exited with code ${code}`));
+        }
+      });
       new ZodIpcConnection({
         listenSchema: ProdChildToWorkerMessages,
         emitSchema: ProdWorkerToChildMessages,
@@ -225,19 +262,6 @@ var ProdBackgroundWorker = class {
           }
         }
       });
-      child.stdout?.on("data", (data) => {
-        console.log(data.toString());
-      });
-      child.stderr?.on("data", (data) => {
-        console.error(data.toString());
-      });
-      child.on("exit", (code) => {
-        if (!resolved) {
-          clearTimeout(timeout);
-          resolved = true;
-          reject(new Error(`Worker exited with code ${code}`));
-        }
-      });
     });
     this._initialized = true;
   }
@@ -250,63 +274,145 @@ var ProdBackgroundWorker = class {
   }
   // We need to notify all the task run processes that a task run has completed,
   // in case they are waiting for it through triggerAndWait
-  async taskRunCompletedNotification(completion, execution) {
-    this._taskRunProcess?.taskRunCompletedNotification(completion, execution);
+  async taskRunCompletedNotification(completion) {
+    this._taskRunProcess?.taskRunCompletedNotification(completion);
   }
   async waitCompletedNotification() {
     this._taskRunProcess?.waitCompletedNotification();
   }
-  async #initializeTaskRunProcess(payload) {
+  async #getFreshTaskRunProcess(payload, messageId) {
     const metadata = this.getMetadata(
       payload.execution.worker.id,
       payload.execution.worker.version
     );
-    if (!this._taskRunProcess) {
-      const taskRunProcess = new TaskRunProcess(
-        payload.execution,
-        this.path,
-        {
-          ...this.params.env,
-          ...payload.environment ?? {}
-        },
-        metadata,
-        this.params
-      );
-      taskRunProcess.onExit.attach(() => {
+    console.log("Getting fresh task run process, setting closed to false", {
+      closed: this._closed
+    });
+    this._closed = false;
+    await this.#killCurrentTaskRunProcessBeforeAttempt();
+    const taskRunProcess = new TaskRunProcess(
+      payload.execution.run.id,
+      payload.execution.run.isTest,
+      this.path,
+      {
+        ...this.params.env,
+        ...payload.environment ?? {}
+      },
+      metadata,
+      this.params,
+      messageId
+    );
+    taskRunProcess.onExit.attach(({ pid }) => {
+      console.log("Task run process exited", { pid });
+      if (this._taskRunProcess?.pid === pid) {
         this._taskRunProcess = void 0;
-      });
-      taskRunProcess.onTaskHeartbeat.attach((id) => {
-        this.onTaskHeartbeat.post(id);
-      });
-      taskRunProcess.onWaitForBatch.attach((message) => {
-        this.onWaitForBatch.post(message);
-      });
-      taskRunProcess.onWaitForDuration.attach((message) => {
-        this.onWaitForDuration.post(message);
-      });
-      taskRunProcess.onWaitForTask.attach((message) => {
-        this.onWaitForTask.post(message);
-      });
-      taskRunProcess.onReadyForCheckpoint.attach((message) => {
-        this.onReadyForCheckpoint.post(message);
-      });
-      taskRunProcess.onCancelCheckpoint.attach((message) => {
-        this.onCancelCheckpoint.post(message);
-      });
-      this.preCheckpointNotification.attach((message) => {
-        taskRunProcess.preCheckpointNotification.post(message);
-      });
-      await taskRunProcess.initialize();
-      this._taskRunProcess = taskRunProcess;
-    }
+      }
+      if (pid) {
+        this._taskRunProcessesBeingKilled.delete(pid);
+      }
+    });
+    taskRunProcess.onIsBeingKilled.attach((taskRunProcess2) => {
+      if (taskRunProcess2?.pid) {
+        this._taskRunProcessesBeingKilled.set(taskRunProcess2.pid, taskRunProcess2);
+      }
+    });
+    taskRunProcess.onTaskHeartbeat.attach((id) => {
+      this.onTaskHeartbeat.post(id);
+    });
+    taskRunProcess.onTaskRunHeartbeat.attach((id) => {
+      this.onTaskRunHeartbeat.post(id);
+    });
+    taskRunProcess.onWaitForBatch.attach((message) => {
+      this.onWaitForBatch.post(message);
+    });
+    taskRunProcess.onWaitForDuration.attach((message) => {
+      this.onWaitForDuration.post(message);
+    });
+    taskRunProcess.onWaitForTask.attach((message) => {
+      this.onWaitForTask.post(message);
+    });
+    taskRunProcess.onReadyForCheckpoint.attach((message) => {
+      this.onReadyForCheckpoint.post(message);
+    });
+    taskRunProcess.onCancelCheckpoint.attach((message) => {
+      this.onCancelCheckpoint.post(message);
+    });
+    this.preCheckpointNotification.attach((message) => {
+      taskRunProcess.preCheckpointNotification.post(message);
+    });
+    this.checkpointCanceledNotification.attach((message) => {
+      taskRunProcess.checkpointCanceledNotification.post(message);
+    });
+    await taskRunProcess.initialize();
+    this._taskRunProcess = taskRunProcess;
     return this._taskRunProcess;
   }
-  // We need to fork the process before we can execute any tasks
-  async executeTaskRun(payload) {
+  async forceKillOldTaskRunProcesses() {
+    for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
+      try {
+        await taskRunProcess.kill("SIGKILL");
+      } catch (error) {
+        console.error("Error while force killing old task run processes", error);
+      }
+    }
+  }
+  async #killCurrentTaskRunProcessBeforeAttempt() {
+    console.log("killCurrentTaskRunProcessBeforeAttempt()", {
+      hasTaskRunProcess: !!this._taskRunProcess
+    });
+    if (!this._taskRunProcess) {
+      return;
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    console.log("Killing current task run process", {
+      isBeingKilled: currentTaskRunProcess?.isBeingKilled,
+      totalBeingKilled: this._taskRunProcessesBeingKilled.size
+    });
+    if (currentTaskRunProcess.isBeingKilled) {
+      if (this._taskRunProcessesBeingKilled.size > 1) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+      }
+    } else {
+      if (this._taskRunProcessesBeingKilled.size > 0) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+        currentTaskRunProcess.kill("SIGTERM", 5e3).catch(() => {
+        });
+      }
+    }
+  }
+  async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
     try {
-      const taskRunProcess = await this.#initializeTaskRunProcess(payload);
+      const initialExit = taskRunProcess.onExit.waitFor(5e3);
+      if (kill) {
+        taskRunProcess.kill(initialSignal);
+      }
+      await initialExit;
+    } catch (error) {
+      console.error("TaskRunProcess graceful kill timeout exceeded", error);
+      this.#tryForcefulExit(taskRunProcess);
+    }
+  }
+  async #tryForcefulExit(taskRunProcess) {
+    try {
+      const forcedKill = taskRunProcess.onExit.waitFor(5e3);
+      taskRunProcess.kill("SIGKILL");
+      await forcedKill;
+    } catch (error) {
+      console.error("TaskRunProcess forced kill timeout exceeded", error);
+      throw new SigKillTimeoutProcessError();
+    }
+  }
+  // We need to fork the process before we can execute any tasks, use a fresh process for each execution
+  async executeTaskRun(payload, messageId) {
+    try {
+      const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
+      console.log("executing task run", {
+        attempt: payload.execution.attempt.id,
+        taskRunPid: taskRunProcess.pid
+      });
       const result = await taskRunProcess.executeTaskRun(payload);
-      await taskRunProcess.cleanup(result.ok || result.retry === void 0);
       if (result.ok) {
         return result;
       }
@@ -353,6 +459,29 @@ var ProdBackgroundWorker = class {
           }
         };
       }
+      if (e instanceof SigKillTimeoutProcessError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT
+          }
+        };
+      }
+      if (e instanceof GracefulExitTimeoutError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
+            message: "Worker process killed while attempt in progress."
+          }
+        };
+      }
       return {
         id: payload.execution.attempt.id,
         ok: false,
@@ -362,10 +491,41 @@ var ProdBackgroundWorker = class {
           code: TaskRunErrorCodes.TASK_EXECUTION_FAILED
         }
       };
+    } finally {
+      await this.#killTaskRunProcess();
     }
   }
   async cancelAttempt(attemptId) {
-    await this._taskRunProcess?.cancel();
+    if (!this._taskRunProcess) {
+      console.error("No task run process to cancel attempt", { attemptId });
+      return;
+    }
+    await this._taskRunProcess.cancel();
+  }
+  async executeTaskRunLazyAttempt(payload) {
+    this.onCreateTaskRunAttempt.post({ runId: payload.runId });
+    let execution;
+    try {
+      const attemptCreated = await this.attemptCreatedNotification.waitFor(3e4);
+      if (!attemptCreated.success) {
+        throw new Error(
+          `Failed to create attempt${attemptCreated.reason ? `: ${attemptCreated.reason}` : ""}`
+        );
+      }
+      execution = attemptCreated.execution;
+    } catch (error) {
+      console.error("Error while creating attempt", error);
+      throw new Error(`Failed to create task run attempt: ${error}`);
+    }
+    const completion = await this.executeTaskRun(
+      {
+        execution,
+        traceContext: payload.traceContext,
+        environment: payload.environment
+      },
+      payload.messageId
+    );
+    return { execution, completion };
   }
   async #correctError(error, execution) {
     return {
@@ -375,26 +535,36 @@ var ProdBackgroundWorker = class {
   }
 };
 var TaskRunProcess = class {
-  constructor(execution, path, env, metadata, worker) {
-    this.execution = execution;
+  constructor(runId, isTest, path, env, metadata, worker, messageId) {
+    this.runId = runId;
+    this.isTest = isTest;
     this.path = path;
     this.env = env;
     this.metadata = metadata;
     this.worker = worker;
+    this.messageId = messageId;
   }
   _ipc;
   _child;
+  _childPid;
   _attemptPromises = /* @__PURE__ */ new Map();
   _attemptStatuses = /* @__PURE__ */ new Map();
   _currentExecution;
   _isBeingKilled = false;
   _isBeingCancelled = false;
+  _gracefulExitTimeoutElapsed = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onExit = new Evt();
+  onIsBeingKilled = new Evt();
   onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
   preCheckpointNotification = Evt.create();
+  checkpointCanceledNotification = Evt.create();
   onReadyForCheckpoint = Evt.create();
   onCancelCheckpoint = Evt.create();
   async initialize() {
@@ -409,7 +579,7 @@ var TaskRunProcess = class {
         "ipc"
       ],
       env: {
-        ...this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
+        ...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
         ...this.env,
         OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
           [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
@@ -417,6 +587,7 @@ var TaskRunProcess = class {
         ...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
       }
     });
+    this._childPid = this._child?.pid;
     this._ipc = new ZodIpcConnection({
       listenSchema: ProdChildToWorkerMessages,
       emitSchema: ProdWorkerToChildMessages,
@@ -437,28 +608,60 @@ var TaskRunProcess = class {
           resolver(result);
         },
         READY_TO_DISPOSE: async (message) => {
+          process.exit(0);
         },
         TASK_HEARTBEAT: async (message) => {
-          this.onTaskHeartbeat.post(message.id);
+          if (this.messageId) {
+            this.onTaskRunHeartbeat.post(this.messageId);
+          } else {
+            this.onTaskHeartbeat.post(message.id);
+          }
         },
         TASKS_READY: async (message) => {
         },
+        WAIT_FOR_TASK: async (message) => {
+          this.onWaitForTask.post(message);
+        },
         WAIT_FOR_BATCH: async (message) => {
           this.onWaitForBatch.post(message);
         },
         WAIT_FOR_DURATION: async (message) => {
           this.onWaitForDuration.post(message);
-          const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor();
-          return { willCheckpointAndRestore };
-        },
-        WAIT_FOR_TASK: async (message) => {
-          this.onWaitForTask.post(message);
+          try {
+            const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor(
+              3e4
+            );
+            return {
+              willCheckpointAndRestore
+            };
+          } catch (error) {
+            console.error("Error while waiting for pre-checkpoint notification", error);
+            return {
+              willCheckpointAndRestore: false
+            };
+          }
         },
         READY_FOR_CHECKPOINT: async (message) => {
           this.onReadyForCheckpoint.post(message);
         },
         CANCEL_CHECKPOINT: async (message) => {
+          const version = "v2";
           this.onCancelCheckpoint.post(message);
+          try {
+            const { checkpointCanceled } = await this.checkpointCanceledNotification.waitFor(
+              3e4
+            );
+            return {
+              version,
+              checkpointCanceled
+            };
+          } catch (error) {
+            console.error("Error while waiting for checkpoint cancellation", error);
+            return {
+              version,
+              checkpointCanceled: true
+            };
+          }
         }
       }
     });
@@ -470,15 +673,36 @@ var TaskRunProcess = class {
     this._isBeingCancelled = true;
     await this.cleanup(true);
   }
-  async cleanup(kill = false) {
+  async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
+    console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
     if (kill && this._isBeingKilled) {
       return;
     }
-    this._isBeingKilled = kill;
-    await this._ipc?.sendWithAck("CLEANUP", {
-      flush: true,
-      kill
+    if (kill) {
+      this._isBeingKilled = true;
+      this.onIsBeingKilled.post(this);
+    }
+    const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
+    const killParentProcess = kill && !killChildProcess;
+    console.log("Cleaning up task run process", {
+      killChildProcess,
+      killParentProcess,
+      ipc: this._ipc,
+      childPid: this._childPid,
+      realChildPid: this._child?.pid
     });
+    await this._ipc?.sendWithAck(
+      "CLEANUP",
+      {
+        flush: true,
+        kill: killParentProcess
+      },
+      3e4
+    );
+    if (killChildProcess) {
+      this._gracefulExitTimeoutElapsed = true;
+      await this.kill("SIGKILL");
+    }
   }
   async executeTaskRun(payload) {
     let resolver;
@@ -502,14 +726,14 @@ var TaskRunProcess = class {
     this._currentExecution = void 0;
     return result;
   }
-  taskRunCompletedNotification(completion, execution) {
+  taskRunCompletedNotification(completion) {
     if (!completion.ok && typeof completion.retry !== "undefined") {
       return;
     }
     if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
       this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
-        completion,
-        execution
+        version: "v2",
+        completion
       });
     }
   }
@@ -518,9 +742,11 @@ var TaskRunProcess = class {
       this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
     }
   }
-  async #handleExit(code) {
+  async #handleExit(code, signal) {
+    console.log("handling child exit", { code, signal });
     for (const [id, status] of this._attemptStatuses.entries()) {
       if (status === "PENDING") {
+        console.log("found pending attempt", { id });
         this._attemptStatuses.set(id, "REJECTED");
         const attemptPromise = this._attemptPromises.get(id);
         if (!attemptPromise) {
@@ -529,56 +755,136 @@ var TaskRunProcess = class {
         const { rejecter } = attemptPromise;
         if (this._isBeingCancelled) {
           rejecter(new CancelledProcessError());
+        } else if (this._gracefulExitTimeoutElapsed) {
+          rejecter(new GracefulExitTimeoutError());
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code));
+          rejecter(new UnexpectedExitError(code ?? -1));
         }
       }
     }
-    this.onExit.post(code);
+    this.onExit.post({ code, signal, pid: this.pid });
   }
   #handleLog(data) {
-    if (!this._currentExecution) {
-      return;
-    }
-    console.log(
-      `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number}] ${data.toString()}`
-    );
+    console.log(data.toString());
   }
   #handleStdErr(data) {
-    if (this._isBeingKilled) {
-      return;
-    }
-    if (!this._currentExecution) {
-      console.error(`[${this.metadata.version}] ${data.toString()}`);
-      return;
-    }
-    console.error(
-      `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number}] ${data.toString()}`
-    );
+    console.error(data.toString());
   }
-  #kill() {
-    if (this._child && !this._child.killed) {
-      this._child?.kill();
+  async kill(signal, timeoutInMs) {
+    this._isBeingKilled = true;
+    const killTimeout = this.onExit.waitFor(timeoutInMs);
+    this.onIsBeingKilled.post(this);
+    this._child?.kill(signal);
+    if (timeoutInMs) {
+      await killTimeout;
     }
   }
+  get isBeingKilled() {
+    return this._isBeingKilled || this._child?.killed;
+  }
+  get pid() {
+    return this._childPid;
+  }
 };
 
 // src/workers/prod/entry-point.ts
 import { setTimeout as setTimeout2 } from "node:timers/promises";
-var HTTP_SERVER_PORT2 = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
+var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
 var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
 var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
-var MACHINE_NAME2 = process.env.MACHINE_NAME || "local";
+var MACHINE_NAME = process.env.MACHINE_NAME || "local";
 var POD_NAME = process.env.POD_NAME || "some-pod";
 var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
-var logger2 = new SimpleLogger(`[${MACHINE_NAME2}][${SHORT_HASH}]`);
+var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
 var ProdWorker = class {
   constructor(port, host = "0.0.0.0") {
     this.host = host;
+    process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
     this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
-    this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
+    this.#backgroundWorker = this.#createBackgroundWorker();
+    this.#httpPort = port;
+    this.#httpServer = this.#createHttpServer();
+  }
+  apiUrl = process.env.TRIGGER_API_URL;
+  apiKey = process.env.TRIGGER_SECRET_KEY;
+  contentHash = process.env.TRIGGER_CONTENT_HASH;
+  projectRef = process.env.TRIGGER_PROJECT_REF;
+  envId = process.env.TRIGGER_ENV_ID;
+  runId = process.env.TRIGGER_RUN_ID || "index-only";
+  deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
+  deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
+  runningInKubernetes = !!process.env.KUBERNETES_PORT;
+  executing = false;
+  completed = /* @__PURE__ */ new Set();
+  paused = false;
+  attemptFriendlyId;
+  nextResumeAfter;
+  waitForPostStart = false;
+  #httpPort;
+  #backgroundWorker;
+  #httpServer;
+  #coordinatorSocket;
+  async #handleSignal(signal) {
+    logger.log("Received signal", { signal });
+    if (signal === "SIGTERM") {
+      let gracefulExitTimeoutElapsed = false;
+      if (this.executing) {
+        const terminationGracePeriodSeconds = 60 * 60;
+        logger.log("Waiting for attempt to complete before exiting", {
+          terminationGracePeriodSeconds
+        });
+        await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
+        gracefulExitTimeoutElapsed = true;
+        logger.log("Termination timeout reached, exiting gracefully.");
+      } else {
+        logger.log("Not executing, exiting immediately.");
+      }
+      await this.#exitGracefully(gracefulExitTimeoutElapsed);
+      return;
+    }
+    logger.log("Unhandled signal", { signal });
+  }
+  async #exitGracefully(gracefulExitTimeoutElapsed = false) {
+    await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
+    if (!gracefulExitTimeoutElapsed) {
+      process.exit(0);
+    }
+  }
+  async #reconnect(isPostStart = false, reconnectImmediately = false) {
+    if (isPostStart) {
+      this.waitForPostStart = false;
+    }
+    this.#coordinatorSocket.close();
+    if (!reconnectImmediately) {
+      await setTimeout2(1e3);
+    }
+    let coordinatorHost = COORDINATOR_HOST;
+    try {
+      if (this.runningInKubernetes) {
+        coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
+          "\n",
+          ""
+        );
+        logger.log("reconnecting", {
+          coordinatorHost: {
+            fromEnv: COORDINATOR_HOST,
+            fromVolume: coordinatorHost,
+            current: this.#coordinatorSocket.socket.io.opts.hostname
+          }
+        });
+      }
+    } catch (error) {
+      logger.error("taskinfo read error during reconnect", {
+        error: error instanceof Error ? error.message : error
+      });
+    } finally {
+      this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
+    }
+  }
+  #createBackgroundWorker() {
+    const backgroundWorker = new ProdBackgroundWorker("worker.js", {
       projectConfig: __PROJECT_CONFIG__,
       env: {
         ...gatherProcessEnv(),
@@ -588,26 +894,69 @@ var ProdWorker = class {
       },
       contentHash: this.contentHash
     });
-    this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
+    backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
       this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
     });
-    this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
+    backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
+      this.#coordinatorSocket.socket.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
+    });
+    backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
+      await this.#prepareForCheckpoint();
       this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
     });
-    this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
-      logger2.log("onCancelCheckpoint() clearing paused state, don't wait for post start hook", {
-        paused: this.paused,
-        nextResumeAfter: this.nextResumeAfter,
-        waitForPostStart: this.waitForPostStart
+    backgroundWorker.onCancelCheckpoint.attach(async (message) => {
+      logger.log("onCancelCheckpoint", { message });
+      const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
+        "CANCEL_CHECKPOINT",
+        {
+          version: "v2",
+          reason: message.reason
+        }
+      );
+      logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
+      if (checkpointCanceled) {
+        if (message.reason === "WAIT_FOR_DURATION") {
+          this.paused = false;
+          this.nextResumeAfter = void 0;
+          this.waitForPostStart = false;
+        }
+      }
+      backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
+    });
+    backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
+      logger.log("onCreateTaskRunAttempt()", { message });
+      const createAttempt = await this.#coordinatorSocket.socket.emitWithAck(
+        "CREATE_TASK_RUN_ATTEMPT",
+        {
+          version: "v1",
+          runId: message.runId
+        }
+      );
+      if (!createAttempt.success) {
+        backgroundWorker.attemptCreatedNotification.post({
+          success: false,
+          reason: createAttempt.reason
+        });
+        return;
+      }
+      backgroundWorker.attemptCreatedNotification.post({
+        success: true,
+        execution: createAttempt.executionPayload.execution
       });
-      this.paused = false;
-      this.nextResumeAfter = void 0;
-      this.waitForPostStart = false;
-      this.#coordinatorSocket.socket.emit("CANCEL_CHECKPOINT", { version: "v1" });
     });
-    this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
+    backgroundWorker.attemptCreatedNotification.attach((message) => {
+      if (!message.success) {
+        return;
+      }
+      this.attemptFriendlyId = message.execution.attempt.id;
+    });
+    backgroundWorker.onWaitForDuration.attach(async (message) => {
       if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError(
+          "NoAttemptId",
+          "Attempt ID not set before waiting for duration"
+        );
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -619,9 +968,10 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
     });
-    this.#backgroundWorker.onWaitForTask.attach(async (message) => {
+    backgroundWorker.onWaitForTask.attach(async (message) => {
       if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for task");
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -633,9 +983,10 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
     });
-    this.#backgroundWorker.onWaitForBatch.attach(async (message) => {
+    backgroundWorker.onWaitForBatch.attach(async (message) => {
       if (!this.attemptFriendlyId) {
-        logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        logger.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for batch");
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -647,84 +998,50 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
     });
-    this.#httpPort = port;
-    this.#httpServer = this.#createHttpServer();
+    return backgroundWorker;
   }
-  apiUrl = process.env.TRIGGER_API_URL;
-  apiKey = process.env.TRIGGER_SECRET_KEY;
-  contentHash = process.env.TRIGGER_CONTENT_HASH;
-  projectRef = process.env.TRIGGER_PROJECT_REF;
-  envId = process.env.TRIGGER_ENV_ID;
-  runId = process.env.TRIGGER_RUN_ID || "index-only";
-  deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
-  deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
-  runningInKubernetes = !!process.env.KUBERNETES_PORT;
-  executing = false;
-  completed = /* @__PURE__ */ new Set();
-  paused = false;
-  attemptFriendlyId;
-  nextResumeAfter;
-  waitForPostStart = false;
-  #httpPort;
-  #backgroundWorker;
-  #httpServer;
-  #coordinatorSocket;
-  async #reconnect(isPostStart = false) {
-    if (isPostStart) {
-      this.waitForPostStart = false;
-    }
-    this.#coordinatorSocket.close();
-    if (!this.runningInKubernetes) {
-      this.#coordinatorSocket.connect();
-      return;
-    }
-    try {
-      const coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
-        "\n",
-        ""
-      );
-      logger2.log("reconnecting", {
-        coordinatorHost: {
-          fromEnv: COORDINATOR_HOST,
-          fromVolume: coordinatorHost,
-          current: this.#coordinatorSocket.socket.io.opts.hostname
-        }
-      });
-      this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
-    } catch (error) {
-      logger2.error("taskinfo read error during reconnect", { error });
-      this.#coordinatorSocket.connect();
-    }
-  }
-  #prepareForWait(reason, willCheckpointAndRestore) {
-    logger2.log(`prepare for ${reason}`, { willCheckpointAndRestore });
+  async #prepareForWait(reason, willCheckpointAndRestore) {
+    logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
     this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
     if (willCheckpointAndRestore) {
       this.paused = true;
       this.nextResumeAfter = reason;
       this.waitForPostStart = true;
+      if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
+        await this.#prepareForCheckpoint();
+      }
     }
   }
   async #prepareForRetry(willCheckpointAndRestore, shouldExit) {
-    logger2.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
+    logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
     if (shouldExit) {
       if (willCheckpointAndRestore) {
-        logger2.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
+        logger.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
       }
-      await this.#backgroundWorker.close();
-      process.exit(0);
+      await this.#exitGracefully();
+      return;
     }
+    this.paused = false;
+    this.waitForPostStart = false;
     this.executing = false;
     this.attemptFriendlyId = void 0;
     if (willCheckpointAndRestore) {
       this.waitForPostStart = true;
+      this.#prepareForCheckpoint(false);
       this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
       return;
     }
   }
+  async #prepareForCheckpoint(flush = true) {
+    if (flush) {
+      await this.#backgroundWorker.flushTelemetry();
+    }
+    await this.#backgroundWorker.forceKillOldTaskRunProcesses();
+  }
   #resumeAfterDuration() {
     this.paused = false;
     this.nextResumeAfter = void 0;
+    this.waitForPostStart = false;
     this.#backgroundWorker.waitCompletedNotification();
   }
   #returnValidatedExtraHeaders(headers) {
@@ -735,9 +1052,10 @@ var ProdWorker = class {
     }
     return headers;
   }
+  // FIXME: If the the worker can't connect for a while, this runs MANY times - it should only run once
   #createCoordinatorSocket(host) {
     const extraHeaders = this.#returnValidatedExtraHeaders({
-      "x-machine-name": MACHINE_NAME2,
+      "x-machine-name": MACHINE_NAME,
       "x-pod-name": POD_NAME,
       "x-trigger-content-hash": this.contentHash,
       "x-trigger-project-ref": this.projectRef,
@@ -749,12 +1067,9 @@ var ProdWorker = class {
     if (this.attemptFriendlyId) {
       extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
     }
-    logger2.log("connecting to coordinator", {
-      host,
-      port: COORDINATOR_PORT,
-      extraHeaders
-    });
-    const coordinatorConnection = new ZodSocketConnection2({
+    logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
+    logger.debug(`connecting with extra headers`, { extraHeaders });
+    const coordinatorConnection = new ZodSocketConnection({
       namespace: "prod-worker",
       host,
       port: COORDINATOR_PORT,
@@ -762,60 +1077,49 @@ var ProdWorker = class {
       serverMessages: CoordinatorToProdWorkerMessages,
       extraHeaders,
       handlers: {
-        RESUME_AFTER_DEPENDENCY: async (message) => {
+        RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
           if (!this.paused) {
-            logger2.error("worker not paused", {
-              completions: message.completions,
-              executions: message.executions
-            });
-            return;
-          }
-          if (message.completions.length !== message.executions.length) {
-            logger2.error("did not receive the same number of completions and executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+            logger.error("Failed to resume after dependency: Worker not paused");
             return;
           }
-          if (message.completions.length === 0 || message.executions.length === 0) {
-            logger2.error("no completions or executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (completions.length === 0) {
+            logger.error("Failed to resume after dependency: No completions");
             return;
           }
           if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
-            logger2.error("not waiting to resume after dependency", {
+            logger.error("Failed to resume after dependency: Invalid next resume", {
               nextResumeAfter: this.nextResumeAfter
             });
             return;
           }
-          if (this.nextResumeAfter === "WAIT_FOR_TASK" && message.completions.length > 1) {
-            logger2.error("waiting for single task but got multiple completions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
+            logger.error(
+              "Failed to resume after dependency: Waiting for single task but got multiple completions",
+              {
+                completions
+              }
+            );
             return;
           }
           this.paused = false;
           this.nextResumeAfter = void 0;
-          for (let i = 0; i < message.completions.length; i++) {
-            const completion = message.completions[i];
-            const execution = message.executions[i];
-            if (!completion || !execution)
+          this.waitForPostStart = false;
+          for (let i = 0; i < completions.length; i++) {
+            const completion = completions[i];
+            if (!completion)
               continue;
-            this.#backgroundWorker.taskRunCompletedNotification(completion, execution);
+            this.#backgroundWorker.taskRunCompletedNotification(completion);
           }
         },
         RESUME_AFTER_DURATION: async (message) => {
           if (!this.paused) {
-            logger2.error("worker not paused", {
+            logger.error("worker not paused", {
               attemptId: message.attemptId
             });
             return;
           }
           if (this.nextResumeAfter !== "WAIT_FOR_DURATION") {
-            logger2.error("not waiting to resume after duration", {
+            logger.error("not waiting to resume after duration", {
               nextResumeAfter: this.nextResumeAfter
             });
             return;
@@ -824,34 +1128,79 @@ var ProdWorker = class {
         },
         EXECUTE_TASK_RUN: async ({ executionPayload }) => {
           if (this.executing) {
-            logger2.error("dropping execute request, already executing");
+            logger.error("dropping execute request, already executing");
             return;
           }
           if (this.completed.has(executionPayload.execution.attempt.id)) {
-            logger2.error("dropping execute request, already completed");
+            logger.error("dropping execute request, already completed");
             return;
           }
           this.executing = true;
           this.attemptFriendlyId = executionPayload.execution.attempt.id;
           const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
-          logger2.log("completed", completion);
+          logger.log("completed", completion);
           this.completed.add(executionPayload.execution.attempt.id);
-          await this.#backgroundWorker.flushTelemetry();
           const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
             version: "v1",
             execution: executionPayload.execution,
             completion
           });
-          logger2.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+          logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
           this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
         },
+        EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
+          if (this.executing) {
+            logger.error("dropping execute request, already executing");
+            return;
+          }
+          this.executing = true;
+          try {
+            const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
+            logger.log("completed", completion);
+            this.completed.add(execution.attempt.id);
+            const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
+              version: "v1",
+              execution,
+              completion
+            });
+            logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+            this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+          } catch (error) {
+            const completion = {
+              ok: false,
+              id: message.lazyPayload.runId,
+              retry: void 0,
+              error: error instanceof Error ? {
+                type: "BUILT_IN_ERROR",
+                name: error.name,
+                message: error.message,
+                stackTrace: error.stack ?? ""
+              } : {
+                type: "BUILT_IN_ERROR",
+                name: "UnknownError",
+                message: String(error),
+                stackTrace: ""
+              }
+            };
+            this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
+              version: "v1",
+              completion
+            });
+          }
+        },
         REQUEST_ATTEMPT_CANCELLATION: async (message) => {
           if (!this.executing) {
+            logger.log("dropping cancel request, not executing", { status: this.#status });
             return;
           }
+          logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
           await this.#backgroundWorker.cancelAttempt(message.attemptId);
         },
-        REQUEST_EXIT: async () => {
+        REQUEST_EXIT: async (message) => {
+          if (message.version === "v2" && message.delayInMs) {
+            logger.log("exit requested with delay", { delayInMs: message.delayInMs });
+            await setTimeout2(message.delayInMs);
+          }
           this.#coordinatorSocket.close();
           process.exit(0);
         },
@@ -859,122 +1208,140 @@ var ProdWorker = class {
           if (this.completed.size < 1) {
             return;
           }
-          this.#coordinatorSocket.socket.emit("READY_FOR_EXECUTION", {
+          this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
             version: "v1",
             runId: this.runId,
             totalCompletions: this.completed.size
           });
         }
       },
-      onConnection: async (socket, handler, sender, logger3) => {
+      onConnection: async (socket, handler, sender, logger2) => {
+        logger2.log("connected to coordinator", { status: this.#status });
         if (this.waitForPostStart) {
-          logger3.log("skip connection handler, waiting for post start hook");
+          logger2.log("skip connection handler, waiting for post start hook");
+          return;
+        }
+        if (this.paused) {
+          if (!this.nextResumeAfter) {
+            logger2.error("Missing next resume reason", { status: this.#status });
+            this.#emitUnrecoverableError(
+              "NoNextResume",
+              "Next resume reason not set while resuming from paused state"
+            );
+            return;
+          }
+          if (!this.attemptFriendlyId) {
+            logger2.error("Missing friendly ID", { status: this.#status });
+            this.#emitUnrecoverableError(
+              "NoAttemptId",
+              "Attempt ID not set while resuming from paused state"
+            );
+            return;
+          }
+          socket.emit("READY_FOR_RESUME", {
+            version: "v1",
+            attemptFriendlyId: this.attemptFriendlyId,
+            type: this.nextResumeAfter
+          });
           return;
         }
         if (process.env.INDEX_TASKS === "true") {
           try {
             const taskResources = await this.#initializeWorker();
             const { success } = await socket.emitWithAck("INDEX_TASKS", {
-              version: "v1",
+              version: "v2",
               deploymentId: this.deploymentId,
-              ...taskResources
+              ...taskResources,
+              supportsLazyAttempts: true
             });
             if (success) {
-              logger3.info("indexing done, shutting down..");
+              logger2.info("indexing done, shutting down..");
               process.exit(0);
             } else {
-              logger3.info("indexing failure, shutting down..");
+              logger2.info("indexing failure, shutting down..");
               process.exit(1);
             }
           } catch (e) {
+            const stderr = this.#backgroundWorker.stderr.join("\n");
             if (e instanceof TaskMetadataParseError) {
-              logger3.error("tasks metadata parse error", { message: e.zodIssues, tasks: e.tasks });
+              logger2.error("tasks metadata parse error", {
+                zodIssues: e.zodIssues,
+                tasks: e.tasks
+              });
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
                 error: {
                   name: "TaskMetadataParseError",
                   message: "There was an error parsing the task metadata",
-                  stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks })
+                  stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
+                  stderr
                 }
               });
             } else if (e instanceof UncaughtExceptionError) {
-              logger3.error("uncaught exception", { message: e.originalError.message });
+              const error = {
+                name: e.originalError.name,
+                message: e.originalError.message,
+                stack: e.originalError.stack,
+                stderr
+              };
+              logger2.error("uncaught exception", { originalError: error });
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
-                  name: e.originalError.name,
-                  message: e.originalError.message,
-                  stack: e.originalError.stack
-                }
+                error
               });
             } else if (e instanceof Error) {
-              logger3.error("error", { message: e.message });
+              const error = {
+                name: e.name,
+                message: e.message,
+                stack: e.stack,
+                stderr
+              };
+              logger2.error("error", { error });
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
-                  name: e.name,
-                  message: e.message,
-                  stack: e.stack
-                }
+                error
               });
             } else if (typeof e === "string") {
-              logger3.error("string error", { message: e });
+              logger2.error("string error", { error: { message: e } });
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
                 error: {
                   name: "Error",
-                  message: e
+                  message: e,
+                  stderr
                 }
               });
             } else {
-              logger3.error("unknown error", { error: e });
+              logger2.error("unknown error", { error: e });
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
                 error: {
                   name: "Error",
-                  message: "Unknown error"
+                  message: "Unknown error",
+                  stderr
                 }
               });
             }
             await setTimeout2(200);
-            process.exit(1);
+            process.exit(111);
           }
         }
-        if (this.paused) {
-          if (!this.nextResumeAfter) {
-            return;
-          }
-          if (!this.attemptFriendlyId) {
-            logger3.error("Missing friendly ID");
-            return;
-          }
-          if (this.nextResumeAfter === "WAIT_FOR_DURATION") {
-            this.#resumeAfterDuration();
-            return;
-          }
-          socket.emit("READY_FOR_RESUME", {
-            version: "v1",
-            attemptFriendlyId: this.attemptFriendlyId,
-            type: this.nextResumeAfter
-          });
-          return;
-        }
         if (this.executing) {
           return;
         }
-        socket.emit("READY_FOR_EXECUTION", {
+        socket.emit("READY_FOR_LAZY_ATTEMPT", {
           version: "v1",
           runId: this.runId,
           totalCompletions: this.completed.size
         });
       },
-      onError: async (socket, err, logger3) => {
-        logger3.error("onError", {
+      onError: async (socket, err, logger2) => {
+        logger2.error("onError", {
           error: {
             name: err.name,
             message: err.message
@@ -982,14 +1349,14 @@ var ProdWorker = class {
         });
         await this.#reconnect();
       },
-      onDisconnect: async (socket, reason, description, logger3) => {
+      onDisconnect: async (socket, reason, description, logger2) => {
       }
     });
     return coordinatorConnection;
   }
   #createHttpServer() {
     const httpServer = createServer(async (req, res) => {
-      logger2.log(`[${req.method}]`, req.url);
+      logger.log(`[${req.method}]`, req.url);
       const reply = new HttpReply(res);
       try {
         const url = new URL(req.url ?? "", `http://${req.headers.host}`);
@@ -998,11 +1365,7 @@ var ProdWorker = class {
             return reply.text("ok");
           }
           case "/status": {
-            return reply.json({
-              executing: this.executing,
-              pause: this.paused,
-              nextResumeAfter: this.nextResumeAfter
-            });
+            return reply.json(this.#status);
           }
           case "/connect": {
             this.#coordinatorSocket.connect();
@@ -1026,7 +1389,7 @@ var ProdWorker = class {
           case "/preStop": {
             const cause = PreStopCauses.safeParse(url.searchParams.get("cause"));
             if (!cause.success) {
-              logger2.error("Failed to parse cause", { cause });
+              logger.error("Failed to parse cause", { cause });
               return reply.text("Failed to parse cause", 400);
             }
             switch (cause.data) {
@@ -1034,17 +1397,16 @@ var ProdWorker = class {
                 break;
               }
               default: {
-                logger2.error("Unhandled cause", { cause: cause.data });
+                logger.error("Unhandled cause", { cause: cause.data });
                 break;
               }
             }
-            logger2.log("preStop", { url: req.url });
             return reply.text("preStop ok");
           }
           case "/postStart": {
             const cause = PostStartCauses.safeParse(url.searchParams.get("cause"));
             if (!cause.success) {
-              logger2.error("Failed to parse cause", { cause });
+              logger.error("Failed to parse cause", { cause });
               return reply.text("Failed to parse cause", 400);
             }
             switch (cause.data) {
@@ -1055,11 +1417,11 @@ var ProdWorker = class {
                 break;
               }
               case "restore": {
-                await this.#reconnect(true);
+                await this.#reconnect(true, true);
                 break;
               }
               default: {
-                logger2.error("Unhandled cause", { cause: cause.data });
+                logger.error("Unhandled cause", { cause: cause.data });
                 break;
               }
             }
@@ -1070,7 +1432,7 @@ var ProdWorker = class {
           }
         }
       } catch (error) {
-        logger2.error("HTTP server error", { error });
+        logger.error("HTTP server error", { error });
         reply.empty(500);
       }
     });
@@ -1078,13 +1440,13 @@ var ProdWorker = class {
       socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
     });
     httpServer.on("listening", () => {
-      logger2.log("http server listening on port", this.#httpPort);
+      logger.log("http server listening on port", this.#httpPort);
     });
     httpServer.on("error", async (error) => {
       if (error.code != "EADDRINUSE") {
         return;
       }
-      logger2.error(`port ${this.#httpPort} already in use, retrying with random port..`);
+      logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
       this.#httpPort = getRandomPortNumber();
       await setTimeout2(100);
       this.start();
@@ -1124,11 +1486,30 @@ var ProdWorker = class {
     const data = await response.json();
     return data?.variables ?? {};
   }
+  get #status() {
+    return {
+      executing: this.executing,
+      paused: this.paused,
+      completed: this.completed.size,
+      nextResumeAfter: this.nextResumeAfter,
+      waitForPostStart: this.waitForPostStart,
+      attemptFriendlyId: this.attemptFriendlyId
+    };
+  }
+  #emitUnrecoverableError(name, message) {
+    this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
+      version: "v1",
+      error: {
+        name,
+        message
+      }
+    });
+  }
   start() {
     this.#httpServer.listen(this.#httpPort, this.host);
   }
 };
-var prodWorker = new ProdWorker(HTTP_SERVER_PORT2);
+var prodWorker = new ProdWorker(HTTP_SERVER_PORT);
 prodWorker.start();
 function gatherProcessEnv() {
   const env = {