trigger.dev 3.0.0-beta.34 → 3.0.0-beta.36

package/dist/workers/prod/entry-point.js

@@ -116,8 +116,6 @@ var TaskMetadataParseError = class extends Error {
     this.name = "TaskMetadataParseError";
   }
 };
-
-// src/workers/prod/backgroundWorker.ts
 var UnexpectedExitError = class extends Error {
   constructor(code) {
     super(`Unexpected exit with code ${code}`);
@@ -137,13 +135,31 @@ var CancelledProcessError = class extends Error {
     this.name = "CancelledProcessError";
   }
 };
+var SigKillTimeoutProcessError = class extends Error {
+  constructor() {
+    super("Process kill timeout");
+    this.name = "SigKillTimeoutProcessError";
+  }
+};
+var GracefulExitTimeoutError = class extends Error {
+  constructor() {
+    super("Graceful exit timeout");
+    this.name = "GracefulExitTimeoutError";
+  }
+};
+
+// src/workers/prod/backgroundWorker.ts
 var ProdBackgroundWorker = class {
   constructor(path, params) {
     this.path = path;
     this.params = params;
   }
   _initialized = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
@@ -151,17 +167,40 @@ var ProdBackgroundWorker = class {
   checkpointCanceledNotification = Evt.create();
   onReadyForCheckpoint = Evt.create();
   onCancelCheckpoint = Evt.create();
+  onCreateTaskRunAttempt = Evt.create();
+  attemptCreatedNotification = Evt.create();
   _onClose = new Evt();
   tasks = [];
   _taskRunProcess;
+  _taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
   _closed = false;
-  async close() {
+  async close(gracefulExitTimeoutElapsed = false) {
+    console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
     if (this._closed) {
       return;
     }
     this._closed = true;
     this.onTaskHeartbeat.detach();
-    await this._taskRunProcess?.cleanup(true);
+    this.onTaskRunHeartbeat.detach();
+    await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
+  }
+  async #killTaskRunProcess(flush = true, initialSignal = "SIGTERM") {
+    console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
+    if (this._closed || !this._taskRunProcess) {
+      return;
+    }
+    if (flush) {
+      await this.flushTelemetry();
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
+      console.error("Error while trying graceful exit", error);
+    });
+    console.log("Killed task run process, setting closed to true", {
+      closed: this._closed,
+      pid: currentTaskRunProcess.pid
+    });
+    this._closed = true;
   }
   async flushTelemetry() {
     await this._taskRunProcess?.cleanup(false);
@@ -251,64 +290,144 @@ var ProdBackgroundWorker = class {
   }
   // We need to notify all the task run processes that a task run has completed,
   // in case they are waiting for it through triggerAndWait
-  async taskRunCompletedNotification(completion, execution) {
-    this._taskRunProcess?.taskRunCompletedNotification(completion, execution);
+  async taskRunCompletedNotification(completion) {
+    this._taskRunProcess?.taskRunCompletedNotification(completion);
   }
   async waitCompletedNotification() {
     this._taskRunProcess?.waitCompletedNotification();
   }
-  async #initializeTaskRunProcess(payload) {
+  async #getFreshTaskRunProcess(payload, messageId) {
     const metadata = this.getMetadata(
       payload.execution.worker.id,
       payload.execution.worker.version
     );
-    if (!this._taskRunProcess) {
-      const taskRunProcess = new TaskRunProcess(
-        payload.execution,
-        this.path,
-        {
-          ...this.params.env,
-          ...payload.environment ?? {}
-        },
-        metadata,
-        this.params
-      );
-      taskRunProcess.onExit.attach(() => {
+    console.log("Getting fresh task run process, setting closed to false", {
+      closed: this._closed
+    });
+    this._closed = false;
+    await this.#killCurrentTaskRunProcessBeforeAttempt();
+    const taskRunProcess = new TaskRunProcess(
+      payload.execution.run.id,
+      payload.execution.run.isTest,
+      this.path,
+      {
+        ...this.params.env,
+        ...payload.environment ?? {}
+      },
+      metadata,
+      this.params,
+      messageId
+    );
+    taskRunProcess.onExit.attach(({ pid }) => {
+      console.log("Task run process exited", { pid });
+      if (this._taskRunProcess?.pid === pid) {
         this._taskRunProcess = void 0;
-      });
-      taskRunProcess.onTaskHeartbeat.attach((id) => {
-        this.onTaskHeartbeat.post(id);
-      });
-      taskRunProcess.onWaitForBatch.attach((message) => {
-        this.onWaitForBatch.post(message);
-      });
-      taskRunProcess.onWaitForDuration.attach((message) => {
-        this.onWaitForDuration.post(message);
-      });
-      taskRunProcess.onWaitForTask.attach((message) => {
-        this.onWaitForTask.post(message);
-      });
-      taskRunProcess.onReadyForCheckpoint.attach((message) => {
-        this.onReadyForCheckpoint.post(message);
-      });
-      taskRunProcess.onCancelCheckpoint.attach((message) => {
-        this.onCancelCheckpoint.post(message);
-      });
-      this.preCheckpointNotification.attach((message) => {
-        taskRunProcess.preCheckpointNotification.post(message);
-      });
-      this.checkpointCanceledNotification.attach((message) => {
-        taskRunProcess.checkpointCanceledNotification.post(message);
-      });
-      await taskRunProcess.initialize();
-      this._taskRunProcess = taskRunProcess;
-    }
+      }
+      if (pid) {
+        this._taskRunProcessesBeingKilled.delete(pid);
+      }
+    });
+    taskRunProcess.onIsBeingKilled.attach((taskRunProcess2) => {
+      if (taskRunProcess2?.pid) {
+        this._taskRunProcessesBeingKilled.set(taskRunProcess2.pid, taskRunProcess2);
+      }
+    });
+    taskRunProcess.onTaskHeartbeat.attach((id) => {
+      this.onTaskHeartbeat.post(id);
+    });
+    taskRunProcess.onTaskRunHeartbeat.attach((id) => {
+      this.onTaskRunHeartbeat.post(id);
+    });
+    taskRunProcess.onWaitForBatch.attach((message) => {
+      this.onWaitForBatch.post(message);
+    });
+    taskRunProcess.onWaitForDuration.attach((message) => {
+      this.onWaitForDuration.post(message);
+    });
+    taskRunProcess.onWaitForTask.attach((message) => {
+      this.onWaitForTask.post(message);
+    });
+    taskRunProcess.onReadyForCheckpoint.attach((message) => {
+      this.onReadyForCheckpoint.post(message);
+    });
+    taskRunProcess.onCancelCheckpoint.attach((message) => {
+      this.onCancelCheckpoint.post(message);
+    });
+    this.preCheckpointNotification.attach((message) => {
+      taskRunProcess.preCheckpointNotification.post(message);
+    });
+    this.checkpointCanceledNotification.attach((message) => {
+      taskRunProcess.checkpointCanceledNotification.post(message);
+    });
+    await taskRunProcess.initialize();
+    this._taskRunProcess = taskRunProcess;
     return this._taskRunProcess;
   }
-  // We need to fork the process before we can execute any tasks
-  async executeTaskRun(payload) {
+  async forceKillOldTaskRunProcesses() {
+    for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
+      try {
+        await taskRunProcess.kill("SIGKILL");
+      } catch (error) {
+        console.error("Error while force killing old task run processes", error);
+      }
+    }
+  }
+  async #killCurrentTaskRunProcessBeforeAttempt() {
+    console.log("killCurrentTaskRunProcessBeforeAttempt()", {
+      hasTaskRunProcess: !!this._taskRunProcess
+    });
+    if (!this._taskRunProcess) {
+      return;
+    }
+    const currentTaskRunProcess = this._taskRunProcess;
+    console.log("Killing current task run process", {
+      isBeingKilled: currentTaskRunProcess?.isBeingKilled,
+      totalBeingKilled: this._taskRunProcessesBeingKilled.size
+    });
+    if (currentTaskRunProcess.isBeingKilled) {
+      if (this._taskRunProcessesBeingKilled.size > 1) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+      }
+    } else {
+      if (this._taskRunProcessesBeingKilled.size > 0) {
+        await this.#tryGracefulExit(currentTaskRunProcess);
+      } else {
+        currentTaskRunProcess.kill("SIGTERM", 5e3).catch(() => {
+        });
+      }
+    }
+  }
+  async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
     try {
-      const taskRunProcess = await this.#initializeTaskRunProcess(payload);
+      const initialExit = taskRunProcess.onExit.waitFor(5e3);
+      if (kill) {
+        taskRunProcess.kill(initialSignal);
+      }
+      await initialExit;
+    } catch (error) {
+      console.error("TaskRunProcess graceful kill timeout exceeded", error);
+      this.#tryForcefulExit(taskRunProcess);
+    }
+  }
+  async #tryForcefulExit(taskRunProcess) {
+    try {
+      const forcedKill = taskRunProcess.onExit.waitFor(5e3);
+      taskRunProcess.kill("SIGKILL");
+      await forcedKill;
+    } catch (error) {
+      console.error("TaskRunProcess forced kill timeout exceeded", error);
+      throw new SigKillTimeoutProcessError();
+    }
+  }
+  // We need to fork the process before we can execute any tasks, use a fresh process for each execution
+  async executeTaskRun(payload, messageId) {
+    try {
+      const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
+      console.log("executing task run", {
+        attempt: payload.execution.attempt.id,
+        taskRunPid: taskRunProcess.pid
+      });
       const result = await taskRunProcess.executeTaskRun(payload);
       if (result.ok) {
         return result;
@@ -356,6 +475,29 @@ var ProdBackgroundWorker = class {
           }
         };
       }
+      if (e instanceof SigKillTimeoutProcessError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT
+          }
+        };
+      }
+      if (e instanceof GracefulExitTimeoutError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: void 0,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
+            message: "Worker process killed while attempt in progress."
+          }
+        };
+      }
       return {
         id: payload.execution.attempt.id,
         ok: false,
@@ -365,10 +507,41 @@ var ProdBackgroundWorker = class {
           code: TaskRunErrorCodes.TASK_EXECUTION_FAILED
         }
       };
+    } finally {
+      await this.#killTaskRunProcess();
     }
   }
   async cancelAttempt(attemptId) {
-    await this._taskRunProcess?.cancel();
+    if (!this._taskRunProcess) {
+      console.error("No task run process to cancel attempt", { attemptId });
+      return;
+    }
+    await this._taskRunProcess.cancel();
+  }
+  async executeTaskRunLazyAttempt(payload) {
+    this.onCreateTaskRunAttempt.post({ runId: payload.runId });
+    let execution;
+    try {
+      const attemptCreated = await this.attemptCreatedNotification.waitFor(3e4);
+      if (!attemptCreated.success) {
+        throw new Error(
+          `Failed to create attempt${attemptCreated.reason ? `: ${attemptCreated.reason}` : ""}`
+        );
+      }
+      execution = attemptCreated.execution;
+    } catch (error) {
+      console.error("Error while creating attempt", error);
+      throw new Error(`Failed to create task run attempt: ${error}`);
+    }
+    const completion = await this.executeTaskRun(
+      {
+        execution,
+        traceContext: payload.traceContext,
+        environment: payload.environment
+      },
+      payload.messageId
+    );
+    return { execution, completion };
   }
   async #correctError(error, execution) {
     return {
@@ -378,22 +551,31 @@ var ProdBackgroundWorker = class {
   }
 };
 var TaskRunProcess = class {
-  constructor(execution, path, env, metadata, worker) {
-    this.execution = execution;
+  constructor(runId, isTest, path, env, metadata, worker, messageId) {
+    this.runId = runId;
+    this.isTest = isTest;
     this.path = path;
     this.env = env;
     this.metadata = metadata;
     this.worker = worker;
+    this.messageId = messageId;
   }
   _ipc;
   _child;
+  _childPid;
   _attemptPromises = /* @__PURE__ */ new Map();
   _attemptStatuses = /* @__PURE__ */ new Map();
   _currentExecution;
   _isBeingKilled = false;
   _isBeingCancelled = false;
+  _gracefulExitTimeoutElapsed = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onExit = new Evt();
+  onIsBeingKilled = new Evt();
   onWaitForBatch = new Evt();
   onWaitForDuration = new Evt();
   onWaitForTask = new Evt();
@@ -413,7 +595,7 @@ var TaskRunProcess = class {
         "ipc"
       ],
       env: {
-        ...this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
+        ...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
         ...this.env,
         OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
           [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
@@ -421,6 +603,7 @@ var TaskRunProcess = class {
         ...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
       }
     });
+    this._childPid = this._child?.pid;
     this._ipc = new ZodIpcConnection({
       listenSchema: ProdChildToWorkerMessages,
       emitSchema: ProdWorkerToChildMessages,
@@ -444,7 +627,11 @@ var TaskRunProcess = class {
           process.exit(0);
         },
         TASK_HEARTBEAT: async (message) => {
-          this.onTaskHeartbeat.post(message.id);
+          if (this.messageId) {
+            this.onTaskRunHeartbeat.post(this.messageId);
+          } else {
+            this.onTaskHeartbeat.post(message.id);
+          }
         },
         TASKS_READY: async (message) => {
         },
@@ -502,15 +689,33 @@ var TaskRunProcess = class {
     this._isBeingCancelled = true;
     await this.cleanup(true);
   }
-  async cleanup(kill = false) {
+  async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
+    console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
     if (kill && this._isBeingKilled) {
       return;
     }
-    this._isBeingKilled = kill;
-    await this._ipc?.sendWithAck("CLEANUP", {
-      flush: true,
-      kill
+    if (kill) {
+      this._isBeingKilled = true;
+      this.onIsBeingKilled.post(this);
+    }
+    const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
+    const killParentProcess = kill && !killChildProcess;
+    console.log("Cleaning up task run process", {
+      killChildProcess,
+      killParentProcess
     });
+    await this._ipc?.sendWithAck(
+      "CLEANUP",
+      {
+        flush: true,
+        kill: killParentProcess
+      },
+      3e4
+    );
+    if (killChildProcess) {
+      this._gracefulExitTimeoutElapsed = true;
+      await this.kill("SIGKILL");
+    }
   }
   async executeTaskRun(payload) {
     let resolver;
@@ -534,14 +739,14 @@ var TaskRunProcess = class {
     this._currentExecution = void 0;
     return result;
   }
-  taskRunCompletedNotification(completion, execution) {
+  taskRunCompletedNotification(completion) {
     if (!completion.ok && typeof completion.retry !== "undefined") {
       return;
     }
     if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
       this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
-        completion,
-        execution
+        version: "v2",
+        completion
       });
     }
   }
@@ -550,9 +755,11 @@ var TaskRunProcess = class {
       this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
     }
   }
-  async #handleExit(code) {
+  async #handleExit(code, signal) {
+    console.log("handling child exit", { code, signal });
     for (const [id, status] of this._attemptStatuses.entries()) {
       if (status === "PENDING") {
+        console.log("found pending attempt", { id });
         this._attemptStatuses.set(id, "REJECTED");
         const attemptPromise = this._attemptPromises.get(id);
         if (!attemptPromise) {
@@ -561,14 +768,16 @@ var TaskRunProcess = class {
         const { rejecter } = attemptPromise;
         if (this._isBeingCancelled) {
           rejecter(new CancelledProcessError());
+        } else if (this._gracefulExitTimeoutElapsed) {
+          rejecter(new GracefulExitTimeoutError());
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code));
+          rejecter(new UnexpectedExitError(code ?? -1));
         }
       }
     }
-    this.onExit.post(code);
+    this.onExit.post({ code, signal, pid: this.pid });
   }
   #handleLog(data) {
     if (!this._currentExecution) {
@@ -590,11 +799,21 @@ var TaskRunProcess = class {
       `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number}] ${data.toString()}`
     );
   }
-  #kill() {
-    if (this._child && !this._child.killed) {
-      this._child?.kill();
+  async kill(signal, timeoutInMs) {
+    this._isBeingKilled = true;
+    const killTimeout = this.onExit.waitFor(timeoutInMs);
+    this.onIsBeingKilled.post(this);
+    this._child?.kill(signal);
+    if (timeoutInMs) {
+      await killTimeout;
     }
   }
+  get isBeingKilled() {
+    return this._isBeingKilled || this._child?.killed;
+  }
+  get pid() {
+    return this._childPid;
+  }
 };
 
 // src/workers/prod/entry-point.ts
@@ -611,7 +830,88 @@ var ProdWorker = class {
     this.host = host;
     process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
     this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
-    this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
+    this.#backgroundWorker = this.#createBackgroundWorker();
+    this.#httpPort = port;
+    this.#httpServer = this.#createHttpServer();
+  }
+  apiUrl = process.env.TRIGGER_API_URL;
+  apiKey = process.env.TRIGGER_SECRET_KEY;
+  contentHash = process.env.TRIGGER_CONTENT_HASH;
+  projectRef = process.env.TRIGGER_PROJECT_REF;
+  envId = process.env.TRIGGER_ENV_ID;
+  runId = process.env.TRIGGER_RUN_ID || "index-only";
+  deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
+  deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
+  runningInKubernetes = !!process.env.KUBERNETES_PORT;
+  executing = false;
+  completed = /* @__PURE__ */ new Set();
+  paused = false;
+  attemptFriendlyId;
+  nextResumeAfter;
+  waitForPostStart = false;
+  #httpPort;
+  #backgroundWorker;
+  #httpServer;
+  #coordinatorSocket;
+  async #handleSignal(signal) {
+    logger2.log("Received signal", { signal });
+    if (signal === "SIGTERM") {
+      let gracefulExitTimeoutElapsed = false;
+      if (this.executing) {
+        const terminationGracePeriodSeconds = 60 * 60;
+        logger2.log("Waiting for attempt to complete before exiting", {
+          terminationGracePeriodSeconds
+        });
+        await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
+        gracefulExitTimeoutElapsed = true;
+        logger2.log("Termination timeout reached, exiting gracefully.");
+      } else {
+        logger2.log("Not executing, exiting immediately.");
+      }
+      await this.#exitGracefully(gracefulExitTimeoutElapsed);
+      return;
+    }
+    logger2.log("Unhandled signal", { signal });
+  }
+  async #exitGracefully(gracefulExitTimeoutElapsed = false) {
+    await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
+    if (!gracefulExitTimeoutElapsed) {
+      process.exit(0);
+    }
+  }
+  async #reconnect(isPostStart = false, reconnectImmediately = false) {
+    if (isPostStart) {
+      this.waitForPostStart = false;
+    }
+    this.#coordinatorSocket.close();
+    if (!reconnectImmediately) {
+      await setTimeout2(1e3);
+    }
+    let coordinatorHost = COORDINATOR_HOST;
+    try {
+      if (this.runningInKubernetes) {
+        coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
+          "\n",
+          ""
+        );
+        logger2.log("reconnecting", {
+          coordinatorHost: {
+            fromEnv: COORDINATOR_HOST,
+            fromVolume: coordinatorHost,
+            current: this.#coordinatorSocket.socket.io.opts.hostname
+          }
+        });
+      }
+    } catch (error) {
+      logger2.error("taskinfo read error during reconnect", {
+        error: error instanceof Error ? error.message : error
+      });
+    } finally {
+      this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
+    }
+  }
+  #createBackgroundWorker() {
+    const backgroundWorker = new ProdBackgroundWorker("worker.js", {
       projectConfig: __PROJECT_CONFIG__,
       env: {
         ...gatherProcessEnv(),
@@ -621,14 +921,17 @@ var ProdWorker = class {
       },
       contentHash: this.contentHash
     });
-    this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
+    backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
       this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
     });
-    this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
-      await this.#backgroundWorker.flushTelemetry();
+    backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
+      this.#coordinatorSocket.socket.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
+    });
+    backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
+      await this.#prepareForCheckpoint();
       this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
     });
-    this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
+    backgroundWorker.onCancelCheckpoint.attach(async (message) => {
       logger2.log("onCancelCheckpoint", { message });
       const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
         "CANCEL_CHECKPOINT",
@@ -637,6 +940,7 @@ var ProdWorker = class {
           reason: message.reason
         }
       );
+      logger2.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
       if (checkpointCanceled) {
         if (message.reason === "WAIT_FOR_DURATION") {
           this.paused = false;
@@ -644,11 +948,42 @@ var ProdWorker = class {
           this.waitForPostStart = false;
         }
       }
-      this.#backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
+      backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
+    });
+    backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
+      logger2.log("onCreateTaskRunAttempt()", { message });
+      const createAttempt = await this.#coordinatorSocket.socket.emitWithAck(
+        "CREATE_TASK_RUN_ATTEMPT",
+        {
+          version: "v1",
+          runId: message.runId
+        }
+      );
+      if (!createAttempt.success) {
+        backgroundWorker.attemptCreatedNotification.post({
+          success: false,
+          reason: createAttempt.reason
+        });
+        return;
+      }
+      backgroundWorker.attemptCreatedNotification.post({
+        success: true,
+        execution: createAttempt.executionPayload.execution
+      });
     });
-    this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
+    backgroundWorker.attemptCreatedNotification.attach((message) => {
+      if (!message.success) {
+        return;
+      }
+      this.attemptFriendlyId = message.execution.attempt.id;
+    });
+    backgroundWorker.onWaitForDuration.attach(async (message) => {
       if (!this.attemptFriendlyId) {
         logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError(
+          "NoAttemptId",
+          "Attempt ID not set before waiting for duration"
+        );
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -660,9 +995,10 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
     });
-    this.#backgroundWorker.onWaitForTask.attach(async (message) => {
+    backgroundWorker.onWaitForTask.attach(async (message) => {
       if (!this.attemptFriendlyId) {
         logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for task");
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -674,9 +1010,10 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
     });
-    this.#backgroundWorker.onWaitForBatch.attach(async (message) => {
+    backgroundWorker.onWaitForBatch.attach(async (message) => {
       if (!this.attemptFriendlyId) {
         logger2.error("Failed to send wait message, attempt friendly ID not set", { message });
+        this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for batch");
         return;
       }
       const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
@@ -688,77 +1025,7 @@ var ProdWorker = class {
       );
       this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
     });
-    this.#httpPort = port;
-    this.#httpServer = this.#createHttpServer();
-  }
-  apiUrl = process.env.TRIGGER_API_URL;
-  apiKey = process.env.TRIGGER_SECRET_KEY;
-  contentHash = process.env.TRIGGER_CONTENT_HASH;
-  projectRef = process.env.TRIGGER_PROJECT_REF;
-  envId = process.env.TRIGGER_ENV_ID;
-  runId = process.env.TRIGGER_RUN_ID || "index-only";
-  deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
-  deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
-  runningInKubernetes = !!process.env.KUBERNETES_PORT;
-  executing = false;
-  completed = /* @__PURE__ */ new Set();
-  paused = false;
-  attemptFriendlyId;
-  nextResumeAfter;
-  waitForPostStart = false;
-  #httpPort;
-  #backgroundWorker;
-  #httpServer;
-  #coordinatorSocket;
-  async #handleSignal(signal) {
-    logger2.log("Received signal", { signal });
-    if (signal === "SIGTERM") {
-      if (this.executing) {
-        const terminationGracePeriodSeconds = 60 * 60;
-        logger2.log("Waiting for attempt to complete before exiting", {
-          terminationGracePeriodSeconds
-        });
-        await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
-        logger2.log("Termination timeout reached, exiting gracefully.");
-      } else {
-        logger2.log("Not executing, exiting immediately.");
-      }
-      await this.#exitGracefully();
-    }
-    logger2.log("Unhandled signal", { signal });
-  }
-  async #exitGracefully() {
-    await this.#backgroundWorker.close();
-    process.exit(0);
-  }
-  async #reconnect(isPostStart = false, reconnectImmediately = false) {
-    if (isPostStart) {
-      this.waitForPostStart = false;
-    }
-    this.#coordinatorSocket.close();
-    if (!reconnectImmediately) {
-      await setTimeout2(1e3);
-    }
-    let coordinatorHost = COORDINATOR_HOST;
-    try {
-      if (this.runningInKubernetes) {
-        coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
-          "\n",
-          ""
-        );
-        logger2.log("reconnecting", {
-          coordinatorHost: {
-            fromEnv: COORDINATOR_HOST,
-            fromVolume: coordinatorHost,
-            current: this.#coordinatorSocket.socket.io.opts.hostname
-          }
-        });
-      }
-    } catch (error) {
-      logger2.error("taskinfo read error during reconnect", { error });
-    } finally {
-      this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
-    }
+    return backgroundWorker;
   }
   async #prepareForWait(reason, willCheckpointAndRestore) {
     logger2.log(`prepare for ${reason}`, { willCheckpointAndRestore });
@@ -768,7 +1035,7 @@ var ProdWorker = class {
       this.nextResumeAfter = reason;
       this.waitForPostStart = true;
       if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
-        await this.#backgroundWorker.flushTelemetry();
+        await this.#prepareForCheckpoint();
       }
     }
   }
@@ -779,15 +1046,25 @@ var ProdWorker = class {
         logger2.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
       }
       await this.#exitGracefully();
+      return;
     }
+    this.paused = false;
+    this.waitForPostStart = false;
     this.executing = false;
     this.attemptFriendlyId = void 0;
     if (willCheckpointAndRestore) {
       this.waitForPostStart = true;
+      this.#prepareForCheckpoint(false);
       this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
       return;
     }
   }
+  async #prepareForCheckpoint(flush = true) {
+    if (flush) {
+      await this.#backgroundWorker.flushTelemetry();
+    }
+    await this.#backgroundWorker.forceKillOldTaskRunProcesses();
+  }
   #resumeAfterDuration() {
     this.paused = false;
     this.nextResumeAfter = void 0;
@@ -817,11 +1094,8 @@ var ProdWorker = class {
     if (this.attemptFriendlyId) {
       extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
     }
-    logger2.log("connecting to coordinator", {
-      host,
-      port: COORDINATOR_PORT,
-      extraHeaders
-    });
+    logger2.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
+    logger2.debug(`connecting with extra headers`, { extraHeaders });
     const coordinatorConnection = new ZodSocketConnection2({
       namespace: "prod-worker",
       host,
@@ -830,50 +1104,38 @@ var ProdWorker = class {
       serverMessages: CoordinatorToProdWorkerMessages,
       extraHeaders,
       handlers: {
-        RESUME_AFTER_DEPENDENCY: async (message) => {
+        RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
           if (!this.paused) {
-            logger2.error("worker not paused", {
-              completions: message.completions,
-              executions: message.executions
-            });
+            logger2.error("Failed to resume after dependency: Worker not paused");
             return;
           }
-          if (message.completions.length !== message.executions.length) {
-            logger2.error("did not receive the same number of completions and executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
-            return;
-          }
-          if (message.completions.length === 0 || message.executions.length === 0) {
-            logger2.error("no completions or executions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (completions.length === 0) {
+            logger2.error("Failed to resume after dependency: No completions");
             return;
           }
           if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
-            logger2.error("not waiting to resume after dependency", {
+            logger2.error("Failed to resume after dependency: Invalid next resume", {
               nextResumeAfter: this.nextResumeAfter
             });
             return;
           }
-          if (this.nextResumeAfter === "WAIT_FOR_TASK" && message.completions.length > 1) {
-            logger2.error("waiting for single task but got multiple completions", {
-              completions: message.completions,
-              executions: message.executions
-            });
+          if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
+            logger2.error(
+              "Failed to resume after dependency: Waiting for single task but got multiple completions",
+              {
+                completions
+              }
+            );
             return;
           }
           this.paused = false;
           this.nextResumeAfter = void 0;
           this.waitForPostStart = false;
-          for (let i = 0; i < message.completions.length; i++) {
-            const completion = message.completions[i];
-            const execution = message.executions[i];
-            if (!completion || !execution)
+          for (let i = 0; i < completions.length; i++) {
+            const completion = completions[i];
+            if (!completion)
               continue;
-            this.#backgroundWorker.taskRunCompletedNotification(completion, execution);
+            this.#backgroundWorker.taskRunCompletedNotification(completion);
           }
         },
         RESUME_AFTER_DURATION: async (message) => {
@@ -913,13 +1175,59 @@ var ProdWorker = class {
           logger2.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
           this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
         },
+        EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
+          if (this.executing) {
+            logger2.error("dropping execute request, already executing");
+            return;
+          }
+          this.executing = true;
+          try {
+            const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
+            logger2.log("completed", completion);
+            this.completed.add(execution.attempt.id);
+            const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
+              version: "v1",
+              execution,
+              completion
+            });
+            logger2.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
+            this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+          } catch (error) {
+            const completion = {
+              ok: false,
+              id: message.lazyPayload.runId,
+              retry: void 0,
+              error: error instanceof Error ? {
+                type: "BUILT_IN_ERROR",
+                name: error.name,
+                message: error.message,
+                stackTrace: error.stack ?? ""
+              } : {
+                type: "BUILT_IN_ERROR",
+                name: "UnknownError",
+                message: String(error),
+                stackTrace: ""
+              }
+            };
+            this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
+              version: "v1",
+              completion
+            });
+          }
+        },
         REQUEST_ATTEMPT_CANCELLATION: async (message) => {
           if (!this.executing) {
+            logger2.log("dropping cancel request, not executing", { status: this.#status });
             return;
           }
+          logger2.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
           await this.#backgroundWorker.cancelAttempt(message.attemptId);
         },
-        REQUEST_EXIT: async () => {
+        REQUEST_EXIT: async (message) => {
+          if (message.version === "v2" && message.delayInMs) {
+            logger2.log("exit requested with delay", { delayInMs: message.delayInMs });
+            await setTimeout2(message.delayInMs);
+          }
           this.#coordinatorSocket.close();
           process.exit(0);
         },
@@ -927,7 +1235,7 @@ var ProdWorker = class {
           if (this.completed.size < 1) {
             return;
           }
-          this.#coordinatorSocket.socket.emit("READY_FOR_EXECUTION", {
+          this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
             version: "v1",
             runId: this.runId,
             totalCompletions: this.completed.size
@@ -935,16 +1243,26 @@ var ProdWorker = class {
         }
       },
       onConnection: async (socket, handler, sender, logger3) => {
+        logger3.log("connected to coordinator", { status: this.#status });
         if (this.waitForPostStart) {
           logger3.log("skip connection handler, waiting for post start hook");
           return;
         }
         if (this.paused) {
           if (!this.nextResumeAfter) {
+            logger3.error("Missing next resume reason", { status: this.#status });
+            this.#emitUnrecoverableError(
+              "NoNextResume",
+              "Next resume reason not set while resuming from paused state"
+            );
             return;
           }
           if (!this.attemptFriendlyId) {
-            logger3.error("Missing friendly ID");
+            logger3.error("Missing friendly ID", { status: this.#status });
+            this.#emitUnrecoverableError(
+              "NoAttemptId",
+              "Attempt ID not set while resuming from paused state"
+            );
             return;
           }
           socket.emit("READY_FOR_RESUME", {
@@ -958,9 +1276,10 @@ var ProdWorker = class {
           try {
             const taskResources = await this.#initializeWorker();
             const { success } = await socket.emitWithAck("INDEX_TASKS", {
-              version: "v1",
+              version: "v2",
               deploymentId: this.deploymentId,
-              ...taskResources
+              ...taskResources,
+              supportsLazyAttempts: true
             });
             if (success) {
               logger3.info("indexing done, shutting down..");
@@ -1036,7 +1355,7 @@ var ProdWorker = class {
         if (this.executing) {
           return;
         }
-        socket.emit("READY_FOR_EXECUTION", {
+        socket.emit("READY_FOR_LAZY_ATTEMPT", {
           version: "v1",
           runId: this.runId,
           totalCompletions: this.completed.size
@@ -1067,12 +1386,7 @@ var ProdWorker = class {
             return reply.text("ok");
           }
           case "/status": {
-            return reply.json({
-              executing: this.executing,
-              paused: this.paused,
-              completed: this.completed.size,
-              nextResumeAfter: this.nextResumeAfter
-            });
+            return reply.json(this.#status);
           }
           case "/connect": {
             this.#coordinatorSocket.connect();
@@ -1193,6 +1507,25 @@ var ProdWorker = class {
     const data = await response.json();
     return data?.variables ?? {};
   }
+  get #status() {
+    return {
+      executing: this.executing,
+      paused: this.paused,
+      completed: this.completed.size,
+      nextResumeAfter: this.nextResumeAfter,
+      waitForPostStart: this.waitForPostStart,
+      attemptFriendlyId: this.attemptFriendlyId
+    };
+  }
+  #emitUnrecoverableError(name, message) {
+    this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
+      version: "v1",
+      error: {
+        name,
+        message
+      }
+    });
+  }
   start() {
     this.#httpServer.listen(this.#httpPort, this.host);
   }