trigger.dev 3.0.0-beta.34 → 3.0.0-beta.35

package/dist/index.js

@@ -799,7 +799,7 @@ import invariant from "tiny-invariant";
 import { z as z4 } from "zod";
 
 // package.json
-var version = "3.0.0-beta.34";
+var version = "3.0.0-beta.35";
 var dependencies = {
   "@anatine/esbuild-decorators": "^0.2.19",
   "@clack/prompts": "^0.7.0",
@@ -816,7 +816,7 @@ var dependencies = {
   "@opentelemetry/sdk-trace-base": "^1.22.0",
   "@opentelemetry/sdk-trace-node": "^1.22.0",
   "@opentelemetry/semantic-conventions": "^1.22.0",
-  "@trigger.dev/core": "workspace:3.0.0-beta.34",
+  "@trigger.dev/core": "workspace:3.0.0-beta.35",
   "@types/degit": "^2.8.3",
   chalk: "^5.2.0",
   chokidar: "^3.5.3",
@@ -949,8 +949,11 @@ import {
   GetDeploymentResponseBody,
   GetProjectsResponseBody,
   GetProjectResponseBody,
-  EnvironmentVariableResponseBody
+  EnvironmentVariableResponseBody,
+  TaskRunExecution,
+  APIError
 } from "@trigger.dev/core/v3";
+import { zodfetch } from "@trigger.dev/core/v3/zodfetch";
 var CliApiClient = class {
   constructor(apiURL, accessToken) {
     this.accessToken = accessToken;
@@ -958,7 +961,7 @@ var CliApiClient = class {
   }
   apiURL;
   async createAuthorizationCode() {
-    return zodfetch(
+    return wrapZodFetch(
       CreateAuthorizationCodeResponseSchema,
       `${this.apiURL}/api/v1/authorization-code`,
       {
@@ -967,7 +970,7 @@ var CliApiClient = class {
     );
   }
   async getPersonalAccessToken(authorizationCode) {
-    return zodfetch(GetPersonalAccessTokenResponseSchema, `${this.apiURL}/api/v1/token`, {
+    return wrapZodFetch(GetPersonalAccessTokenResponseSchema, `${this.apiURL}/api/v1/token`, {
       method: "POST",
       body: JSON.stringify({
         authorizationCode
@@ -978,7 +981,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("whoAmI: No access token");
     }
-    return zodfetch(WhoAmIResponseSchema, `${this.apiURL}/api/v2/whoami`, {
+    return wrapZodFetch(WhoAmIResponseSchema, `${this.apiURL}/api/v2/whoami`, {
       headers: {
         Authorization: `Bearer ${this.accessToken}`,
         "Content-Type": "application/json"
@@ -989,7 +992,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("getProject: No access token");
     }
-    return zodfetch(GetProjectResponseBody, `${this.apiURL}/api/v1/projects/${projectRef}`, {
+    return wrapZodFetch(GetProjectResponseBody, `${this.apiURL}/api/v1/projects/${projectRef}`, {
       headers: {
         Authorization: `Bearer ${this.accessToken}`,
         "Content-Type": "application/json"
@@ -1000,7 +1003,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("getProjects: No access token");
     }
-    return zodfetch(GetProjectsResponseBody, `${this.apiURL}/api/v1/projects`, {
+    return wrapZodFetch(GetProjectsResponseBody, `${this.apiURL}/api/v1/projects`, {
       headers: {
         Authorization: `Bearer ${this.accessToken}`,
         "Content-Type": "application/json"
@@ -1011,7 +1014,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("createBackgroundWorker: No access token");
     }
-    return zodfetch(
+    return wrapZodFetch(
       CreateBackgroundWorkerResponse,
       `${this.apiURL}/api/v1/projects/${projectRef}/background-workers`,
       {
@@ -1024,6 +1027,18 @@ var CliApiClient = class {
       }
     );
   }
+  async createTaskRunAttempt(runFriendlyId) {
+    if (!this.accessToken) {
+      throw new Error("creatTaskRunAttempt: No access token");
+    }
+    return wrapZodFetch(TaskRunExecution, `${this.apiURL}/api/v1/runs/${runFriendlyId}/attempts`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.accessToken}`,
+        "Content-Type": "application/json"
+      }
+    });
+  }
   async getProjectEnv({
     projectRef,
     env
@@ -1031,18 +1046,22 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("getProjectDevEnv: No access token");
     }
-    return zodfetch(GetProjectEnvResponse, `${this.apiURL}/api/v1/projects/${projectRef}/${env}`, {
-      headers: {
-        Authorization: `Bearer ${this.accessToken}`,
-        "Content-Type": "application/json"
+    return wrapZodFetch(
+      GetProjectEnvResponse,
+      `${this.apiURL}/api/v1/projects/${projectRef}/${env}`,
+      {
+        headers: {
+          Authorization: `Bearer ${this.accessToken}`,
+          "Content-Type": "application/json"
+        }
       }
-    });
+    );
   }
   async getEnvironmentVariables(projectRef) {
     if (!this.accessToken) {
       throw new Error("getEnvironmentVariables: No access token");
     }
-    return zodfetch(
+    return wrapZodFetch(
       GetEnvironmentVariablesResponseBody,
       `${this.apiURL}/api/v1/projects/${projectRef}/envvars`,
       {
@@ -1057,7 +1076,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("importEnvVars: No access token");
     }
-    return zodfetch(
+    return wrapZodFetch(
       EnvironmentVariableResponseBody,
       `${this.apiURL}/api/v1/projects/${projectRef}/envvars/${slug}/import`,
       {
@@ -1074,7 +1093,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("initializeDeployment: No access token");
     }
-    return zodfetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, {
+    return wrapZodFetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, {
       method: "POST",
       headers: {
         Authorization: `Bearer ${this.accessToken}`,
@@ -1087,7 +1106,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("startDeploymentIndexing: No access token");
     }
-    return zodfetch(
+    return wrapZodFetch(
       StartDeploymentIndexingResponseBody,
       `${this.apiURL}/api/v1/deployments/${deploymentId}/start-indexing`,
       {
@@ -1104,7 +1123,7 @@ var CliApiClient = class {
     if (!this.accessToken) {
       throw new Error("getDeployment: No access token");
     }
-    return zodfetch(
+    return wrapZodFetch(
       GetDeploymentResponseBody,
       `${this.apiURL}/api/v1/deployments/${deploymentId}`,
       {
@@ -1116,45 +1135,38 @@ var CliApiClient = class {
     );
   }
 };
-async function zodfetch(schema, url, requestInit) {
+async function wrapZodFetch(schema, url, requestInit) {
   try {
-    const response = await fetch(url, requestInit);
-    if ((!requestInit || requestInit.method === "GET") && response.status === 404) {
+    const response = await zodfetch(schema, url, requestInit, {
+      retry: {
+        minTimeoutInMs: 500,
+        maxTimeoutInMs: 5e3,
+        maxAttempts: 3,
+        factor: 2,
+        randomize: false
+      }
+    });
+    return {
+      success: true,
+      data: response
+    };
+  } catch (error) {
+    if (error instanceof APIError) {
       return {
         success: false,
-        error: `404: ${response.statusText}`
+        error: error.message
       };
-    }
-    if (response.status >= 400 && response.status < 500) {
-      const body = await response.json();
-      if (!body.error) {
-        return { success: false, error: "Something went wrong" };
-      }
-      return { success: false, error: body.error };
-    }
-    if (response.status !== 200) {
+    } else if (error instanceof Error) {
       return {
         success: false,
-        error: `Failed to fetch ${url}, got status code ${response.status}`
+        error: error.message
       };
-    }
-    const jsonBody = await response.json();
-    const parsedResult = schema.safeParse(jsonBody);
-    if (parsedResult.success) {
-      return { success: true, data: parsedResult.data };
-    }
-    if ("error" in jsonBody) {
+    } else {
       return {
         success: false,
-        error: typeof jsonBody.error === "string" ? jsonBody.error : JSON.stringify(jsonBody.error)
+        error: String(error)
       };
     }
-    return { success: false, error: parsedResult.error.message };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : JSON.stringify(error)
-    };
   }
 }
 
@@ -2599,7 +2611,7 @@ async function whoAmI(options, embedded = false) {
         loadingSpinner.stop(
           `You must login first. Use \`trigger.dev login --profile ${options?.profile ?? "default"}\` to login.`
         );
-        outro2("Whoami failed");
+        outro2(`Whoami failed: ${authentication.error}`);
       }
     }
     return {
@@ -4003,7 +4015,11 @@ async function _deployCommand(dir, options) {
         `Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?`
       );
     } else {
-      throw new Error("You must login first. Use `trigger.dev login` to login.");
+      throw new Error(
+        `You must login first. Use the \`login\` CLI command.
+
+${authorization.error}`
+      );
     }
   }
   span?.setAttributes({
@@ -4363,7 +4379,7 @@ async function failDeploy(shortCode, errorSummary, logs, deploymentSpinner, warn
     outro5(`${chalkError("Error:")} ${errorSummary}.`);
   }
 }
-async function waitForDeploymentToFinish(deploymentId, client, timeoutInSeconds = 60) {
+async function waitForDeploymentToFinish(deploymentId, client, timeoutInSeconds = 180) {
   return tracer.startActiveSpan("waitForDeploymentToFinish", async (span) => {
     try {
       const start = Date.now();
@@ -5274,6 +5290,31 @@ var TaskMetadataParseError = class extends Error {
     this.name = "TaskMetadataParseError";
   }
 };
+var UnexpectedExitError = class extends Error {
+  constructor(code) {
+    super(`Unexpected exit with code ${code}`);
+    this.code = code;
+    this.name = "UnexpectedExitError";
+  }
+};
+var CleanupProcessError = class extends Error {
+  constructor() {
+    super("Cancelled");
+    this.name = "CleanupProcessError";
+  }
+};
+var CancelledProcessError = class extends Error {
+  constructor() {
+    super("Cancelled");
+    this.name = "CancelledProcessError";
+  }
+};
+var SigKillTimeoutProcessError = class extends Error {
+  constructor() {
+    super("Process kill timeout");
+    this.name = "SigKillTimeoutProcessError";
+  }
+};
 
 // src/workers/dev/backgroundWorker.ts
 import {
@@ -5293,23 +5334,31 @@ import terminalLink3 from "terminal-link";
 var BackgroundWorkerCoordinator = class {
   constructor(baseURL) {
     this.baseURL = baseURL;
-    this.onTaskCompleted.attach(async ({ completion, execution }) => {
+    this.onTaskCompleted.attach(async ({ completion }) => {
       if (!completion.ok && typeof completion.retry !== "undefined") {
         return;
       }
-      await this.#notifyWorkersOfTaskCompletion(completion, execution);
+      await this.#notifyWorkersOfTaskCompletion(completion);
+    });
+    this.onTaskFailedToRun.attach(async ({ completion }) => {
+      await this.#notifyWorkersOfTaskCompletion(completion);
     });
   }
   onTaskCompleted = new Evt();
+  onTaskFailedToRun = new Evt();
   onWorkerRegistered = new Evt();
+  /**
+   * @deprecated use onWorkerTaskRunHeartbeat instead
+   */
   onWorkerTaskHeartbeat = new Evt();
+  onWorkerTaskRunHeartbeat = new Evt();
   onWorkerDeprecated = new Evt();
   _backgroundWorkers = /* @__PURE__ */ new Map();
   _records = /* @__PURE__ */ new Map();
   _deprecatedWorkers = /* @__PURE__ */ new Set();
-  async #notifyWorkersOfTaskCompletion(completion, execution) {
+  async #notifyWorkersOfTaskCompletion(completion) {
     for (const worker of this._backgroundWorkers.values()) {
-      await worker.taskRunCompletedNotification(completion, execution);
+      await worker.taskRunCompletedNotification(completion);
     }
   }
   get currentWorkers() {
@@ -5334,6 +5383,9 @@ var BackgroundWorkerCoordinator = class {
     worker.onTaskHeartbeat.attach((id) => {
       this.onWorkerTaskHeartbeat.post({ id, backgroundWorkerId: record.id, worker });
     });
+    worker.onTaskRunHeartbeat.attach((id) => {
+      this.onWorkerTaskRunHeartbeat.post({ id, backgroundWorkerId: record.id, worker });
+    });
   }
   close() {
     for (const worker of this._backgroundWorkers.values()) {
@@ -5356,10 +5408,14 @@ var BackgroundWorkerCoordinator = class {
           return;
         }
         await worker.cancelRun(message.taskRunId);
+        break;
+      }
+      case "EXECUTE_RUN_LAZY_ATTEMPT": {
+        await this.#executeTaskRunLazyAttempt(id, message.payload);
       }
     }
   }
-  async #executeTaskRun(id, payload) {
+  async #executeTaskRunLazyAttempt(id, payload) {
     const worker = this._backgroundWorkers.get(id);
     if (!worker) {
       logger.error(`Could not find worker ${id}`);
@@ -5370,96 +5426,80 @@ var BackgroundWorkerCoordinator = class {
       logger.error(`Could not find worker record ${id}`);
       return;
     }
-    const { execution } = payload;
-    const logsUrl = `${this.baseURL}/runs/${execution.run.id}`;
-    const pipe = chalkGrey("|");
-    const bullet = chalkGrey("\u25CB");
-    const link = chalkLink(terminalLink3("View logs", logsUrl));
-    let timestampPrefix = chalkGrey(prettyPrintDate(payload.execution.attempt.startedAt));
-    const workerPrefix = chalkWorker(record.version);
-    const taskPrefix = chalkTask(execution.task.id);
-    const runId = chalkRun(`${execution.run.id}.${execution.attempt.number}`);
-    logger.log(
-      `${bullet} ${timestampPrefix} ${chalkGrey(
-        "->"
-      )} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId}`
-    );
-    const now = performance.now();
-    const completion = await worker.executeTaskRun(payload);
-    const elapsed = performance.now() - now;
-    const retryingText = chalkGrey(
-      !completion.ok && completion.skippedRetrying ? " (retrying skipped)" : !completion.ok && completion.retry !== void 0 ? ` (retrying in ${completion.retry.delay}ms)` : ""
-    );
-    const resultText = !completion.ok ? completion.error.type === "INTERNAL_ERROR" && (completion.error.code === TaskRunErrorCodes.TASK_EXECUTION_ABORTED || completion.error.code === TaskRunErrorCodes.TASK_RUN_CANCELLED) ? chalkWarning("Cancelled") : `${chalkError("Error")}${retryingText}` : chalkSuccess("Success");
-    const errorText = !completion.ok ? this.#formatErrorLog(completion.error) : "retry" in completion ? `retry in ${completion.retry}ms` : "";
-    const elapsedText = chalkGrey(`(${formatDurationMilliseconds(elapsed, { style: "short" })})`);
-    timestampPrefix = chalkGrey(prettyPrintDate());
-    logger.log(
-      `${bullet} ${timestampPrefix} ${chalkGrey(
-        "->"
-      )} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId} ${pipe} ${resultText} ${elapsedText}${errorText}`
-    );
-    this.onTaskCompleted.post({ completion, execution, worker, backgroundWorkerId: id });
-  }
-  #formatErrorLog(error) {
-    switch (error.type) {
-      case "INTERNAL_ERROR": {
-        return "";
-      }
-      case "STRING_ERROR": {
-        return `
-
-${chalkError("X Error:")} ${error.raw}
-`;
-      }
-      case "CUSTOM_ERROR": {
-        return `
-
-${chalkError("X Error:")} ${error.raw}
-`;
-      }
-      case "BUILT_IN_ERROR": {
-        return `
-
-${error.stackTrace.replace(/^Error: /, chalkError("X Error: "))}
-`;
-      }
+    try {
+      const { completion, execution } = await worker.executeTaskRunLazyAttempt(
+        payload,
+        this.baseURL
+      );
+      this.onTaskCompleted.post({
+        completion,
+        execution,
+        worker,
+        backgroundWorkerId: id
+      });
+    } catch (error) {
+      this.onTaskFailedToRun.post({
+        backgroundWorkerId: id,
+        worker,
+        completion: {
+          ok: false,
+          id: payload.runId,
+          retry: void 0,
+          error: error instanceof Error ? {
+            type: "BUILT_IN_ERROR",
+            name: error.name,
+            message: error.message,
+            stackTrace: error.stack ?? ""
+          } : {
+            type: "BUILT_IN_ERROR",
+            name: "UnknownError",
+            message: String(error),
+            stackTrace: ""
+          }
+        }
+      });
     }
   }
-};
-var UnexpectedExitError = class extends Error {
-  constructor(code) {
-    super(`Unexpected exit with code ${code}`);
-    this.code = code;
-    this.name = "UnexpectedExitError";
-  }
-};
-var CleanupProcessError = class extends Error {
-  constructor() {
-    super("Cancelled");
-    this.name = "CleanupProcessError";
-  }
-};
-var CancelledProcessError = class extends Error {
-  constructor() {
-    super("Cancelled");
-    this.name = "CancelledProcessError";
+  async #executeTaskRun(id, payload) {
+    const worker = this._backgroundWorkers.get(id);
+    if (!worker) {
+      logger.error(`Could not find worker ${id}`);
+      return;
+    }
+    const record = this._records.get(id);
+    if (!record) {
+      logger.error(`Could not find worker record ${id}`);
+      return;
+    }
+    const completion = await worker.executeTaskRun(payload, this.baseURL);
+    this.onTaskCompleted.post({
+      completion,
+      execution: payload.execution,
+      worker,
+      backgroundWorkerId: id
+    });
   }
 };
 var BackgroundWorker = class {
-  constructor(path7, params) {
+  constructor(path7, params, apiClient2) {
     this.path = path7;
     this.params = params;
+    this.apiClient = apiClient2;
   }
   _initialized = false;
   _handler = new ZodMessageHandler({
     schema: childToWorkerMessages
   });
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   _onClose = new Evt();
   tasks = [];
   metadata;
   _taskRunProcesses = /* @__PURE__ */ new Map();
+  _taskRunProcessesBeingKilled = /* @__PURE__ */ new Set();
   _closed = false;
   _fullEnv = {};
   close() {
@@ -5468,6 +5508,7 @@ var BackgroundWorker = class {
     }
     this._closed = true;
     this.onTaskHeartbeat.detach();
+    this.onTaskRunHeartbeat.detach();
     for (const taskRunProcess of this._taskRunProcesses.values()) {
       taskRunProcess.cleanup(true);
     }
@@ -5475,6 +5516,9 @@ var BackgroundWorker = class {
     safeDeleteFileSync(this.path);
     safeDeleteFileSync(`${this.path}.map`);
   }
+  get inProgressRuns() {
+    return Array.from(this._taskRunProcesses.keys());
+  }
   async initialize() {
     if (this._initialized) {
       throw new Error("Worker already initialized");
@@ -5557,36 +5601,105 @@ var BackgroundWorker = class {
   }
   // We need to notify all the task run processes that a task run has completed,
   // in case they are waiting for it through triggerAndWait
-  async taskRunCompletedNotification(completion, execution) {
+  async taskRunCompletedNotification(completion) {
     for (const taskRunProcess of this._taskRunProcesses.values()) {
-      taskRunProcess.taskRunCompletedNotification(completion, execution);
+      taskRunProcess.taskRunCompletedNotification(completion);
     }
   }
-  async #initializeTaskRunProcess(payload) {
+  #prefixedMessage(payload, message = "") {
+    return `[${payload.execution.run.id}.${payload.execution.attempt.number}] ${message}`;
+  }
+  async #getFreshTaskRunProcess(payload, messageId) {
+    logger.debug(this.#prefixedMessage(payload, "getFreshTaskRunProcess()"));
     if (!this.metadata) {
       throw new Error("Worker not registered");
     }
-    if (!this._taskRunProcesses.has(payload.execution.run.id)) {
-      const taskRunProcess = new TaskRunProcess(
-        payload.execution,
-        this.path,
-        {
-          ...this._fullEnv,
-          ...payload.environment ?? {}
-        },
-        this.metadata,
-        this.params
-      );
-      taskRunProcess.onExit.attach(() => {
+    this._closed = false;
+    logger.debug(this.#prefixedMessage(payload, "killing current task run process before attempt"));
+    await this.#killCurrentTaskRunProcessBeforeAttempt(payload.execution.run.id);
+    logger.debug(this.#prefixedMessage(payload, "creating new task run process"));
+    const taskRunProcess = new TaskRunProcess(
+      payload.execution.run.id,
+      payload.execution.run.isTest,
+      this.path,
+      {
+        ...this._fullEnv,
+        ...payload.environment ?? {},
+        ...this.#readEnvVars()
+      },
+      this.metadata,
+      this.params,
+      messageId
+    );
+    taskRunProcess.onExit.attach(({ pid }) => {
+      logger.debug(this.#prefixedMessage(payload, "onExit()"), { pid });
+      const taskRunProcess2 = this._taskRunProcesses.get(payload.execution.run.id);
+      if (taskRunProcess2?.pid === pid) {
         this._taskRunProcesses.delete(payload.execution.run.id);
-      });
-      taskRunProcess.onTaskHeartbeat.attach((id) => {
-        this.onTaskHeartbeat.post(id);
-      });
-      await taskRunProcess.initialize();
-      this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess);
+      }
+      if (pid) {
+        this._taskRunProcessesBeingKilled.delete(pid);
+      }
+    });
+    taskRunProcess.onIsBeingKilled.attach((pid) => {
+      if (pid) {
+        this._taskRunProcessesBeingKilled.add(pid);
+      }
+    });
+    taskRunProcess.onTaskHeartbeat.attach((id) => {
+      this.onTaskHeartbeat.post(id);
+    });
+    taskRunProcess.onTaskRunHeartbeat.attach((id) => {
+      this.onTaskRunHeartbeat.post(id);
+    });
+    await taskRunProcess.initialize();
+    this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess);
+    return taskRunProcess;
+  }
+  async #killCurrentTaskRunProcessBeforeAttempt(runId) {
+    const taskRunProcess = this._taskRunProcesses.get(runId);
+    if (!taskRunProcess) {
+      logger.debug(`[${runId}] no current task process to kill`);
+      return;
+    }
+    logger.debug(`[${runId}] killing current task process`, {
+      pid: taskRunProcess.pid
+    });
+    if (taskRunProcess.isBeingKilled) {
+      if (this._taskRunProcessesBeingKilled.size > 1) {
+        await this.#tryGracefulExit(taskRunProcess);
+      } else {
+      }
+    } else {
+      if (this._taskRunProcessesBeingKilled.size > 0) {
+        await this.#tryGracefulExit(taskRunProcess);
+      } else {
+        taskRunProcess.kill("SIGTERM", 5e3).catch(() => {
+        });
+      }
+    }
+  }
+  async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
+    try {
+      const initialExit = taskRunProcess.onExit.waitFor(5e3);
+      if (kill) {
+        taskRunProcess.kill(initialSignal);
+      }
+      await initialExit;
+    } catch (error) {
+      logger.error("TaskRunProcess graceful kill timeout exceeded", error);
+      this.#tryForcefulExit(taskRunProcess);
+    }
+  }
+  async #tryForcefulExit(taskRunProcess) {
+    try {
+      const forcedKill = taskRunProcess.onExit.waitFor(5e3);
+      taskRunProcess.kill("SIGKILL");
+      await forcedKill;
+    } catch (error) {
+      logger.error("TaskRunProcess forced kill timeout exceeded", error);
+      throw new SigKillTimeoutProcessError();
     }
-    return this._taskRunProcesses.get(payload.execution.run.id);
   }
   async cancelRun(taskRunId) {
     const taskRunProcess = this._taskRunProcesses.get(taskRunId);
@@ -5595,12 +5708,66 @@ var BackgroundWorker = class {
     }
     await taskRunProcess.cancel();
   }
+  async executeTaskRunLazyAttempt(payload, baseURL) {
+    const attemptResponse = await this.apiClient.createTaskRunAttempt(payload.runId);
+    if (!attemptResponse.success) {
+      throw new Error(`Failed to create task run attempt: ${attemptResponse.error}`);
+    }
+    const execution = attemptResponse.data;
+    const completion = await this.executeTaskRun(
+      { execution, traceContext: payload.traceContext, environment: payload.environment },
+      baseURL,
+      payload.messageId
+    );
+    return { execution, completion };
+  }
   // We need to fork the process before we can execute any tasks
-  async executeTaskRun(payload) {
+  async executeTaskRun(payload, baseURL, messageId) {
+    if (this._closed) {
+      throw new Error("Worker is closed");
+    }
+    if (!this.metadata) {
+      throw new Error("Worker not registered");
+    }
+    const { execution } = payload;
+    const logsUrl = `${baseURL}/runs/${execution.run.id}`;
+    const pipe = chalkGrey("|");
+    const bullet = chalkGrey("\u25CB");
+    const link = chalkLink(terminalLink3("View logs", logsUrl));
+    let timestampPrefix = chalkGrey(prettyPrintDate(payload.execution.attempt.startedAt));
+    const workerPrefix = chalkWorker(this.metadata.version);
+    const taskPrefix = chalkTask(execution.task.id);
+    const runId = chalkRun(`${execution.run.id}.${execution.attempt.number}`);
+    logger.log(
+      `${bullet} ${timestampPrefix} ${chalkGrey(
+        "->"
+      )} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId}`
+    );
+    const now = performance.now();
+    const completion = await this.#doExecuteTaskRun(payload, messageId);
+    const elapsed = performance.now() - now;
+    const retryingText = chalkGrey(
+      !completion.ok && completion.skippedRetrying ? " (retrying skipped)" : !completion.ok && completion.retry !== void 0 ? ` (retrying in ${completion.retry.delay}ms)` : ""
+    );
+    const resultText = !completion.ok ? completion.error.type === "INTERNAL_ERROR" && (completion.error.code === TaskRunErrorCodes.TASK_EXECUTION_ABORTED || completion.error.code === TaskRunErrorCodes.TASK_RUN_CANCELLED) ? chalkWarning("Cancelled") : `${chalkError("Error")}${retryingText}` : chalkSuccess("Success");
+    const errorText = !completion.ok ? formatErrorLog(completion.error) : "retry" in completion ? `retry in ${completion.retry}ms` : "";
+    const elapsedText = chalkGrey(`(${formatDurationMilliseconds(elapsed, { style: "short" })})`);
+    timestampPrefix = chalkGrey(prettyPrintDate());
+    logger.log(
+      `${bullet} ${timestampPrefix} ${chalkGrey(
+        "->"
+      )} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId} ${pipe} ${resultText} ${elapsedText}${errorText}`
+    );
+    return completion;
+  }
+  async #doExecuteTaskRun(payload, messageId) {
     try {
-      const taskRunProcess = await this.#initializeTaskRunProcess(payload);
+      const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
+      logger.debug(this.#prefixedMessage(payload, "executing task run"), {
+        pid: taskRunProcess.pid
+      });
       const result = await taskRunProcess.executeTaskRun(payload);
-      await taskRunProcess.cleanup(result.ok || result.retry === void 0);
+      await taskRunProcess.cleanup(true);
       if (result.ok) {
         return result;
       }
@@ -5677,12 +5844,14 @@ var BackgroundWorker = class {
   }
 };
 var TaskRunProcess = class {
-  constructor(execution, path7, env, metadata, worker) {
-    this.execution = execution;
+  constructor(runId, isTest, path7, env, metadata, worker, messageId) {
+    this.runId = runId;
+    this.isTest = isTest;
     this.path = path7;
     this.env = env;
     this.metadata = metadata;
     this.worker = worker;
+    this.messageId = messageId;
     this._sender = new ZodMessageSender({
       schema: workerToChildMessages,
       sender: async (message) => {
@@ -5697,20 +5866,26 @@ var TaskRunProcess = class {
   });
   _sender;
   _child;
+  _childPid;
   _attemptPromises = /* @__PURE__ */ new Map();
   _attemptStatuses = /* @__PURE__ */ new Map();
   _currentExecution;
   _isBeingKilled = false;
   _isBeingCancelled = false;
+  /**
+   * @deprecated use onTaskRunHeartbeat instead
+   */
   onTaskHeartbeat = new Evt();
+  onTaskRunHeartbeat = new Evt();
   onExit = new Evt();
+  onIsBeingKilled = new Evt();
   async cancel() {
     this._isBeingCancelled = true;
     await this.cleanup(true);
   }
   async initialize() {
     const fullEnv = {
-      ...this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
+      ...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
       ...this.env,
       OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
         [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
@@ -5719,7 +5894,7 @@ var TaskRunProcess = class {
       ...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
     };
     const cwd = dirname2(this.path);
-    logger.debug(`[${this.execution.run.id}] initializing task run process`, {
+    logger.debug(`[${this.runId}] initializing task run process`, {
       env: fullEnv,
       path: this.path,
       cwd
@@ -5738,6 +5913,7 @@ var TaskRunProcess = class {
       env: fullEnv,
       execArgv: this.worker.debuggerOn ? ["--inspect-brk", "--trace-uncaught", "--no-warnings=ExperimentalWarning"] : ["--trace-uncaught", "--no-warnings=ExperimentalWarning"]
     });
+    this._childPid = this._child?.pid;
     this._child.on("message", this.#handleMessage.bind(this));
     this._child.on("exit", this.#handleExit.bind(this));
     this._child.stdout?.on("data", this.#handleLog.bind(this));
@@ -5747,15 +5923,21 @@ var TaskRunProcess = class {
     if (kill && this._isBeingKilled) {
       return;
     }
-    logger.debug(`[${this.execution.run.id}] cleaning up task run process`, { kill });
+    if (kill) {
+      this._isBeingKilled = true;
+      this.onIsBeingKilled.post(this._child?.pid);
+    }
+    logger.debug(`[${this.runId}] cleaning up task run process`, { kill, pid: this.pid });
     await this._sender.send("CLEANUP", {
       flush: true,
       kill
     });
-    this._isBeingKilled = kill;
+    if (!kill) {
+      return;
+    }
     setTimeout(() => {
       if (this._child && !this._child.killed) {
-        logger.debug(`[${this.execution.run.id}] killing task run process after timeout`);
+        logger.debug(`[${this.runId}] killing task run process after timeout`, { pid: this.pid });
         this._child.kill();
       }
     }, 5e3);
@@ -5780,20 +5962,19 @@ var TaskRunProcess = class {
     this._currentExecution = void 0;
     return result;
   }
-  taskRunCompletedNotification(completion, execution) {
+  taskRunCompletedNotification(completion) {
     if (!completion.ok && typeof completion.retry !== "undefined") {
       return;
     }
-    if (execution.run.id === this.execution.run.id) {
+    if (completion.id === this.runId) {
       return;
     }
-    logger.debug(`[${this.execution.run.id}] task run completed notification`, {
-      completion,
-      execution
+    logger.debug(`[${this.runId}] task run completed notification`, {
+      completion
     });
     this._sender.send("TASK_RUN_COMPLETED_NOTIFICATION", {
-      completion,
-      execution
+      version: "v2",
+      completion
     });
   }
   async #handleMessage(msg) {
@@ -5815,12 +5996,16 @@ var TaskRunProcess = class {
         break;
       }
       case "READY_TO_DISPOSE": {
-        logger.debug(`[${this.execution.run.id}] task run process is ready to dispose`);
+        logger.debug(`[${this.runId}] task run process is ready to dispose`);
         this.#kill();
         break;
       }
       case "TASK_HEARTBEAT": {
-        this.onTaskHeartbeat.post(message.payload.id);
+        if (this.messageId) {
+          this.onTaskRunHeartbeat.post(this.messageId);
+        } else {
+          this.onTaskHeartbeat.post(message.payload.id);
+        }
         break;
       }
       case "TASKS_READY": {
@@ -5828,8 +6013,8 @@ var TaskRunProcess = class {
       }
     }
   }
-  async #handleExit(code) {
-    logger.debug(`[${this.execution.run.id}] task run process exiting`, { code });
+  async #handleExit(code, signal) {
+    logger.debug(`[${this.runId}] handle task run process exit`, { code, signal, pid: this.pid });
     for (const [id, status] of this._attemptStatuses.entries()) {
       if (status === "PENDING") {
         this._attemptStatuses.set(id, "REJECTED");
@@ -5843,11 +6028,11 @@ var TaskRunProcess = class {
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code));
+          rejecter(new UnexpectedExitError(code ?? -1));
         }
       }
     }
-    this.onExit.post(code);
+    this.onExit.post({ code, signal, pid: this.pid });
   }
   #handleLog(data) {
     if (!this._currentExecution) {
@@ -5877,12 +6062,57 @@ var TaskRunProcess = class {
     );
   }
   #kill() {
+    logger.debug(`[${this.runId}] #kill()`, { pid: this.pid });
     if (this._child && !this._child.killed) {
-      logger.debug(`[${this.execution.run.id}] killing task run process`);
       this._child?.kill();
     }
   }
+  async kill(signal, timeoutInMs) {
+    logger.debug(`[${this.runId}] killing task run process`, {
+      signal,
+      timeoutInMs,
+      pid: this.pid
+    });
+    this._isBeingKilled = true;
+    const killTimeout = this.onExit.waitFor(timeoutInMs);
+    this.onIsBeingKilled.post(this._child?.pid);
+    this._child?.kill(signal);
+    if (timeoutInMs) {
+      await killTimeout;
+    }
+  }
+  get isBeingKilled() {
+    return this._isBeingKilled || this._child?.killed;
+  }
+  get pid() {
+    return this._childPid;
+  }
 };
+function formatErrorLog(error) {
+  switch (error.type) {
+    case "INTERNAL_ERROR": {
+      return "";
+    }
+    case "STRING_ERROR": {
+      return `
+
+${chalkError("X Error:")} ${error.raw}
+`;
+    }
+    case "CUSTOM_ERROR": {
+      return `
+
+${chalkError("X Error:")} ${error.raw}
+`;
+    }
+    case "BUILT_IN_ERROR": {
+      return `
+
+${error.stackTrace.replace(/^Error: /, chalkError("X Error: "))}
+`;
+    }
+  }
+}
 
 // src/utilities/runtimeCheck.ts
 function runtimeCheck(minimumMajor, minimumMinor) {
@@ -5945,7 +6175,11 @@ async function devCommand(dir, options) {
         )} Connecting to the server failed. Please check your internet connection or contact eric@trigger.dev for help.`
       );
     } else {
-      logger.log(`${chalkError("X Error:")} You must login first. Use the \`login\` CLI command.`);
+      logger.log(
+        `${chalkError("X Error:")} You must login first. Use the \`login\` CLI command.
+
+${authorization.error}`
+      );
     }
     process.exitCode = 1;
     return;
@@ -6084,6 +6318,17 @@ function useDev({
         });
       }
     );
+    backgroundWorkerCoordinator.onWorkerTaskRunHeartbeat.attach(
+      async ({ worker, backgroundWorkerId, id }) => {
+        await sender.send("BACKGROUND_WORKER_MESSAGE", {
+          backgroundWorkerId,
+          data: {
+            type: "TASK_RUN_HEARTBEAT",
+            id
+          }
+        });
+      }
+    );
     backgroundWorkerCoordinator.onTaskCompleted.attach(
       async ({ backgroundWorkerId, completion, execution }) => {
         await sender.send("BACKGROUND_WORKER_MESSAGE", {
@@ -6096,6 +6341,17 @@ function useDev({
         });
       }
     );
+    backgroundWorkerCoordinator.onTaskFailedToRun.attach(
+      async ({ backgroundWorkerId, completion }) => {
+        await sender.send("BACKGROUND_WORKER_MESSAGE", {
+          backgroundWorkerId,
+          data: {
+            type: "TASK_RUN_FAILED_TO_RUN",
+            completion
+          }
+        });
+      }
+    );
     backgroundWorkerCoordinator.onWorkerRegistered.attach(async ({ id, worker, record }) => {
       await sender.send("READY_FOR_TASKS", {
         backgroundWorkerId: id
@@ -6116,7 +6372,8 @@ function useDev({
           SERVER_READY: async (payload) => {
             for (const worker of backgroundWorkerCoordinator.currentWorkers) {
               await sender.send("READY_FOR_TASKS", {
-                backgroundWorkerId: worker.id
+                backgroundWorkerId: worker.id,
+                inProgressRuns: worker.worker.inProgressRuns
               });
             }
           },
@@ -6252,19 +6509,23 @@ function useDev({
                 }
                 const environmentVariablesResponse = await environmentClient.getEnvironmentVariables(config.project);
                 const processEnv = await gatherProcessEnv();
-                const backgroundWorker = new BackgroundWorker(fullPath, {
-                  projectConfig: config,
-                  dependencies: dependencies2,
-                  env: {
-                    ...processEnv,
-                    TRIGGER_API_URL: apiUrl,
-                    TRIGGER_SECRET_KEY: apiKey,
-                    ...environmentVariablesResponse.success ? environmentVariablesResponse.data.variables : {}
+                const backgroundWorker = new BackgroundWorker(
+                  fullPath,
+                  {
+                    projectConfig: config,
+                    dependencies: dependencies2,
+                    env: {
+                      ...processEnv,
+                      TRIGGER_API_URL: apiUrl,
+                      TRIGGER_SECRET_KEY: apiKey,
+                      ...environmentVariablesResponse.success ? environmentVariablesResponse.data.variables : {}
+                    },
+                    debuggerOn,
+                    debugOtel,
+                    resolveEnvVariables: createResolveEnvironmentVariablesFunction(configModule)
                   },
-                  debuggerOn,
-                  debugOtel,
-                  resolveEnvVariables: createResolveEnvironmentVariablesFunction(configModule)
-                });
+                  environmentClient
+                );
                 try {
                   await backgroundWorker.initialize();
                   latestWorkerContentHash = contentHash;
@@ -6307,7 +6568,8 @@ function useDev({
                       cliPackageVersion: version,
                       tasks: taskResources,
                       contentHash
-                    }
+                    },
+                    supportsLazyAttempts: true
                   };
                   const backgroundWorkerRecord = await environmentClient.createBackgroundWorker(
                     config.project,
@@ -6495,18 +6757,9 @@ ${task.filePath} -> ${task.exportName}`).join("")}`;
 }
 async function gatherProcessEnv() {
   const env = {
+    ...process.env,
     NODE_ENV: process.env.NODE_ENV ?? "development",
-    PATH: process.env.PATH,
-    USER: process.env.USER,
-    SHELL: process.env.SHELL,
-    NVM_INC: process.env.NVM_INC,
-    NVM_DIR: process.env.NVM_DIR,
-    NVM_BIN: process.env.NVM_BIN,
-    LANG: process.env.LANG,
-    TERM: process.env.TERM,
-    NODE_PATH: await amendNodePathWithPnpmNodeModules(process.env.NODE_PATH),
-    HOME: process.env.HOME,
-    BUN_INSTALL: process.env.BUN_INSTALL
+    NODE_PATH: await amendNodePathWithPnpmNodeModules(process.env.NODE_PATH)
   };
   return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== void 0));
 }