trigger.dev 0.0.0-v3-prerelease-20250108142426 → 0.0.0-v4-prerelease-20250916125920

package/dist/esm/entryPoints/dev-run-controller.js

@@ -0,0 +1,652 @@
+import { IntervalService, SuspendedProcessError, } from "@trigger.dev/core/v3";
+import { setTimeout as sleep } from "timers/promises";
+import { TaskRunProcess } from "../executions/taskRunProcess.js";
+import { assertExhaustive } from "../utilities/assertExhaustive.js";
+import { logger } from "../utilities/logger.js";
+import { sanitizeEnvVars } from "../utilities/sanitizeEnvVars.js";
+import { join } from "node:path";
+import { eventBus } from "../utilities/eventBus.js";
+export class DevRunController {
+    opts;
+    taskRunProcess;
+    worker;
+    httpClient;
+    snapshotPoller;
+    snapshotPollIntervalSeconds;
+    cwd;
+    isCompletingRun = false;
+    isShuttingDown = false;
+    state = { phase: "IDLE" };
+    enterRunPhase(run, snapshot) {
+        this.onExitRunPhase(run);
+        this.state = { phase: "RUN", run, snapshot };
+        this.snapshotPoller.start();
+    }
+    constructor(opts) {
+        this.opts = opts;
+        logger.debug("[DevRunController] Creating controller", {
+            run: opts.runFriendlyId,
+        });
+        this.worker = opts.worker;
+        this.snapshotPollIntervalSeconds = 5;
+        this.cwd = opts.cwd;
+        this.httpClient = opts.httpClient;
+        this.snapshotPoller = new IntervalService({
+            onInterval: async () => {
+                if (!this.runFriendlyId) {
+                    logger.debug("[DevRunController] Skipping snapshot poll, no run ID");
+                    return;
+                }
+                logger.debug("[DevRunController] Polling for latest snapshot");
+                this.httpClient.dev.sendDebugLog(this.runFriendlyId, {
+                    time: new Date(),
+                    message: `snapshot poll: started`,
+                    properties: {
+                        snapshotId: this.snapshotFriendlyId,
+                    },
+                });
+                const response = await this.httpClient.dev.getRunExecutionData(this.runFriendlyId);
+                if (!response.success) {
+                    logger.debug("[DevRunController] Snapshot poll failed", { error: response.error });
+                    this.httpClient.dev.sendDebugLog(this.runFriendlyId, {
+                        time: new Date(),
+                        message: `snapshot poll: failed`,
+                        properties: {
+                            snapshotId: this.snapshotFriendlyId,
+                            error: response.error,
+                        },
+                    });
+                    return;
+                }
+                await this.handleSnapshotChange(response.data.execution);
+            },
+            intervalMs: this.snapshotPollIntervalSeconds * 1000,
+            leadingEdge: false,
+            onError: async (error) => {
+                logger.debug("[DevRunController] Failed to poll for snapshot", { error });
+            },
+        });
+        process.on("SIGTERM", this.sigterm.bind(this));
+    }
+    async sigterm() {
+        logger.debug("[DevRunController] Received SIGTERM, stopping worker");
+        await this.stop();
+    }
+    // This should only be used when we're already executing a run. Attempt number changes are not allowed.
+    updateRunPhase(run, snapshot) {
+        if (this.state.phase !== "RUN") {
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `updateRunPhase: Invalid phase for updating snapshot: ${this.state.phase}`,
+                properties: {
+                    currentPhase: this.state.phase,
+                    snapshotId: snapshot.friendlyId,
+                },
+            });
+            throw new Error(`Invalid phase for updating snapshot: ${this.state.phase}`);
+        }
+        if (this.state.run.friendlyId !== run.friendlyId) {
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `updateRunPhase: Mismatched run IDs`,
+                properties: {
+                    currentRunId: this.state.run.friendlyId,
+                    newRunId: run.friendlyId,
+                    currentSnapshotId: this.state.snapshot.friendlyId,
+                    newSnapshotId: snapshot.friendlyId,
+                },
+            });
+            throw new Error("Mismatched run IDs");
+        }
+        if (this.state.snapshot.friendlyId === snapshot.friendlyId) {
+            logger.debug("updateRunPhase: Snapshot not changed", { run, snapshot });
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `updateRunPhase: Snapshot not changed`,
+                properties: {
+                    snapshotId: snapshot.friendlyId,
+                },
+            });
+            return;
+        }
+        if (this.state.run.attemptNumber !== run.attemptNumber) {
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `updateRunPhase: Attempt number changed`,
+                properties: {
+                    oldAttemptNumber: this.state.run.attemptNumber ?? undefined,
+                    newAttemptNumber: run.attemptNumber ?? undefined,
+                },
+            });
+            throw new Error("Attempt number changed");
+        }
+        this.state = {
+            phase: "RUN",
+            run: {
+                friendlyId: run.friendlyId,
+                attemptNumber: run.attemptNumber,
+            },
+            snapshot: {
+                friendlyId: snapshot.friendlyId,
+            },
+        };
+    }
+    onExitRunPhase(newRun = undefined) {
+        // We're not in a run phase, nothing to do
+        if (this.state.phase !== "RUN") {
+            logger.debug("onExitRunPhase: Not in run phase, skipping", { phase: this.state.phase });
+            return;
+        }
+        // This is still the same run, so we're not exiting the phase
+        if (newRun?.friendlyId === this.state.run.friendlyId) {
+            logger.debug("onExitRunPhase: Same run, skipping", { newRun });
+            return;
+        }
+        logger.debug("onExitRunPhase: Exiting run phase", { newRun });
+        this.snapshotPoller.stop();
+        const { run, snapshot } = this.state;
+        this.unsubscribeFromRunNotifications({ run, snapshot });
+    }
+    subscribeToRunNotifications({ run, snapshot }) {
+        logger.debug("[DevRunController] Subscribing to run notifications", { run, snapshot });
+        this.opts.onSubscribeToRunNotifications(run, snapshot);
+    }
+    unsubscribeFromRunNotifications({ run, snapshot }) {
+        logger.debug("[DevRunController] Unsubscribing from run notifications", { run, snapshot });
+        this.opts.onUnsubscribeFromRunNotifications(run, snapshot);
+    }
+    get runFriendlyId() {
+        if (this.state.phase !== "RUN") {
+            return undefined;
+        }
+        return this.state.run.friendlyId;
+    }
+    get snapshotFriendlyId() {
+        if (this.state.phase !== "RUN") {
+            return;
+        }
+        return this.state.snapshot.friendlyId;
+    }
+    get workerFriendlyId() {
+        if (!this.opts.worker.serverWorker) {
+            throw new Error("No version for dev worker");
+        }
+        return this.opts.worker.serverWorker.id;
+    }
+    handleSnapshotChangeLock = false;
+    async handleSnapshotChange({ run, snapshot, completedWaitpoints, }) {
+        if (this.handleSnapshotChangeLock) {
+            logger.debug("handleSnapshotChange: already in progress");
+            return;
+        }
+        this.handleSnapshotChangeLock = true;
+        // Reset the (fallback) snapshot poll interval so we don't do unnecessary work
+        this.snapshotPoller.resetCurrentInterval();
+        try {
+            if (!this.snapshotFriendlyId) {
+                logger.debug("handleSnapshotChange: Missing snapshot ID", {
+                    runId: run.friendlyId,
+                    snapshotId: this.snapshotFriendlyId,
+                });
+                this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                    time: new Date(),
+                    message: `snapshot change: missing snapshot ID`,
+                    properties: {
+                        newSnapshotId: snapshot.friendlyId,
+                        newSnapshotStatus: snapshot.executionStatus,
+                    },
+                });
+                return;
+            }
+            if (this.snapshotFriendlyId === snapshot.friendlyId) {
+                logger.debug("handleSnapshotChange: snapshot not changed, skipping", { snapshot });
+                this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                    time: new Date(),
+                    message: `snapshot change: skipping, no change`,
+                    properties: {
+                        snapshotId: this.snapshotFriendlyId,
+                        snapshotStatus: snapshot.executionStatus,
+                    },
+                });
+                return;
+            }
+            logger.debug(`handleSnapshotChange: ${snapshot.executionStatus}`, {
+                run,
+                oldSnapshotId: this.snapshotFriendlyId,
+                newSnapshot: snapshot,
+                completedWaitpoints: completedWaitpoints.length,
+            });
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `snapshot change: ${snapshot.executionStatus}`,
+                properties: {
+                    oldSnapshotId: this.snapshotFriendlyId,
+                    newSnapshotId: snapshot.friendlyId,
+                    completedWaitpoints: completedWaitpoints.length,
+                },
+            });
+            try {
+                this.updateRunPhase(run, snapshot);
+            }
+            catch (error) {
+                logger.debug("handleSnapshotChange: failed to update run phase", {
+                    run,
+                    snapshot,
+                    error,
+                });
+                this.runFinished();
+                return;
+            }
+            switch (snapshot.executionStatus) {
+                case "PENDING_CANCEL": {
+                    try {
+                        await this.cancelAttempt();
+                    }
+                    catch (error) {
+                        logger.debug("Failed to cancel attempt, killing task run process", {
+                            error,
+                        });
+                        try {
+                            await this.taskRunProcess?.kill("SIGKILL");
+                        }
+                        catch (error) {
+                            logger.debug("Failed to cancel attempt, failed to kill task run process", { error });
+                        }
+                        return;
+                    }
+                    return;
+                }
+                case "FINISHED": {
+                    if (this.isCompletingRun) {
+                        logger.debug("Run is finished but we're completing it, skipping");
+                        return;
+                    }
+                    await this.exitTaskRunProcessWithoutFailingRun({
+                        flush: true,
+                        reason: "already-finished",
+                    });
+                    return;
+                }
+                case "EXECUTING_WITH_WAITPOINTS": {
+                    logger.debug("Run is executing with waitpoints", { snapshot });
+                    try {
+                        await this.taskRunProcess?.cleanup(false);
+                    }
+                    catch (error) {
+                        logger.debug("Failed to cleanup task run process", { error });
+                    }
+                    if (snapshot.friendlyId !== this.snapshotFriendlyId) {
+                        logger.debug("Snapshot changed after cleanup, abort", {
+                            oldSnapshotId: snapshot.friendlyId,
+                            newSnapshotId: this.snapshotFriendlyId,
+                        });
+                        return;
+                    }
+                    //no snapshots in DEV, so we just return.
+                    return;
+                }
+                case "SUSPENDED": {
+                    logger.debug("Run shouldn't be suspended in DEV", {
+                        run,
+                        snapshot,
+                    });
+                    return;
+                }
+                case "PENDING_EXECUTING": {
+                    logger.debug("Run is pending execution", { run, snapshot });
+                    if (completedWaitpoints.length === 0) {
+                        logger.log("No waitpoints to complete, nothing to do");
+                        return;
+                    }
+                    logger.debug("Run shouldn't be PENDING_EXECUTING with completedWaitpoints in DEV", {
+                        run,
+                        snapshot,
+                    });
+                    return;
+                }
+                case "EXECUTING": {
+                    logger.debug("Run is now executing", { run, snapshot });
+                    if (completedWaitpoints.length === 0) {
+                        return;
+                    }
+                    logger.debug("Processing completed waitpoints", { completedWaitpoints });
+                    if (!this.taskRunProcess) {
+                        logger.debug("No task run process, ignoring completed waitpoints", {
+                            completedWaitpoints,
+                        });
+                        return;
+                    }
+                    for (const waitpoint of completedWaitpoints) {
+                        this.taskRunProcess.waitpointCompleted(waitpoint);
+                    }
+                    return;
+                }
+                case "RUN_CREATED":
+                case "QUEUED_EXECUTING":
+                case "QUEUED": {
+                    logger.debug("Status change not handled", { status: snapshot.executionStatus });
+                    return;
+                }
+                default: {
+                    assertExhaustive(snapshot.executionStatus);
+                }
+            }
+        }
+        catch (error) {
+            logger.debug("handleSnapshotChange: unexpected error", { error });
+            this.httpClient.dev.sendDebugLog(run.friendlyId, {
+                time: new Date(),
+                message: `snapshot change: unexpected error`,
+                properties: {
+                    snapshotId: snapshot.friendlyId,
+                    error: error instanceof Error ? error.message : String(error),
+                },
+            });
+        }
+        finally {
+            this.handleSnapshotChangeLock = false;
+        }
+    }
+    async startAndExecuteRunAttempt({ runFriendlyId, snapshotFriendlyId, dequeuedAt, isWarmStart = false, }) {
+        this.subscribeToRunNotifications({
+            run: { friendlyId: runFriendlyId },
+            snapshot: { friendlyId: snapshotFriendlyId },
+        });
+        const attemptStartedAt = Date.now();
+        const start = await this.httpClient.dev.startRunAttempt(runFriendlyId, snapshotFriendlyId);
+        if (!start.success) {
+            logger.debug("[DevRunController] Failed to start run", { error: start.error });
+            this.runFinished();
+            return;
+        }
+        const attemptDuration = Date.now() - attemptStartedAt;
+        const { run, snapshot, execution, envVars } = start.data;
+        eventBus.emit("runStarted", this.opts.worker, execution);
+        logger.debug("[DevRunController] Started run", {
+            runId: run.friendlyId,
+            snapshot: snapshot.friendlyId,
+        });
+        this.enterRunPhase(run, snapshot);
+        const metrics = [
+            {
+                name: "start",
+                event: "create_attempt",
+                timestamp: attemptStartedAt,
+                duration: attemptDuration,
+            },
+        ].concat(dequeuedAt
+            ? [
+                {
+                    name: "start",
+                    event: "dequeue",
+                    timestamp: dequeuedAt.getTime(),
+                    duration: 0,
+                },
+            ]
+            : []);
+        try {
+            return await this.executeRun({ run, snapshot, execution, envVars, metrics });
+        }
+        catch (error) {
+            logger.debug("Error while executing attempt", {
+                error,
+            });
+            if (error instanceof SuspendedProcessError) {
+                logger.debug("Attempt execution suspended", { error });
+                this.runFinished();
+                return;
+            }
+            logger.debug("Submitting attempt completion", {
+                runId: run.friendlyId,
+                snapshotId: snapshot.friendlyId,
+                updatedSnapshotId: this.snapshotFriendlyId,
+            });
+            const completion = {
+                id: execution.run.id,
+                ok: false,
+                retry: undefined,
+                error: TaskRunProcess.parseExecuteError(error),
+            };
+            const completionResult = await this.httpClient.dev.completeRunAttempt(run.friendlyId, this.snapshotFriendlyId ?? snapshot.friendlyId, { completion });
+            if (!completionResult.success) {
+                logger.debug("Failed to submit completion after error", {
+                    error: completionResult.error,
+                });
+                this.runFinished();
+                return;
+            }
+            logger.debug("Attempt completion submitted after error", completionResult.data.result);
+            try {
+                await this.handleCompletionResult(completion, completionResult.data.result, execution);
+            }
+            catch (error) {
+                logger.debug("Failed to handle completion result after error", { error });
+                this.runFinished();
+                return;
+            }
+        }
+    }
+    async executeRun({ run, snapshot, execution, envVars, metrics, }) {
+        if (!this.opts.worker.serverWorker) {
+            throw new Error(`No server worker for Dev ${run.friendlyId}`);
+        }
+        if (!this.opts.worker.manifest) {
+            throw new Error(`No worker manifest for Dev ${run.friendlyId}`);
+        }
+        this.snapshotPoller.start();
+        logger.debug("getProcess", {
+            build: this.opts.worker.build,
+        });
+        this.isCompletingRun = false;
+        // Get process from pool instead of creating new one
+        const { taskRunProcess, isReused } = await this.opts.taskRunProcessPool.getProcess(this.opts.worker.manifest, {
+            id: "unmanaged",
+            contentHash: this.opts.worker.build.contentHash,
+            version: this.opts.worker.serverWorker?.version,
+            engine: "V2",
+        }, execution.machine, {
+            TRIGGER_WORKER_MANIFEST_PATH: join(this.opts.worker.build.outputPath, "index.json"),
+            RUN_WORKER_SHOW_LOGS: this.opts.logLevel === "debug" ? "true" : "false",
+            TRIGGER_WORKER_VERSION: this.opts.worker.serverWorker?.version,
+        }, this.cwd);
+        this.taskRunProcess = taskRunProcess;
+        taskRunProcess.unsafeDetachEvtHandlers();
+        taskRunProcess.onTaskRunHeartbeat.attach(async (runId) => {
+            if (!this.runFriendlyId || !this.snapshotFriendlyId) {
+                logger.debug("[DevRunController] Skipping heartbeat, no run ID or snapshot ID");
+                return;
+            }
+            logger.debug("[DevRunController] Sending heartbeat");
+            const response = await this.httpClient.dev.heartbeatRun(this.runFriendlyId, this.snapshotFriendlyId, {
+                cpu: 0,
+                memory: 0,
+            });
+            if (!response.success) {
+                logger.debug("[DevRunController] Heartbeat failed", { error: response.error });
+            }
+        });
+        // Update the process environment for this specific run
+        // Note: We may need to enhance TaskRunProcess to support updating env vars
+        logger.debug("executing task run process from pool", {
+            attemptNumber: execution.attempt.number,
+            runId: execution.run.id,
+        });
+        const completion = await this.taskRunProcess.execute({
+            payload: {
+                execution,
+                traceContext: execution.run.traceContext ?? {},
+                metrics,
+            },
+            messageId: run.friendlyId,
+            env: {
+                ...sanitizeEnvVars(envVars ?? {}),
+                ...sanitizeEnvVars(this.opts.worker.params.env),
+                TRIGGER_PROJECT_REF: execution.project.ref,
+            },
+        }, isReused);
+        logger.debug("Completed run", completion);
+        this.isCompletingRun = true;
+        // Return process to pool instead of killing it
+        try {
+            const version = this.opts.worker.serverWorker?.version || "unknown";
+            await this.opts.taskRunProcessPool.returnProcess(this.taskRunProcess, version);
+            this.taskRunProcess = undefined;
+        }
+        catch (error) {
+            logger.debug("Failed to return task run process to pool, submitting completion anyway", {
+                error,
+            });
+        }
+        if (!this.runFriendlyId || !this.snapshotFriendlyId) {
+            logger.debug("executeRun: Missing run ID or snapshot ID after execution", {
+                runId: this.runFriendlyId,
+                snapshotId: this.snapshotFriendlyId,
+            });
+            this.runFinished();
+            return;
+        }
+        const completionResult = await this.httpClient.dev.completeRunAttempt(this.runFriendlyId, this.snapshotFriendlyId, {
+            completion,
+        });
+        if (!completionResult.success) {
+            logger.debug("Failed to submit completion", {
+                error: completionResult.error,
+            });
+            this.runFinished();
+            return;
+        }
+        logger.debug("Attempt completion submitted", completionResult.data.result);
+        try {
+            await this.handleCompletionResult(completion, completionResult.data.result, execution);
+        }
+        catch (error) {
+            logger.debug("Failed to handle completion result", { error });
+            this.runFinished();
+            return;
+        }
+    }
+    async handleCompletionResult(completion, result, execution) {
+        logger.debug("[DevRunController] Handling completion result", { completion, result });
+        const { attemptStatus, snapshot: completionSnapshot, run } = result;
+        try {
+            this.updateRunPhase(run, completionSnapshot);
+        }
+        catch (error) {
+            logger.debug("Failed to update run phase after completion", { error });
+            this.runFinished();
+            return;
+        }
+        if (attemptStatus === "RETRY_QUEUED") {
+            logger.debug("Retry queued");
+            this.runFinished();
+            return;
+        }
+        if (attemptStatus === "RETRY_IMMEDIATELY") {
+            if (completion.ok) {
+                throw new Error("Should retry but completion OK.");
+            }
+            if (!completion.retry) {
+                throw new Error("Should retry but missing retry params.");
+            }
+            await sleep(completion.retry.delay);
+            if (!this.snapshotFriendlyId) {
+                throw new Error("Missing snapshot ID after retry");
+            }
+            this.startAndExecuteRunAttempt({
+                runFriendlyId: run.friendlyId,
+                snapshotFriendlyId: this.snapshotFriendlyId,
+            }).finally(() => { });
+            return;
+        }
+        if (attemptStatus === "RUN_PENDING_CANCEL") {
+            logger.debug("Run pending cancel");
+            return;
+        }
+        eventBus.emit("runCompleted", this.opts.worker, execution, completion, completion.usage?.durationMs ?? 0);
+        if (attemptStatus === "RUN_FINISHED") {
+            logger.debug("Run finished");
+            this.runFinished();
+            return;
+        }
+        assertExhaustive(attemptStatus);
+    }
+    async exitTaskRunProcessWithoutFailingRun({ flush, reason, }) {
+        await this.taskRunProcess?.suspend({ flush });
+        // No services should be left running after this line - let's make sure of it
+        this.shutdownExecution(`exitTaskRunProcessWithoutFailingRun: ${reason}`);
+    }
+    shutdownExecution(reason) {
+        if (this.isShuttingDown) {
+            logger.debug(`[shutdown] ${reason} (already shutting down)`, {
+                newReason: reason,
+            });
+            return;
+        }
+        logger.debug(`[shutdown] ${reason}`);
+        this.isShuttingDown = true;
+        this.snapshotPoller.stop();
+        this.opts.onFinished();
+        this.taskRunProcess?.unsafeDetachEvtHandlers();
+    }
+    async runFinished() {
+        // Return the process to the pool instead of killing it directly
+        if (this.taskRunProcess) {
+            try {
+                const version = this.opts.worker.serverWorker?.version || "unknown";
+                await this.opts.taskRunProcessPool.returnProcess(this.taskRunProcess, version);
+                this.taskRunProcess = undefined;
+            }
+            catch (error) {
+                logger.debug("Failed to return task run process to pool during runFinished", { error });
+            }
+        }
+        this.snapshotPoller.stop();
+        this.opts.onFinished();
+    }
+    async cancelAttempt() {
+        logger.debug("Cancelling attempt", { runId: this.runFriendlyId });
+        await this.taskRunProcess?.cancel();
+    }
+    async start(dequeueMessage) {
+        logger.debug("[DevRunController] Starting up");
+        await this.startAndExecuteRunAttempt({
+            runFriendlyId: dequeueMessage.run.friendlyId,
+            snapshotFriendlyId: dequeueMessage.snapshot.friendlyId,
+            dequeuedAt: dequeueMessage.dequeuedAt,
+        }).finally(async () => { });
+    }
+    async stop() {
+        logger.debug("[DevRunController] Shutting down");
+        process.off("SIGTERM", this.sigterm);
+        if (this.taskRunProcess && !this.taskRunProcess.isBeingKilled) {
+            try {
+                const version = this.opts.worker.serverWorker?.version || "unknown";
+                await this.opts.taskRunProcessPool.returnProcess(this.taskRunProcess, version);
+                this.taskRunProcess = undefined;
+            }
+            catch (error) {
+                logger.debug("Failed to return task run process to pool during stop", { error });
+            }
+        }
+        this.snapshotPoller.stop();
+    }
+    async getLatestSnapshot() {
+        if (!this.runFriendlyId) {
+            return;
+        }
+        logger.debug("[DevRunController] Received notification, manually getting the latest snapshot.");
+        const response = await this.httpClient.dev.getRunExecutionData(this.runFriendlyId);
+        if (!response.success) {
+            logger.debug("Failed to get latest snapshot", { error: response.error });
+            return;
+        }
+        await this.handleSnapshotChange(response.data.execution);
+    }
+    resubscribeToRunNotifications() {
+        if (this.state.phase !== "RUN") {
+            return;
+        }
+        this.subscribeToRunNotifications(this.state);
+    }
+}
+//# sourceMappingURL=dev-run-controller.js.map