trigger.dev 0.0.0-re2-20250502115816 → 0.0.0-re2-20250503165707

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/dist/esm/commands/deploy.js +25 -44
  2. package/dist/esm/commands/deploy.js.map +1 -1
  3. package/dist/esm/entryPoints/dev-run-worker.js +51 -13
  4. package/dist/esm/entryPoints/dev-run-worker.js.map +1 -1
  5. package/dist/esm/entryPoints/managed/controller.d.ts +1 -0
  6. package/dist/esm/entryPoints/managed/controller.js +31 -74
  7. package/dist/esm/entryPoints/managed/controller.js.map +1 -1
  8. package/dist/esm/entryPoints/managed/env.d.ts +5 -0
  9. package/dist/esm/entryPoints/managed/env.js +4 -0
  10. package/dist/esm/entryPoints/managed/env.js.map +1 -1
  11. package/dist/esm/entryPoints/managed/execution.d.ts +14 -24
  12. package/dist/esm/entryPoints/managed/execution.js +210 -270
  13. package/dist/esm/entryPoints/managed/execution.js.map +1 -1
  14. package/dist/esm/entryPoints/managed/logger.d.ts +5 -15
  15. package/dist/esm/entryPoints/managed/logger.js +6 -19
  16. package/dist/esm/entryPoints/managed/logger.js.map +1 -1
  17. package/dist/esm/entryPoints/managed/poller.js +1 -5
  18. package/dist/esm/entryPoints/managed/poller.js.map +1 -1
  19. package/dist/esm/entryPoints/managed-run-worker.js +50 -12
  20. package/dist/esm/entryPoints/managed-run-worker.js.map +1 -1
  21. package/dist/esm/executions/taskRunProcess.d.ts +13 -9
  22. package/dist/esm/executions/taskRunProcess.js +76 -39
  23. package/dist/esm/executions/taskRunProcess.js.map +1 -1
  24. package/dist/esm/version.js +1 -1
  25. package/package.json +3 -7
  26. package/dist/esm/entryPoints/managed/snapshot.d.ts +0 -48
  27. package/dist/esm/entryPoints/managed/snapshot.js +0 -237
  28. package/dist/esm/entryPoints/managed/snapshot.js.map +0 -1
@@ -5,7 +5,6 @@ import { RunExecutionSnapshotPoller } from "./poller.js";
5
5
  import { assertExhaustive, tryCatch } from "@trigger.dev/core/utils";
6
6
  import { MetadataClient } from "./overrides.js";
7
7
  import { randomBytes } from "node:crypto";
8
- import { SnapshotManager } from "./snapshot.js";
9
8
  class ExecutionAbortError extends Error {
10
9
  constructor(message) {
11
10
  super(message);
@@ -16,9 +15,9 @@ export class RunExecution {
16
15
  id;
17
16
  executionAbortController;
18
17
  _runFriendlyId;
18
+ currentSnapshotId;
19
19
  currentAttemptNumber;
20
20
  currentTaskRunEnv;
21
- snapshotManager;
22
21
  dequeuedAt;
23
22
  podScheduledAt;
24
23
  workerManifest;
@@ -30,7 +29,6 @@ export class RunExecution {
30
29
  snapshotPoller;
31
30
  lastHeartbeat;
32
31
  isShuttingDown = false;
33
- shutdownReason;
34
32
  constructor(opts) {
35
33
  this.id = randomBytes(4).toString("hex");
36
34
  this.workerManifest = opts.workerManifest;
@@ -40,33 +38,11 @@ export class RunExecution {
40
38
  this.restoreCount = 0;
41
39
  this.executionAbortController = new AbortController();
42
40
  }
43
- /**
44
- * Cancels the current execution.
45
- */
46
- async cancel() {
47
- if (this.isShuttingDown) {
48
- throw new Error("cancel called after execution shut down");
49
- }
50
- this.sendDebugLog("cancelling attempt", { runId: this.runFriendlyId });
51
- await this.taskRunProcess?.cancel();
52
- }
53
- /**
54
- * Kills the current execution.
55
- */
56
- async kill({ exitExecution = true } = {}) {
57
- await this.taskRunProcess?.kill("SIGKILL");
58
- if (exitExecution) {
59
- this.shutdown("kill");
60
- }
61
- }
62
41
  /**
63
42
  * Prepares the execution with task run environment variables.
64
43
  * This should be called before executing, typically after a successful run to prepare for the next one.
65
44
  */
66
45
  prepareForExecution(opts) {
67
- if (this.isShuttingDown) {
68
- throw new Error("prepareForExecution called after execution shut down");
69
- }
70
46
  if (this.taskRunProcess) {
71
47
  throw new Error("prepareForExecution called after process was already created");
72
48
  }
@@ -113,12 +89,6 @@ export class RunExecution {
113
89
  this.sendDebugLog("onTaskRunHeartbeat: failed", { error: error.message });
114
90
  }
115
91
  });
116
- taskRunProcess.onSendDebugLog.attach(async (debugLog) => {
117
- this.sendRuntimeDebugLog(debugLog.message, debugLog.properties);
118
- });
119
- taskRunProcess.onSetSuspendable.attach(async ({ suspendable }) => {
120
- this.suspendable = suspendable;
121
- });
122
92
  return taskRunProcess;
123
93
  }
124
94
  /**
@@ -133,20 +103,52 @@ export class RunExecution {
133
103
  }
134
104
  /**
135
105
  * Called by the RunController when it receives a websocket notification
136
- * or when the snapshot poller detects a change.
137
- *
138
- * This is the main entry point for snapshot changes, but processing is deferred to the snapshot manager.
106
+ * or when the snapshot poller detects a change
139
107
  */
140
- async enqueueSnapshotChangeAndWait(runData) {
108
+ async handleSnapshotChange(runData) {
141
109
  if (this.isShuttingDown) {
142
- this.sendDebugLog("enqueueSnapshotChangeAndWait: shutting down, skipping");
110
+ this.sendDebugLog("handleSnapshotChange: shutting down, skipping");
111
+ return;
112
+ }
113
+ const { run, snapshot, completedWaitpoints } = runData;
114
+ const snapshotMetadata = {
115
+ incomingRunId: run.friendlyId,
116
+ incomingSnapshotId: snapshot.friendlyId,
117
+ completedWaitpoints: completedWaitpoints.length,
118
+ };
119
+ // Ensure we have run details
120
+ if (!this.runFriendlyId || !this.currentSnapshotId) {
121
+ this.sendDebugLog("handleSnapshotChange: missing run or snapshot ID", snapshotMetadata, run.friendlyId);
143
122
  return;
144
123
  }
145
- if (!this.snapshotManager) {
146
- this.sendDebugLog("enqueueSnapshotChangeAndWait: missing snapshot manager");
124
+ // Ensure the run ID matches
125
+ if (run.friendlyId !== this.runFriendlyId) {
126
+ // Send debug log to both runs
127
+ this.sendDebugLog("handleSnapshotChange: mismatched run IDs", snapshotMetadata);
128
+ this.sendDebugLog("handleSnapshotChange: mismatched run IDs", snapshotMetadata, run.friendlyId);
147
129
  return;
148
130
  }
149
- await this.snapshotManager.handleSnapshotChange(runData);
131
+ this.snapshotChangeQueue.push(runData);
132
+ await this.processSnapshotChangeQueue();
133
+ }
134
+ snapshotChangeQueue = [];
135
+ snapshotChangeQueueLock = false;
136
+ async processSnapshotChangeQueue() {
137
+ if (this.snapshotChangeQueueLock) {
138
+ return;
139
+ }
140
+ this.snapshotChangeQueueLock = true;
141
+ while (this.snapshotChangeQueue.length > 0) {
142
+ const runData = this.snapshotChangeQueue.shift();
143
+ if (!runData) {
144
+ continue;
145
+ }
146
+ const [error] = await tryCatch(this.processSnapshotChange(runData));
147
+ if (error) {
148
+ this.sendDebugLog("Failed to process snapshot change", { error: error.message });
149
+ }
150
+ }
151
+ this.snapshotChangeQueueLock = false;
150
152
  }
151
153
  async processSnapshotChange(runData) {
152
154
  const { run, snapshot, completedWaitpoints } = runData;
@@ -154,25 +156,28 @@ export class RunExecution {
154
156
  incomingSnapshotId: snapshot.friendlyId,
155
157
  completedWaitpoints: completedWaitpoints.length,
156
158
  };
157
- if (!this.snapshotManager) {
158
- this.sendDebugLog("handleSnapshotChange: missing snapshot manager", snapshotMetadata);
159
+ // Check if the incoming snapshot is newer than the current one
160
+ if (!this.currentSnapshotId || snapshot.friendlyId < this.currentSnapshotId) {
161
+ this.sendDebugLog("handleSnapshotChange: received older snapshot, skipping", snapshotMetadata);
162
+ return;
163
+ }
164
+ if (snapshot.friendlyId === this.currentSnapshotId) {
159
165
  return;
160
166
  }
161
167
  if (this.currentAttemptNumber && this.currentAttemptNumber !== run.attemptNumber) {
162
- this.sendDebugLog("error: attempt number mismatch", snapshotMetadata);
163
- // This is a rogue execution, a new one will already have been created elsewhere
164
- await this.exitTaskRunProcessWithoutFailingRun({ flush: false });
168
+ this.sendDebugLog("ERROR: attempt number mismatch", snapshotMetadata);
169
+ await this.taskRunProcess?.suspend();
165
170
  return;
166
171
  }
167
- // DO NOT REMOVE (very noisy, but helpful for debugging)
168
- // this.sendDebugLog(`processing snapshot change: ${snapshot.executionStatus}`, snapshotMetadata);
172
+ this.sendDebugLog(`snapshot has changed to: ${snapshot.executionStatus}`, snapshotMetadata);
169
173
  // Reset the snapshot poll interval so we don't do unnecessary work
170
- this.snapshotPoller?.updateSnapshotId(snapshot.friendlyId);
171
174
  this.snapshotPoller?.resetCurrentInterval();
172
- await this.processEnvOverrides("snapshot change");
175
+ // Update internal state
176
+ this.currentSnapshotId = snapshot.friendlyId;
177
+ // Update services
178
+ this.snapshotPoller?.updateSnapshotId(snapshot.friendlyId);
173
179
  switch (snapshot.executionStatus) {
174
180
  case "PENDING_CANCEL": {
175
- this.sendDebugLog("run was cancelled", snapshotMetadata);
176
181
  const [error] = await tryCatch(this.cancel());
177
182
  if (error) {
178
183
  this.sendDebugLog("snapshot change: failed to cancel attempt", {
@@ -184,39 +189,83 @@ export class RunExecution {
184
189
  return;
185
190
  }
186
191
  case "QUEUED": {
187
- this.sendDebugLog("run was re-queued", snapshotMetadata);
192
+ this.sendDebugLog("Run was re-queued", snapshotMetadata);
188
193
  // Pretend we've just suspended the run. This will kill the process without failing the run.
189
- await this.exitTaskRunProcessWithoutFailingRun({ flush: true });
194
+ await this.taskRunProcess?.suspend();
190
195
  return;
191
196
  }
192
197
  case "FINISHED": {
193
- this.sendDebugLog("run is finished", snapshotMetadata);
198
+ this.sendDebugLog("Run is finished", snapshotMetadata);
194
199
  // Pretend we've just suspended the run. This will kill the process without failing the run.
195
- await this.exitTaskRunProcessWithoutFailingRun({ flush: true });
200
+ await this.taskRunProcess?.suspend();
196
201
  return;
197
202
  }
198
203
  case "QUEUED_EXECUTING":
199
204
  case "EXECUTING_WITH_WAITPOINTS": {
200
- this.sendDebugLog("run is executing with waitpoints", snapshotMetadata);
201
- // Wait for next status change - suspension is handled by the snapshot manager
205
+ this.sendDebugLog("Run is executing with waitpoints", snapshotMetadata);
206
+ const [error] = await tryCatch(this.taskRunProcess?.cleanup(false));
207
+ if (error) {
208
+ this.sendDebugLog("Failed to cleanup task run process, carrying on", {
209
+ ...snapshotMetadata,
210
+ error: error.message,
211
+ });
212
+ }
213
+ if (snapshot.friendlyId !== this.currentSnapshotId) {
214
+ this.sendDebugLog("Snapshot changed after cleanup, abort", snapshotMetadata);
215
+ this.abortExecution();
216
+ return;
217
+ }
218
+ await sleep(this.env.TRIGGER_PRE_SUSPEND_WAIT_MS);
219
+ if (snapshot.friendlyId !== this.currentSnapshotId) {
220
+ this.sendDebugLog("Snapshot changed after suspend threshold, abort", snapshotMetadata);
221
+ this.abortExecution();
222
+ return;
223
+ }
224
+ if (!this.runFriendlyId || !this.currentSnapshotId) {
225
+ this.sendDebugLog("handleSnapshotChange: Missing run ID or snapshot ID after suspension, abort", snapshotMetadata);
226
+ this.abortExecution();
227
+ return;
228
+ }
229
+ const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, this.currentSnapshotId);
230
+ if (!suspendResult.success) {
231
+ this.sendDebugLog("Failed to suspend run, staying alive 🎶", {
232
+ ...snapshotMetadata,
233
+ error: suspendResult.error,
234
+ });
235
+ this.sendDebugLog("checkpoint: suspend request failed", {
236
+ ...snapshotMetadata,
237
+ error: suspendResult.error,
238
+ });
239
+ // This is fine, we'll wait for the next status change
240
+ return;
241
+ }
242
+ if (!suspendResult.data.ok) {
243
+ this.sendDebugLog("checkpoint: failed to suspend run", {
244
+ snapshotId: this.currentSnapshotId,
245
+ error: suspendResult.data.error,
246
+ });
247
+ // This is fine, we'll wait for the next status change
248
+ return;
249
+ }
250
+ this.sendDebugLog("Suspending, any day now 🚬", snapshotMetadata);
251
+ // Wait for next status change
202
252
  return;
203
253
  }
204
254
  case "SUSPENDED": {
205
- this.sendDebugLog("run was suspended", snapshotMetadata);
255
+ this.sendDebugLog("Run was suspended, kill the process", snapshotMetadata);
206
256
  // This will kill the process and fail the execution with a SuspendedProcessError
207
- // We don't flush because we already did before suspending
208
- await this.exitTaskRunProcessWithoutFailingRun({ flush: false });
257
+ await this.taskRunProcess?.suspend();
209
258
  return;
210
259
  }
211
260
  case "PENDING_EXECUTING": {
212
- this.sendDebugLog("run is pending execution", snapshotMetadata);
261
+ this.sendDebugLog("Run is pending execution", snapshotMetadata);
213
262
  if (completedWaitpoints.length === 0) {
214
- this.sendDebugLog("no waitpoints to complete, nothing to do", snapshotMetadata);
263
+ this.sendDebugLog("No waitpoints to complete, nothing to do", snapshotMetadata);
215
264
  return;
216
265
  }
217
266
  const [error] = await tryCatch(this.restore());
218
267
  if (error) {
219
- this.sendDebugLog("failed to restore execution", {
268
+ this.sendDebugLog("Failed to restore execution", {
220
269
  ...snapshotMetadata,
221
270
  error: error.message,
222
271
  });
@@ -226,13 +275,13 @@ export class RunExecution {
226
275
  return;
227
276
  }
228
277
  case "EXECUTING": {
278
+ this.sendDebugLog("Run is now executing", snapshotMetadata);
229
279
  if (completedWaitpoints.length === 0) {
230
- this.sendDebugLog("run is executing without completed waitpoints", snapshotMetadata);
231
280
  return;
232
281
  }
233
- this.sendDebugLog("run is executing with completed waitpoints", snapshotMetadata);
282
+ this.sendDebugLog("Processing completed waitpoints", snapshotMetadata);
234
283
  if (!this.taskRunProcess) {
235
- this.sendDebugLog("no task run process, ignoring completed waitpoints", snapshotMetadata);
284
+ this.sendDebugLog("No task run process, ignoring completed waitpoints", snapshotMetadata);
236
285
  this.abortExecution();
237
286
  return;
238
287
  }
@@ -242,7 +291,7 @@ export class RunExecution {
242
291
  return;
243
292
  }
244
293
  case "RUN_CREATED": {
245
- this.sendDebugLog("aborting execution: invalid status change: RUN_CREATED", snapshotMetadata);
294
+ this.sendDebugLog("Invalid status change", snapshotMetadata);
246
295
  this.abortExecution();
247
296
  return;
248
297
  }
@@ -252,16 +301,16 @@ export class RunExecution {
252
301
  }
253
302
  }
254
303
  async startAttempt({ isWarmStart, }) {
255
- if (!this.runFriendlyId || !this.snapshotManager) {
256
- throw new Error("Cannot start attempt: missing run or snapshot manager");
304
+ if (!this.runFriendlyId || !this.currentSnapshotId) {
305
+ throw new Error("Cannot start attempt: missing run or snapshot ID");
257
306
  }
258
- this.sendDebugLog("starting attempt");
307
+ this.sendDebugLog("Starting attempt");
259
308
  const attemptStartedAt = Date.now();
260
309
  // Check for abort before each major async operation
261
310
  if (this.executionAbortController.signal.aborted) {
262
311
  throw new ExecutionAbortError("Execution aborted before start");
263
312
  }
264
- const start = await this.httpClient.startRunAttempt(this.runFriendlyId, this.snapshotManager.snapshotId, { isWarmStart });
313
+ const start = await this.httpClient.startRunAttempt(this.runFriendlyId, this.currentSnapshotId, { isWarmStart });
265
314
  if (this.executionAbortController.signal.aborted) {
266
315
  throw new ExecutionAbortError("Execution aborted after start");
267
316
  }
@@ -269,14 +318,14 @@ export class RunExecution {
269
318
  throw new Error(`Start API call failed: ${start.error}`);
270
319
  }
271
320
  // A snapshot was just created, so update the snapshot ID
272
- this.snapshotManager.updateSnapshot(start.data.snapshot.friendlyId, start.data.snapshot.executionStatus);
321
+ this.currentSnapshotId = start.data.snapshot.friendlyId;
273
322
  // Also set or update the attempt number - we do this to detect illegal attempt number changes, e.g. from stalled runners coming back online
274
323
  const attemptNumber = start.data.run.attemptNumber;
275
324
  if (attemptNumber && attemptNumber > 0) {
276
325
  this.currentAttemptNumber = attemptNumber;
277
326
  }
278
327
  else {
279
- this.sendDebugLog("error: invalid attempt number returned from start attempt", {
328
+ this.sendDebugLog("ERROR: invalid attempt number returned from start attempt", {
280
329
  attemptNumber: String(attemptNumber),
281
330
  });
282
331
  }
@@ -285,7 +334,7 @@ export class RunExecution {
285
334
  dequeuedAt: this.dequeuedAt?.getTime(),
286
335
  podScheduledAt: this.podScheduledAt?.getTime(),
287
336
  });
288
- this.sendDebugLog("started attempt");
337
+ this.sendDebugLog("Started attempt");
289
338
  return { ...start.data, metrics };
290
339
  }
291
340
  /**
@@ -293,47 +342,34 @@ export class RunExecution {
293
342
  * When this returns, the child process will have been cleaned up.
294
343
  */
295
344
  async execute(runOpts) {
296
- if (this.isShuttingDown) {
297
- throw new Error("execute called after execution shut down");
298
- }
299
345
  // Setup initial state
300
346
  this.runFriendlyId = runOpts.runFriendlyId;
301
- // Create snapshot manager
302
- this.snapshotManager = new SnapshotManager({
303
- runFriendlyId: runOpts.runFriendlyId,
304
- initialSnapshotId: runOpts.snapshotFriendlyId,
305
- // We're just guessing here, but "PENDING_EXECUTING" is probably fine
306
- initialStatus: "PENDING_EXECUTING",
307
- logger: this.logger,
308
- onSnapshotChange: this.processSnapshotChange.bind(this),
309
- onSuspendable: this.handleSuspendable.bind(this),
310
- });
347
+ this.currentSnapshotId = runOpts.snapshotFriendlyId;
311
348
  this.dequeuedAt = runOpts.dequeuedAt;
312
349
  this.podScheduledAt = runOpts.podScheduledAt;
313
350
  // Create and start services
314
351
  this.snapshotPoller = new RunExecutionSnapshotPoller({
315
352
  runFriendlyId: this.runFriendlyId,
316
- snapshotFriendlyId: this.snapshotManager.snapshotId,
353
+ snapshotFriendlyId: this.currentSnapshotId,
317
354
  httpClient: this.httpClient,
318
355
  logger: this.logger,
319
356
  snapshotPollIntervalSeconds: this.env.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS,
320
- handleSnapshotChange: this.enqueueSnapshotChangeAndWait.bind(this),
357
+ handleSnapshotChange: this.handleSnapshotChange.bind(this),
321
358
  });
322
359
  this.snapshotPoller.start();
323
360
  const [startError, start] = await tryCatch(this.startAttempt({ isWarmStart: runOpts.isWarmStart }));
324
361
  if (startError) {
325
- this.sendDebugLog("failed to start attempt", { error: startError.message });
326
- this.shutdown("failed to start attempt");
362
+ this.sendDebugLog("Failed to start attempt", { error: startError.message });
363
+ this.stopServices();
327
364
  return;
328
365
  }
329
366
  const [executeError] = await tryCatch(this.executeRunWrapper(start));
330
367
  if (executeError) {
331
- this.sendDebugLog("failed to execute run", { error: executeError.message });
332
- this.shutdown("failed to execute run");
368
+ this.sendDebugLog("Failed to execute run", { error: executeError.message });
369
+ this.stopServices();
333
370
  return;
334
371
  }
335
- // This is here for safety, but it
336
- this.shutdown("execute call finished");
372
+ this.stopServices();
337
373
  }
338
374
  async executeRunWrapper({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
339
375
  this.currentTaskRunEnv = envVars;
@@ -345,11 +381,13 @@ export class RunExecution {
345
381
  metrics,
346
382
  isWarmStart,
347
383
  }));
384
+ this.sendDebugLog("Run execution completed", { error: executeError?.message });
348
385
  if (!executeError) {
386
+ this.stopServices();
349
387
  return;
350
388
  }
351
389
  if (executeError instanceof SuspendedProcessError) {
352
- this.sendDebugLog("run was suspended", {
390
+ this.sendDebugLog("Run was suspended", {
353
391
  run: run.friendlyId,
354
392
  snapshot: snapshot.friendlyId,
355
393
  error: executeError.message,
@@ -357,14 +395,14 @@ export class RunExecution {
357
395
  return;
358
396
  }
359
397
  if (executeError instanceof ExecutionAbortError) {
360
- this.sendDebugLog("run was interrupted", {
398
+ this.sendDebugLog("Run was interrupted", {
361
399
  run: run.friendlyId,
362
400
  snapshot: snapshot.friendlyId,
363
401
  error: executeError.message,
364
402
  });
365
403
  return;
366
404
  }
367
- this.sendDebugLog("error while executing attempt", {
405
+ this.sendDebugLog("Error while executing attempt", {
368
406
  error: executeError.message,
369
407
  runId: run.friendlyId,
370
408
  snapshotId: snapshot.friendlyId,
@@ -377,8 +415,9 @@ export class RunExecution {
377
415
  };
378
416
  const [completeError] = await tryCatch(this.complete({ completion }));
379
417
  if (completeError) {
380
- this.sendDebugLog("failed to complete run", { error: completeError.message });
418
+ this.sendDebugLog("Failed to complete run", { error: completeError.message });
381
419
  }
420
+ this.stopServices();
382
421
  }
383
422
  async executeRun({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
384
423
  // For immediate retries, we need to ensure the task run process is prepared for the next attempt
@@ -386,7 +425,7 @@ export class RunExecution {
386
425
  this.taskRunProcess &&
387
426
  !this.taskRunProcess.isPreparedForNextAttempt) {
388
427
  this.sendDebugLog("killing existing task run process before executing next attempt");
389
- await this.kill({ exitExecution: false }).catch(() => { });
428
+ await this.kill().catch(() => { });
390
429
  }
391
430
  // To skip this step and eagerly create the task run process, run prepareForExecution first
392
431
  if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
@@ -395,7 +434,7 @@ export class RunExecution {
395
434
  this.sendDebugLog("executing task run process", { runId: execution.run.id });
396
435
  // Set up an abort handler that will cleanup the task run process
397
436
  this.executionAbortController.signal.addEventListener("abort", async () => {
398
- this.sendDebugLog("execution aborted during task run, cleaning up process", {
437
+ this.sendDebugLog("Execution aborted during task run, cleaning up process", {
399
438
  runId: execution.run.id,
400
439
  });
401
440
  await this.taskRunProcess?.cleanup(true);
@@ -410,24 +449,39 @@ export class RunExecution {
410
449
  env: envVars,
411
450
  }, isWarmStart);
412
451
  // If we get here, the task completed normally
413
- this.sendDebugLog("completed run attempt", { attemptSuccess: completion.ok });
452
+ this.sendDebugLog("Completed run attempt", { attemptSuccess: completion.ok });
414
453
  // The execution has finished, so we can cleanup the task run process. Killing it should be safe.
415
454
  const [error] = await tryCatch(this.taskRunProcess.cleanup(true));
416
455
  if (error) {
417
- this.sendDebugLog("failed to cleanup task run process, submitting completion anyway", {
456
+ this.sendDebugLog("Failed to cleanup task run process, submitting completion anyway", {
418
457
  error: error.message,
419
458
  });
420
459
  }
421
460
  const [completionError] = await tryCatch(this.complete({ completion }));
422
461
  if (completionError) {
423
- this.sendDebugLog("failed to complete run", { error: completionError.message });
462
+ this.sendDebugLog("Failed to complete run", { error: completionError.message });
463
+ }
464
+ }
465
+ /**
466
+ * Cancels the current execution.
467
+ */
468
+ async cancel() {
469
+ this.sendDebugLog("cancelling attempt", { runId: this.runFriendlyId });
470
+ await this.taskRunProcess?.cancel();
471
+ }
472
+ exit() {
473
+ if (this.taskRunProcess?.isPreparedForNextRun) {
474
+ this.taskRunProcess?.forceExit();
424
475
  }
425
476
  }
477
+ async kill() {
478
+ await this.taskRunProcess?.kill("SIGKILL");
479
+ }
426
480
  async complete({ completion }) {
427
- if (!this.runFriendlyId || !this.snapshotManager) {
428
- throw new Error("cannot complete run: missing run or snapshot manager");
481
+ if (!this.runFriendlyId || !this.currentSnapshotId) {
482
+ throw new Error("Cannot complete run: missing run or snapshot ID");
429
483
  }
430
- const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.snapshotManager.snapshotId, { completion });
484
+ const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.currentSnapshotId, { completion });
431
485
  if (!completionResult.success) {
432
486
  throw new Error(`failed to submit completion: ${completionResult.error}`);
433
487
  }
@@ -437,57 +491,39 @@ export class RunExecution {
437
491
  });
438
492
  }
439
493
  async handleCompletionResult({ completion, result, }) {
440
- this.sendDebugLog(`completion result: ${result.attemptStatus}`, {
494
+ this.sendDebugLog("Handling completion result", {
441
495
  attemptSuccess: completion.ok,
442
496
  attemptStatus: result.attemptStatus,
443
497
  snapshotId: result.snapshot.friendlyId,
444
498
  runId: result.run.friendlyId,
445
499
  });
446
- const snapshotStatus = this.convertAttemptStatusToSnapshotStatus(result.attemptStatus);
447
- // Update our snapshot ID to match the completion result to ensure any subsequent API calls use the correct snapshot
448
- this.updateSnapshotAfterCompletion(result.snapshot.friendlyId, snapshotStatus);
500
+ // Update our snapshot ID to match the completion result
501
+ // This ensures any subsequent API calls use the correct snapshot
502
+ this.currentSnapshotId = result.snapshot.friendlyId;
449
503
  const { attemptStatus } = result;
450
- switch (attemptStatus) {
451
- case "RUN_FINISHED":
452
- case "RUN_PENDING_CANCEL":
453
- case "RETRY_QUEUED": {
454
- return;
455
- }
456
- case "RETRY_IMMEDIATELY": {
457
- if (attemptStatus !== "RETRY_IMMEDIATELY") {
458
- return;
459
- }
460
- if (completion.ok) {
461
- throw new Error("Should retry but completion OK.");
462
- }
463
- if (!completion.retry) {
464
- throw new Error("Should retry but missing retry params.");
465
- }
466
- await this.retryImmediately({ retryOpts: completion.retry });
467
- return;
504
+ if (attemptStatus === "RUN_FINISHED") {
505
+ this.sendDebugLog("Run finished");
506
+ return;
507
+ }
508
+ if (attemptStatus === "RUN_PENDING_CANCEL") {
509
+ this.sendDebugLog("Run pending cancel");
510
+ return;
511
+ }
512
+ if (attemptStatus === "RETRY_QUEUED") {
513
+ this.sendDebugLog("Retry queued");
514
+ return;
515
+ }
516
+ if (attemptStatus === "RETRY_IMMEDIATELY") {
517
+ if (completion.ok) {
518
+ throw new Error("Should retry but completion OK.");
468
519
  }
469
- default: {
470
- assertExhaustive(attemptStatus);
520
+ if (!completion.retry) {
521
+ throw new Error("Should retry but missing retry params.");
471
522
  }
523
+ await this.retryImmediately({ retryOpts: completion.retry });
524
+ return;
472
525
  }
473
- }
474
- updateSnapshotAfterCompletion(snapshotId, status) {
475
- this.snapshotManager?.updateSnapshot(snapshotId, status);
476
- this.snapshotPoller?.updateSnapshotId(snapshotId);
477
- }
478
- convertAttemptStatusToSnapshotStatus(attemptStatus) {
479
- switch (attemptStatus) {
480
- case "RUN_FINISHED":
481
- return "FINISHED";
482
- case "RUN_PENDING_CANCEL":
483
- return "PENDING_CANCEL";
484
- case "RETRY_QUEUED":
485
- return "QUEUED";
486
- case "RETRY_IMMEDIATELY":
487
- return "EXECUTING";
488
- default:
489
- assertExhaustive(attemptStatus);
490
- }
526
+ assertExhaustive(attemptStatus);
491
527
  }
492
528
  measureExecutionMetrics({ attemptCreatedAt, dequeuedAt, podScheduledAt, }) {
493
529
  const metrics = [
@@ -517,7 +553,7 @@ export class RunExecution {
517
553
  return metrics;
518
554
  }
519
555
  async retryImmediately({ retryOpts }) {
520
- this.sendDebugLog("retrying run immediately", {
556
+ this.sendDebugLog("Retrying run immediately", {
521
557
  timestamp: retryOpts.timestamp,
522
558
  delay: retryOpts.delay,
523
559
  });
@@ -529,65 +565,52 @@ export class RunExecution {
529
565
  // Start and execute next attempt
530
566
  const [startError, start] = await tryCatch(this.startAttempt({ isWarmStart: true }));
531
567
  if (startError) {
532
- this.sendDebugLog("failed to start attempt for retry", { error: startError.message });
533
- this.shutdown("retryImmediately: failed to start attempt");
568
+ this.sendDebugLog("Failed to start attempt for retry", { error: startError.message });
569
+ this.stopServices();
534
570
  return;
535
571
  }
536
572
  const [executeError] = await tryCatch(this.executeRunWrapper({ ...start, isWarmStart: true }));
537
573
  if (executeError) {
538
- this.sendDebugLog("failed to execute run for retry", { error: executeError.message });
539
- this.shutdown("retryImmediately: failed to execute run");
574
+ this.sendDebugLog("Failed to execute run for retry", { error: executeError.message });
575
+ this.stopServices();
540
576
  return;
541
577
  }
578
+ this.stopServices();
542
579
  }
543
580
  /**
544
581
  * Restores a suspended execution from PENDING_EXECUTING
545
582
  */
546
583
  async restore() {
547
- this.sendDebugLog("restoring execution");
548
- if (!this.runFriendlyId || !this.snapshotManager) {
549
- throw new Error("Cannot restore: missing run or snapshot manager");
584
+ this.sendDebugLog("Restoring execution");
585
+ if (!this.runFriendlyId || !this.currentSnapshotId) {
586
+ throw new Error("Cannot restore: missing run or snapshot ID");
550
587
  }
551
588
  // Short delay to give websocket time to reconnect
552
589
  await sleep(100);
553
590
  // Process any env overrides
554
- await this.processEnvOverrides("restore");
555
- const continuationResult = await this.httpClient.continueRunExecution(this.runFriendlyId, this.snapshotManager.snapshotId);
591
+ await this.processEnvOverrides();
592
+ const continuationResult = await this.httpClient.continueRunExecution(this.runFriendlyId, this.currentSnapshotId);
556
593
  if (!continuationResult.success) {
557
594
  throw new Error(continuationResult.error);
558
595
  }
559
596
  // Track restore count
560
597
  this.restoreCount++;
561
598
  }
562
- async exitTaskRunProcessWithoutFailingRun({ flush }) {
563
- await this.taskRunProcess?.suspend({ flush });
564
- // No services should be left running after this line - let's make sure of it
565
- this.shutdown("exitTaskRunProcessWithoutFailingRun");
566
- }
567
599
  /**
568
600
  * Processes env overrides from the metadata service. Generally called when we're resuming from a suspended state.
569
601
  */
570
- async processEnvOverrides(reason) {
602
+ async processEnvOverrides() {
571
603
  if (!this.env.TRIGGER_METADATA_URL) {
572
- this.sendDebugLog("no metadata url, skipping env overrides", { reason });
604
+ this.sendDebugLog("No metadata URL, skipping env overrides");
573
605
  return;
574
606
  }
575
607
  const metadataClient = new MetadataClient(this.env.TRIGGER_METADATA_URL);
576
608
  const overrides = await metadataClient.getEnvOverrides();
577
609
  if (!overrides) {
578
- this.sendDebugLog("no env overrides, skipping", { reason });
610
+ this.sendDebugLog("No env overrides, skipping");
579
611
  return;
580
612
  }
581
- this.sendDebugLog(`processing env overrides: ${reason}`, {
582
- overrides,
583
- currentEnv: this.env.raw,
584
- });
585
- if (this.env.TRIGGER_RUNNER_ID !== overrides.TRIGGER_RUNNER_ID) {
586
- this.sendDebugLog("runner ID changed -> run was restored from a checkpoint", {
587
- currentRunnerId: this.env.TRIGGER_RUNNER_ID,
588
- newRunnerId: overrides.TRIGGER_RUNNER_ID,
589
- });
590
- }
613
+ this.sendDebugLog("Processing env overrides", overrides);
591
614
  // Override the env with the new values
592
615
  this.env.override(overrides);
593
616
  // Update services with new values
@@ -605,17 +628,17 @@ export class RunExecution {
605
628
  }
606
629
  async onHeartbeat() {
607
630
  if (!this.runFriendlyId) {
608
- this.sendDebugLog("heartbeat: missing run ID");
631
+ this.sendDebugLog("Heartbeat: missing run ID");
609
632
  return;
610
633
  }
611
- if (!this.snapshotManager) {
612
- this.sendDebugLog("heartbeat: missing snapshot manager");
634
+ if (!this.currentSnapshotId) {
635
+ this.sendDebugLog("Heartbeat: missing snapshot ID");
613
636
  return;
614
637
  }
615
- this.sendDebugLog("heartbeat");
616
- const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.snapshotManager.snapshotId);
638
+ this.sendDebugLog("Heartbeat: started");
639
+ const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.currentSnapshotId);
617
640
  if (!response.success) {
618
- this.sendDebugLog("heartbeat: failed", { error: response.error });
641
+ this.sendDebugLog("Heartbeat: failed", { error: response.error });
619
642
  }
620
643
  this.lastHeartbeat = new Date();
621
644
  }
@@ -626,33 +649,13 @@ export class RunExecution {
626
649
  properties: {
627
650
  ...properties,
628
651
  runId: this.runFriendlyId,
629
- snapshotId: this.currentSnapshotFriendlyId,
630
- executionId: this.id,
631
- executionRestoreCount: this.restoreCount,
632
- lastHeartbeat: this.lastHeartbeat?.toISOString(),
633
- },
634
- });
635
- }
636
- sendRuntimeDebugLog(message, properties, runIdOverride) {
637
- this.logger.sendDebugLog({
638
- runId: runIdOverride ?? this.runFriendlyId,
639
- message: `[runtime] ${message}`,
640
- print: false,
641
- properties: {
642
- ...properties,
643
- runId: this.runFriendlyId,
644
- snapshotId: this.currentSnapshotFriendlyId,
652
+ snapshotId: this.currentSnapshotId,
645
653
  executionId: this.id,
646
654
  executionRestoreCount: this.restoreCount,
647
655
  lastHeartbeat: this.lastHeartbeat?.toISOString(),
648
656
  },
649
657
  });
650
658
  }
651
- set suspendable(suspendable) {
652
- this.snapshotManager?.setSuspendable(suspendable).catch((error) => {
653
- this.sendDebugLog("failed to set suspendable", { error: error.message });
654
- });
655
- }
656
659
  // Ensure we can only set this once
657
660
  set runFriendlyId(id) {
658
661
  if (this._runFriendlyId) {
@@ -664,7 +667,7 @@ export class RunExecution {
664
667
  return this._runFriendlyId;
665
668
  }
666
669
  get currentSnapshotFriendlyId() {
667
- return this.snapshotManager?.snapshotId;
670
+ return this.currentSnapshotId;
668
671
  }
669
672
  get taskRunEnv() {
670
673
  return this.currentTaskRunEnv;
@@ -679,82 +682,19 @@ export class RunExecution {
679
682
  }
680
683
  abortExecution() {
681
684
  if (this.isAborted) {
682
- this.sendDebugLog("execution already aborted");
685
+ this.sendDebugLog("Execution already aborted");
683
686
  return;
684
687
  }
685
688
  this.executionAbortController.abort();
686
- this.shutdown("abortExecution");
689
+ this.stopServices();
687
690
  }
688
- shutdown(reason) {
691
+ stopServices() {
689
692
  if (this.isShuttingDown) {
690
- this.sendDebugLog(`[shutdown] ${reason} (already shutting down)`, {
691
- firstShutdownReason: this.shutdownReason,
692
- });
693
693
  return;
694
694
  }
695
- this.sendDebugLog(`[shutdown] ${reason}`);
696
695
  this.isShuttingDown = true;
697
- this.shutdownReason = reason;
698
696
  this.snapshotPoller?.stop();
699
- this.snapshotManager?.cleanup();
700
- this.taskRunProcess?.unsafeDetachEvtHandlers();
701
- }
702
- async handleSuspendable(suspendableSnapshot) {
703
- this.sendDebugLog("handleSuspendable", { suspendableSnapshot });
704
- if (!this.snapshotManager) {
705
- this.sendDebugLog("handleSuspendable: missing snapshot manager");
706
- return;
707
- }
708
- // Ensure this is the current snapshot
709
- if (suspendableSnapshot.id !== this.currentSnapshotFriendlyId) {
710
- this.sendDebugLog("snapshot changed before cleanup, abort", {
711
- suspendableSnapshot,
712
- currentSnapshotId: this.currentSnapshotFriendlyId,
713
- });
714
- this.abortExecution();
715
- return;
716
- }
717
- // First cleanup the task run process
718
- const [error] = await tryCatch(this.taskRunProcess?.cleanup(false));
719
- if (error) {
720
- this.sendDebugLog("failed to cleanup task run process, carrying on", {
721
- suspendableSnapshot,
722
- error: error.message,
723
- });
724
- }
725
- // Double check snapshot hasn't changed after cleanup
726
- if (suspendableSnapshot.id !== this.currentSnapshotFriendlyId) {
727
- this.sendDebugLog("snapshot changed after cleanup, abort", {
728
- suspendableSnapshot,
729
- currentSnapshotId: this.currentSnapshotFriendlyId,
730
- });
731
- this.abortExecution();
732
- return;
733
- }
734
- if (!this.runFriendlyId) {
735
- this.sendDebugLog("missing run ID for suspension, abort", { suspendableSnapshot });
736
- this.abortExecution();
737
- return;
738
- }
739
- // Call the suspend API with the current snapshot ID
740
- const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, suspendableSnapshot.id);
741
- if (!suspendResult.success) {
742
- this.sendDebugLog("suspension request failed, staying alive 🎶", {
743
- suspendableSnapshot,
744
- error: suspendResult.error,
745
- });
746
- // This is fine, we'll wait for the next status change
747
- return;
748
- }
749
- if (!suspendResult.data.ok) {
750
- this.sendDebugLog("suspension request returned error, staying alive 🎶", {
751
- suspendableSnapshot,
752
- error: suspendResult.data.error,
753
- });
754
- // This is fine, we'll wait for the next status change
755
- return;
756
- }
757
- this.sendDebugLog("suspending, any day now 🚬", { suspendableSnapshot });
697
+ this.taskRunProcess?.onTaskRunHeartbeat.detach();
758
698
  }
759
699
  }
760
700
  //# sourceMappingURL=execution.js.map