trigger.dev 0.0.0-re2-20250502115816 → 0.0.0-re2-20250503165707
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/commands/deploy.js +25 -44
- package/dist/esm/commands/deploy.js.map +1 -1
- package/dist/esm/entryPoints/dev-run-worker.js +51 -13
- package/dist/esm/entryPoints/dev-run-worker.js.map +1 -1
- package/dist/esm/entryPoints/managed/controller.d.ts +1 -0
- package/dist/esm/entryPoints/managed/controller.js +31 -74
- package/dist/esm/entryPoints/managed/controller.js.map +1 -1
- package/dist/esm/entryPoints/managed/env.d.ts +5 -0
- package/dist/esm/entryPoints/managed/env.js +4 -0
- package/dist/esm/entryPoints/managed/env.js.map +1 -1
- package/dist/esm/entryPoints/managed/execution.d.ts +14 -24
- package/dist/esm/entryPoints/managed/execution.js +210 -270
- package/dist/esm/entryPoints/managed/execution.js.map +1 -1
- package/dist/esm/entryPoints/managed/logger.d.ts +5 -15
- package/dist/esm/entryPoints/managed/logger.js +6 -19
- package/dist/esm/entryPoints/managed/logger.js.map +1 -1
- package/dist/esm/entryPoints/managed/poller.js +1 -5
- package/dist/esm/entryPoints/managed/poller.js.map +1 -1
- package/dist/esm/entryPoints/managed-run-worker.js +50 -12
- package/dist/esm/entryPoints/managed-run-worker.js.map +1 -1
- package/dist/esm/executions/taskRunProcess.d.ts +13 -9
- package/dist/esm/executions/taskRunProcess.js +76 -39
- package/dist/esm/executions/taskRunProcess.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +3 -7
- package/dist/esm/entryPoints/managed/snapshot.d.ts +0 -48
- package/dist/esm/entryPoints/managed/snapshot.js +0 -237
- package/dist/esm/entryPoints/managed/snapshot.js.map +0 -1
|
@@ -5,7 +5,6 @@ import { RunExecutionSnapshotPoller } from "./poller.js";
|
|
|
5
5
|
import { assertExhaustive, tryCatch } from "@trigger.dev/core/utils";
|
|
6
6
|
import { MetadataClient } from "./overrides.js";
|
|
7
7
|
import { randomBytes } from "node:crypto";
|
|
8
|
-
import { SnapshotManager } from "./snapshot.js";
|
|
9
8
|
class ExecutionAbortError extends Error {
|
|
10
9
|
constructor(message) {
|
|
11
10
|
super(message);
|
|
@@ -16,9 +15,9 @@ export class RunExecution {
|
|
|
16
15
|
id;
|
|
17
16
|
executionAbortController;
|
|
18
17
|
_runFriendlyId;
|
|
18
|
+
currentSnapshotId;
|
|
19
19
|
currentAttemptNumber;
|
|
20
20
|
currentTaskRunEnv;
|
|
21
|
-
snapshotManager;
|
|
22
21
|
dequeuedAt;
|
|
23
22
|
podScheduledAt;
|
|
24
23
|
workerManifest;
|
|
@@ -30,7 +29,6 @@ export class RunExecution {
|
|
|
30
29
|
snapshotPoller;
|
|
31
30
|
lastHeartbeat;
|
|
32
31
|
isShuttingDown = false;
|
|
33
|
-
shutdownReason;
|
|
34
32
|
constructor(opts) {
|
|
35
33
|
this.id = randomBytes(4).toString("hex");
|
|
36
34
|
this.workerManifest = opts.workerManifest;
|
|
@@ -40,33 +38,11 @@ export class RunExecution {
|
|
|
40
38
|
this.restoreCount = 0;
|
|
41
39
|
this.executionAbortController = new AbortController();
|
|
42
40
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Cancels the current execution.
|
|
45
|
-
*/
|
|
46
|
-
async cancel() {
|
|
47
|
-
if (this.isShuttingDown) {
|
|
48
|
-
throw new Error("cancel called after execution shut down");
|
|
49
|
-
}
|
|
50
|
-
this.sendDebugLog("cancelling attempt", { runId: this.runFriendlyId });
|
|
51
|
-
await this.taskRunProcess?.cancel();
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Kills the current execution.
|
|
55
|
-
*/
|
|
56
|
-
async kill({ exitExecution = true } = {}) {
|
|
57
|
-
await this.taskRunProcess?.kill("SIGKILL");
|
|
58
|
-
if (exitExecution) {
|
|
59
|
-
this.shutdown("kill");
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
41
|
/**
|
|
63
42
|
* Prepares the execution with task run environment variables.
|
|
64
43
|
* This should be called before executing, typically after a successful run to prepare for the next one.
|
|
65
44
|
*/
|
|
66
45
|
prepareForExecution(opts) {
|
|
67
|
-
if (this.isShuttingDown) {
|
|
68
|
-
throw new Error("prepareForExecution called after execution shut down");
|
|
69
|
-
}
|
|
70
46
|
if (this.taskRunProcess) {
|
|
71
47
|
throw new Error("prepareForExecution called after process was already created");
|
|
72
48
|
}
|
|
@@ -113,12 +89,6 @@ export class RunExecution {
|
|
|
113
89
|
this.sendDebugLog("onTaskRunHeartbeat: failed", { error: error.message });
|
|
114
90
|
}
|
|
115
91
|
});
|
|
116
|
-
taskRunProcess.onSendDebugLog.attach(async (debugLog) => {
|
|
117
|
-
this.sendRuntimeDebugLog(debugLog.message, debugLog.properties);
|
|
118
|
-
});
|
|
119
|
-
taskRunProcess.onSetSuspendable.attach(async ({ suspendable }) => {
|
|
120
|
-
this.suspendable = suspendable;
|
|
121
|
-
});
|
|
122
92
|
return taskRunProcess;
|
|
123
93
|
}
|
|
124
94
|
/**
|
|
@@ -133,20 +103,52 @@ export class RunExecution {
|
|
|
133
103
|
}
|
|
134
104
|
/**
|
|
135
105
|
* Called by the RunController when it receives a websocket notification
|
|
136
|
-
* or when the snapshot poller detects a change
|
|
137
|
-
*
|
|
138
|
-
* This is the main entry point for snapshot changes, but processing is deferred to the snapshot manager.
|
|
106
|
+
* or when the snapshot poller detects a change
|
|
139
107
|
*/
|
|
140
|
-
async
|
|
108
|
+
async handleSnapshotChange(runData) {
|
|
141
109
|
if (this.isShuttingDown) {
|
|
142
|
-
this.sendDebugLog("
|
|
110
|
+
this.sendDebugLog("handleSnapshotChange: shutting down, skipping");
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
const { run, snapshot, completedWaitpoints } = runData;
|
|
114
|
+
const snapshotMetadata = {
|
|
115
|
+
incomingRunId: run.friendlyId,
|
|
116
|
+
incomingSnapshotId: snapshot.friendlyId,
|
|
117
|
+
completedWaitpoints: completedWaitpoints.length,
|
|
118
|
+
};
|
|
119
|
+
// Ensure we have run details
|
|
120
|
+
if (!this.runFriendlyId || !this.currentSnapshotId) {
|
|
121
|
+
this.sendDebugLog("handleSnapshotChange: missing run or snapshot ID", snapshotMetadata, run.friendlyId);
|
|
143
122
|
return;
|
|
144
123
|
}
|
|
145
|
-
|
|
146
|
-
|
|
124
|
+
// Ensure the run ID matches
|
|
125
|
+
if (run.friendlyId !== this.runFriendlyId) {
|
|
126
|
+
// Send debug log to both runs
|
|
127
|
+
this.sendDebugLog("handleSnapshotChange: mismatched run IDs", snapshotMetadata);
|
|
128
|
+
this.sendDebugLog("handleSnapshotChange: mismatched run IDs", snapshotMetadata, run.friendlyId);
|
|
147
129
|
return;
|
|
148
130
|
}
|
|
149
|
-
|
|
131
|
+
this.snapshotChangeQueue.push(runData);
|
|
132
|
+
await this.processSnapshotChangeQueue();
|
|
133
|
+
}
|
|
134
|
+
snapshotChangeQueue = [];
|
|
135
|
+
snapshotChangeQueueLock = false;
|
|
136
|
+
async processSnapshotChangeQueue() {
|
|
137
|
+
if (this.snapshotChangeQueueLock) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
this.snapshotChangeQueueLock = true;
|
|
141
|
+
while (this.snapshotChangeQueue.length > 0) {
|
|
142
|
+
const runData = this.snapshotChangeQueue.shift();
|
|
143
|
+
if (!runData) {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const [error] = await tryCatch(this.processSnapshotChange(runData));
|
|
147
|
+
if (error) {
|
|
148
|
+
this.sendDebugLog("Failed to process snapshot change", { error: error.message });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
this.snapshotChangeQueueLock = false;
|
|
150
152
|
}
|
|
151
153
|
async processSnapshotChange(runData) {
|
|
152
154
|
const { run, snapshot, completedWaitpoints } = runData;
|
|
@@ -154,25 +156,28 @@ export class RunExecution {
|
|
|
154
156
|
incomingSnapshotId: snapshot.friendlyId,
|
|
155
157
|
completedWaitpoints: completedWaitpoints.length,
|
|
156
158
|
};
|
|
157
|
-
if
|
|
158
|
-
|
|
159
|
+
// Check if the incoming snapshot is newer than the current one
|
|
160
|
+
if (!this.currentSnapshotId || snapshot.friendlyId < this.currentSnapshotId) {
|
|
161
|
+
this.sendDebugLog("handleSnapshotChange: received older snapshot, skipping", snapshotMetadata);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (snapshot.friendlyId === this.currentSnapshotId) {
|
|
159
165
|
return;
|
|
160
166
|
}
|
|
161
167
|
if (this.currentAttemptNumber && this.currentAttemptNumber !== run.attemptNumber) {
|
|
162
|
-
this.sendDebugLog("
|
|
163
|
-
|
|
164
|
-
await this.exitTaskRunProcessWithoutFailingRun({ flush: false });
|
|
168
|
+
this.sendDebugLog("ERROR: attempt number mismatch", snapshotMetadata);
|
|
169
|
+
await this.taskRunProcess?.suspend();
|
|
165
170
|
return;
|
|
166
171
|
}
|
|
167
|
-
|
|
168
|
-
// this.sendDebugLog(`processing snapshot change: ${snapshot.executionStatus}`, snapshotMetadata);
|
|
172
|
+
this.sendDebugLog(`snapshot has changed to: ${snapshot.executionStatus}`, snapshotMetadata);
|
|
169
173
|
// Reset the snapshot poll interval so we don't do unnecessary work
|
|
170
|
-
this.snapshotPoller?.updateSnapshotId(snapshot.friendlyId);
|
|
171
174
|
this.snapshotPoller?.resetCurrentInterval();
|
|
172
|
-
|
|
175
|
+
// Update internal state
|
|
176
|
+
this.currentSnapshotId = snapshot.friendlyId;
|
|
177
|
+
// Update services
|
|
178
|
+
this.snapshotPoller?.updateSnapshotId(snapshot.friendlyId);
|
|
173
179
|
switch (snapshot.executionStatus) {
|
|
174
180
|
case "PENDING_CANCEL": {
|
|
175
|
-
this.sendDebugLog("run was cancelled", snapshotMetadata);
|
|
176
181
|
const [error] = await tryCatch(this.cancel());
|
|
177
182
|
if (error) {
|
|
178
183
|
this.sendDebugLog("snapshot change: failed to cancel attempt", {
|
|
@@ -184,39 +189,83 @@ export class RunExecution {
|
|
|
184
189
|
return;
|
|
185
190
|
}
|
|
186
191
|
case "QUEUED": {
|
|
187
|
-
this.sendDebugLog("
|
|
192
|
+
this.sendDebugLog("Run was re-queued", snapshotMetadata);
|
|
188
193
|
// Pretend we've just suspended the run. This will kill the process without failing the run.
|
|
189
|
-
await this.
|
|
194
|
+
await this.taskRunProcess?.suspend();
|
|
190
195
|
return;
|
|
191
196
|
}
|
|
192
197
|
case "FINISHED": {
|
|
193
|
-
this.sendDebugLog("
|
|
198
|
+
this.sendDebugLog("Run is finished", snapshotMetadata);
|
|
194
199
|
// Pretend we've just suspended the run. This will kill the process without failing the run.
|
|
195
|
-
await this.
|
|
200
|
+
await this.taskRunProcess?.suspend();
|
|
196
201
|
return;
|
|
197
202
|
}
|
|
198
203
|
case "QUEUED_EXECUTING":
|
|
199
204
|
case "EXECUTING_WITH_WAITPOINTS": {
|
|
200
|
-
this.sendDebugLog("
|
|
201
|
-
|
|
205
|
+
this.sendDebugLog("Run is executing with waitpoints", snapshotMetadata);
|
|
206
|
+
const [error] = await tryCatch(this.taskRunProcess?.cleanup(false));
|
|
207
|
+
if (error) {
|
|
208
|
+
this.sendDebugLog("Failed to cleanup task run process, carrying on", {
|
|
209
|
+
...snapshotMetadata,
|
|
210
|
+
error: error.message,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
if (snapshot.friendlyId !== this.currentSnapshotId) {
|
|
214
|
+
this.sendDebugLog("Snapshot changed after cleanup, abort", snapshotMetadata);
|
|
215
|
+
this.abortExecution();
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
await sleep(this.env.TRIGGER_PRE_SUSPEND_WAIT_MS);
|
|
219
|
+
if (snapshot.friendlyId !== this.currentSnapshotId) {
|
|
220
|
+
this.sendDebugLog("Snapshot changed after suspend threshold, abort", snapshotMetadata);
|
|
221
|
+
this.abortExecution();
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
if (!this.runFriendlyId || !this.currentSnapshotId) {
|
|
225
|
+
this.sendDebugLog("handleSnapshotChange: Missing run ID or snapshot ID after suspension, abort", snapshotMetadata);
|
|
226
|
+
this.abortExecution();
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, this.currentSnapshotId);
|
|
230
|
+
if (!suspendResult.success) {
|
|
231
|
+
this.sendDebugLog("Failed to suspend run, staying alive 🎶", {
|
|
232
|
+
...snapshotMetadata,
|
|
233
|
+
error: suspendResult.error,
|
|
234
|
+
});
|
|
235
|
+
this.sendDebugLog("checkpoint: suspend request failed", {
|
|
236
|
+
...snapshotMetadata,
|
|
237
|
+
error: suspendResult.error,
|
|
238
|
+
});
|
|
239
|
+
// This is fine, we'll wait for the next status change
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
if (!suspendResult.data.ok) {
|
|
243
|
+
this.sendDebugLog("checkpoint: failed to suspend run", {
|
|
244
|
+
snapshotId: this.currentSnapshotId,
|
|
245
|
+
error: suspendResult.data.error,
|
|
246
|
+
});
|
|
247
|
+
// This is fine, we'll wait for the next status change
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
this.sendDebugLog("Suspending, any day now 🚬", snapshotMetadata);
|
|
251
|
+
// Wait for next status change
|
|
202
252
|
return;
|
|
203
253
|
}
|
|
204
254
|
case "SUSPENDED": {
|
|
205
|
-
this.sendDebugLog("
|
|
255
|
+
this.sendDebugLog("Run was suspended, kill the process", snapshotMetadata);
|
|
206
256
|
// This will kill the process and fail the execution with a SuspendedProcessError
|
|
207
|
-
|
|
208
|
-
await this.exitTaskRunProcessWithoutFailingRun({ flush: false });
|
|
257
|
+
await this.taskRunProcess?.suspend();
|
|
209
258
|
return;
|
|
210
259
|
}
|
|
211
260
|
case "PENDING_EXECUTING": {
|
|
212
|
-
this.sendDebugLog("
|
|
261
|
+
this.sendDebugLog("Run is pending execution", snapshotMetadata);
|
|
213
262
|
if (completedWaitpoints.length === 0) {
|
|
214
|
-
this.sendDebugLog("
|
|
263
|
+
this.sendDebugLog("No waitpoints to complete, nothing to do", snapshotMetadata);
|
|
215
264
|
return;
|
|
216
265
|
}
|
|
217
266
|
const [error] = await tryCatch(this.restore());
|
|
218
267
|
if (error) {
|
|
219
|
-
this.sendDebugLog("
|
|
268
|
+
this.sendDebugLog("Failed to restore execution", {
|
|
220
269
|
...snapshotMetadata,
|
|
221
270
|
error: error.message,
|
|
222
271
|
});
|
|
@@ -226,13 +275,13 @@ export class RunExecution {
|
|
|
226
275
|
return;
|
|
227
276
|
}
|
|
228
277
|
case "EXECUTING": {
|
|
278
|
+
this.sendDebugLog("Run is now executing", snapshotMetadata);
|
|
229
279
|
if (completedWaitpoints.length === 0) {
|
|
230
|
-
this.sendDebugLog("run is executing without completed waitpoints", snapshotMetadata);
|
|
231
280
|
return;
|
|
232
281
|
}
|
|
233
|
-
this.sendDebugLog("
|
|
282
|
+
this.sendDebugLog("Processing completed waitpoints", snapshotMetadata);
|
|
234
283
|
if (!this.taskRunProcess) {
|
|
235
|
-
this.sendDebugLog("
|
|
284
|
+
this.sendDebugLog("No task run process, ignoring completed waitpoints", snapshotMetadata);
|
|
236
285
|
this.abortExecution();
|
|
237
286
|
return;
|
|
238
287
|
}
|
|
@@ -242,7 +291,7 @@ export class RunExecution {
|
|
|
242
291
|
return;
|
|
243
292
|
}
|
|
244
293
|
case "RUN_CREATED": {
|
|
245
|
-
this.sendDebugLog("
|
|
294
|
+
this.sendDebugLog("Invalid status change", snapshotMetadata);
|
|
246
295
|
this.abortExecution();
|
|
247
296
|
return;
|
|
248
297
|
}
|
|
@@ -252,16 +301,16 @@ export class RunExecution {
|
|
|
252
301
|
}
|
|
253
302
|
}
|
|
254
303
|
async startAttempt({ isWarmStart, }) {
|
|
255
|
-
if (!this.runFriendlyId || !this.
|
|
256
|
-
throw new Error("Cannot start attempt: missing run or snapshot
|
|
304
|
+
if (!this.runFriendlyId || !this.currentSnapshotId) {
|
|
305
|
+
throw new Error("Cannot start attempt: missing run or snapshot ID");
|
|
257
306
|
}
|
|
258
|
-
this.sendDebugLog("
|
|
307
|
+
this.sendDebugLog("Starting attempt");
|
|
259
308
|
const attemptStartedAt = Date.now();
|
|
260
309
|
// Check for abort before each major async operation
|
|
261
310
|
if (this.executionAbortController.signal.aborted) {
|
|
262
311
|
throw new ExecutionAbortError("Execution aborted before start");
|
|
263
312
|
}
|
|
264
|
-
const start = await this.httpClient.startRunAttempt(this.runFriendlyId, this.
|
|
313
|
+
const start = await this.httpClient.startRunAttempt(this.runFriendlyId, this.currentSnapshotId, { isWarmStart });
|
|
265
314
|
if (this.executionAbortController.signal.aborted) {
|
|
266
315
|
throw new ExecutionAbortError("Execution aborted after start");
|
|
267
316
|
}
|
|
@@ -269,14 +318,14 @@ export class RunExecution {
|
|
|
269
318
|
throw new Error(`Start API call failed: ${start.error}`);
|
|
270
319
|
}
|
|
271
320
|
// A snapshot was just created, so update the snapshot ID
|
|
272
|
-
this.
|
|
321
|
+
this.currentSnapshotId = start.data.snapshot.friendlyId;
|
|
273
322
|
// Also set or update the attempt number - we do this to detect illegal attempt number changes, e.g. from stalled runners coming back online
|
|
274
323
|
const attemptNumber = start.data.run.attemptNumber;
|
|
275
324
|
if (attemptNumber && attemptNumber > 0) {
|
|
276
325
|
this.currentAttemptNumber = attemptNumber;
|
|
277
326
|
}
|
|
278
327
|
else {
|
|
279
|
-
this.sendDebugLog("
|
|
328
|
+
this.sendDebugLog("ERROR: invalid attempt number returned from start attempt", {
|
|
280
329
|
attemptNumber: String(attemptNumber),
|
|
281
330
|
});
|
|
282
331
|
}
|
|
@@ -285,7 +334,7 @@ export class RunExecution {
|
|
|
285
334
|
dequeuedAt: this.dequeuedAt?.getTime(),
|
|
286
335
|
podScheduledAt: this.podScheduledAt?.getTime(),
|
|
287
336
|
});
|
|
288
|
-
this.sendDebugLog("
|
|
337
|
+
this.sendDebugLog("Started attempt");
|
|
289
338
|
return { ...start.data, metrics };
|
|
290
339
|
}
|
|
291
340
|
/**
|
|
@@ -293,47 +342,34 @@ export class RunExecution {
|
|
|
293
342
|
* When this returns, the child process will have been cleaned up.
|
|
294
343
|
*/
|
|
295
344
|
async execute(runOpts) {
|
|
296
|
-
if (this.isShuttingDown) {
|
|
297
|
-
throw new Error("execute called after execution shut down");
|
|
298
|
-
}
|
|
299
345
|
// Setup initial state
|
|
300
346
|
this.runFriendlyId = runOpts.runFriendlyId;
|
|
301
|
-
|
|
302
|
-
this.snapshotManager = new SnapshotManager({
|
|
303
|
-
runFriendlyId: runOpts.runFriendlyId,
|
|
304
|
-
initialSnapshotId: runOpts.snapshotFriendlyId,
|
|
305
|
-
// We're just guessing here, but "PENDING_EXECUTING" is probably fine
|
|
306
|
-
initialStatus: "PENDING_EXECUTING",
|
|
307
|
-
logger: this.logger,
|
|
308
|
-
onSnapshotChange: this.processSnapshotChange.bind(this),
|
|
309
|
-
onSuspendable: this.handleSuspendable.bind(this),
|
|
310
|
-
});
|
|
347
|
+
this.currentSnapshotId = runOpts.snapshotFriendlyId;
|
|
311
348
|
this.dequeuedAt = runOpts.dequeuedAt;
|
|
312
349
|
this.podScheduledAt = runOpts.podScheduledAt;
|
|
313
350
|
// Create and start services
|
|
314
351
|
this.snapshotPoller = new RunExecutionSnapshotPoller({
|
|
315
352
|
runFriendlyId: this.runFriendlyId,
|
|
316
|
-
snapshotFriendlyId: this.
|
|
353
|
+
snapshotFriendlyId: this.currentSnapshotId,
|
|
317
354
|
httpClient: this.httpClient,
|
|
318
355
|
logger: this.logger,
|
|
319
356
|
snapshotPollIntervalSeconds: this.env.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS,
|
|
320
|
-
handleSnapshotChange: this.
|
|
357
|
+
handleSnapshotChange: this.handleSnapshotChange.bind(this),
|
|
321
358
|
});
|
|
322
359
|
this.snapshotPoller.start();
|
|
323
360
|
const [startError, start] = await tryCatch(this.startAttempt({ isWarmStart: runOpts.isWarmStart }));
|
|
324
361
|
if (startError) {
|
|
325
|
-
this.sendDebugLog("
|
|
326
|
-
this.
|
|
362
|
+
this.sendDebugLog("Failed to start attempt", { error: startError.message });
|
|
363
|
+
this.stopServices();
|
|
327
364
|
return;
|
|
328
365
|
}
|
|
329
366
|
const [executeError] = await tryCatch(this.executeRunWrapper(start));
|
|
330
367
|
if (executeError) {
|
|
331
|
-
this.sendDebugLog("
|
|
332
|
-
this.
|
|
368
|
+
this.sendDebugLog("Failed to execute run", { error: executeError.message });
|
|
369
|
+
this.stopServices();
|
|
333
370
|
return;
|
|
334
371
|
}
|
|
335
|
-
|
|
336
|
-
this.shutdown("execute call finished");
|
|
372
|
+
this.stopServices();
|
|
337
373
|
}
|
|
338
374
|
async executeRunWrapper({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
|
|
339
375
|
this.currentTaskRunEnv = envVars;
|
|
@@ -345,11 +381,13 @@ export class RunExecution {
|
|
|
345
381
|
metrics,
|
|
346
382
|
isWarmStart,
|
|
347
383
|
}));
|
|
384
|
+
this.sendDebugLog("Run execution completed", { error: executeError?.message });
|
|
348
385
|
if (!executeError) {
|
|
386
|
+
this.stopServices();
|
|
349
387
|
return;
|
|
350
388
|
}
|
|
351
389
|
if (executeError instanceof SuspendedProcessError) {
|
|
352
|
-
this.sendDebugLog("
|
|
390
|
+
this.sendDebugLog("Run was suspended", {
|
|
353
391
|
run: run.friendlyId,
|
|
354
392
|
snapshot: snapshot.friendlyId,
|
|
355
393
|
error: executeError.message,
|
|
@@ -357,14 +395,14 @@ export class RunExecution {
|
|
|
357
395
|
return;
|
|
358
396
|
}
|
|
359
397
|
if (executeError instanceof ExecutionAbortError) {
|
|
360
|
-
this.sendDebugLog("
|
|
398
|
+
this.sendDebugLog("Run was interrupted", {
|
|
361
399
|
run: run.friendlyId,
|
|
362
400
|
snapshot: snapshot.friendlyId,
|
|
363
401
|
error: executeError.message,
|
|
364
402
|
});
|
|
365
403
|
return;
|
|
366
404
|
}
|
|
367
|
-
this.sendDebugLog("
|
|
405
|
+
this.sendDebugLog("Error while executing attempt", {
|
|
368
406
|
error: executeError.message,
|
|
369
407
|
runId: run.friendlyId,
|
|
370
408
|
snapshotId: snapshot.friendlyId,
|
|
@@ -377,8 +415,9 @@ export class RunExecution {
|
|
|
377
415
|
};
|
|
378
416
|
const [completeError] = await tryCatch(this.complete({ completion }));
|
|
379
417
|
if (completeError) {
|
|
380
|
-
this.sendDebugLog("
|
|
418
|
+
this.sendDebugLog("Failed to complete run", { error: completeError.message });
|
|
381
419
|
}
|
|
420
|
+
this.stopServices();
|
|
382
421
|
}
|
|
383
422
|
async executeRun({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
|
|
384
423
|
// For immediate retries, we need to ensure the task run process is prepared for the next attempt
|
|
@@ -386,7 +425,7 @@ export class RunExecution {
|
|
|
386
425
|
this.taskRunProcess &&
|
|
387
426
|
!this.taskRunProcess.isPreparedForNextAttempt) {
|
|
388
427
|
this.sendDebugLog("killing existing task run process before executing next attempt");
|
|
389
|
-
await this.kill(
|
|
428
|
+
await this.kill().catch(() => { });
|
|
390
429
|
}
|
|
391
430
|
// To skip this step and eagerly create the task run process, run prepareForExecution first
|
|
392
431
|
if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
|
|
@@ -395,7 +434,7 @@ export class RunExecution {
|
|
|
395
434
|
this.sendDebugLog("executing task run process", { runId: execution.run.id });
|
|
396
435
|
// Set up an abort handler that will cleanup the task run process
|
|
397
436
|
this.executionAbortController.signal.addEventListener("abort", async () => {
|
|
398
|
-
this.sendDebugLog("
|
|
437
|
+
this.sendDebugLog("Execution aborted during task run, cleaning up process", {
|
|
399
438
|
runId: execution.run.id,
|
|
400
439
|
});
|
|
401
440
|
await this.taskRunProcess?.cleanup(true);
|
|
@@ -410,24 +449,39 @@ export class RunExecution {
|
|
|
410
449
|
env: envVars,
|
|
411
450
|
}, isWarmStart);
|
|
412
451
|
// If we get here, the task completed normally
|
|
413
|
-
this.sendDebugLog("
|
|
452
|
+
this.sendDebugLog("Completed run attempt", { attemptSuccess: completion.ok });
|
|
414
453
|
// The execution has finished, so we can cleanup the task run process. Killing it should be safe.
|
|
415
454
|
const [error] = await tryCatch(this.taskRunProcess.cleanup(true));
|
|
416
455
|
if (error) {
|
|
417
|
-
this.sendDebugLog("
|
|
456
|
+
this.sendDebugLog("Failed to cleanup task run process, submitting completion anyway", {
|
|
418
457
|
error: error.message,
|
|
419
458
|
});
|
|
420
459
|
}
|
|
421
460
|
const [completionError] = await tryCatch(this.complete({ completion }));
|
|
422
461
|
if (completionError) {
|
|
423
|
-
this.sendDebugLog("
|
|
462
|
+
this.sendDebugLog("Failed to complete run", { error: completionError.message });
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Cancels the current execution.
|
|
467
|
+
*/
|
|
468
|
+
async cancel() {
|
|
469
|
+
this.sendDebugLog("cancelling attempt", { runId: this.runFriendlyId });
|
|
470
|
+
await this.taskRunProcess?.cancel();
|
|
471
|
+
}
|
|
472
|
+
exit() {
|
|
473
|
+
if (this.taskRunProcess?.isPreparedForNextRun) {
|
|
474
|
+
this.taskRunProcess?.forceExit();
|
|
424
475
|
}
|
|
425
476
|
}
|
|
477
|
+
async kill() {
|
|
478
|
+
await this.taskRunProcess?.kill("SIGKILL");
|
|
479
|
+
}
|
|
426
480
|
async complete({ completion }) {
|
|
427
|
-
if (!this.runFriendlyId || !this.
|
|
428
|
-
throw new Error("
|
|
481
|
+
if (!this.runFriendlyId || !this.currentSnapshotId) {
|
|
482
|
+
throw new Error("Cannot complete run: missing run or snapshot ID");
|
|
429
483
|
}
|
|
430
|
-
const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.
|
|
484
|
+
const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.currentSnapshotId, { completion });
|
|
431
485
|
if (!completionResult.success) {
|
|
432
486
|
throw new Error(`failed to submit completion: ${completionResult.error}`);
|
|
433
487
|
}
|
|
@@ -437,57 +491,39 @@ export class RunExecution {
|
|
|
437
491
|
});
|
|
438
492
|
}
|
|
439
493
|
async handleCompletionResult({ completion, result, }) {
|
|
440
|
-
this.sendDebugLog(
|
|
494
|
+
this.sendDebugLog("Handling completion result", {
|
|
441
495
|
attemptSuccess: completion.ok,
|
|
442
496
|
attemptStatus: result.attemptStatus,
|
|
443
497
|
snapshotId: result.snapshot.friendlyId,
|
|
444
498
|
runId: result.run.friendlyId,
|
|
445
499
|
});
|
|
446
|
-
|
|
447
|
-
//
|
|
448
|
-
this.
|
|
500
|
+
// Update our snapshot ID to match the completion result
|
|
501
|
+
// This ensures any subsequent API calls use the correct snapshot
|
|
502
|
+
this.currentSnapshotId = result.snapshot.friendlyId;
|
|
449
503
|
const { attemptStatus } = result;
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
}
|
|
466
|
-
await this.retryImmediately({ retryOpts: completion.retry });
|
|
467
|
-
return;
|
|
504
|
+
if (attemptStatus === "RUN_FINISHED") {
|
|
505
|
+
this.sendDebugLog("Run finished");
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
if (attemptStatus === "RUN_PENDING_CANCEL") {
|
|
509
|
+
this.sendDebugLog("Run pending cancel");
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
if (attemptStatus === "RETRY_QUEUED") {
|
|
513
|
+
this.sendDebugLog("Retry queued");
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
if (attemptStatus === "RETRY_IMMEDIATELY") {
|
|
517
|
+
if (completion.ok) {
|
|
518
|
+
throw new Error("Should retry but completion OK.");
|
|
468
519
|
}
|
|
469
|
-
|
|
470
|
-
|
|
520
|
+
if (!completion.retry) {
|
|
521
|
+
throw new Error("Should retry but missing retry params.");
|
|
471
522
|
}
|
|
523
|
+
await this.retryImmediately({ retryOpts: completion.retry });
|
|
524
|
+
return;
|
|
472
525
|
}
|
|
473
|
-
|
|
474
|
-
updateSnapshotAfterCompletion(snapshotId, status) {
|
|
475
|
-
this.snapshotManager?.updateSnapshot(snapshotId, status);
|
|
476
|
-
this.snapshotPoller?.updateSnapshotId(snapshotId);
|
|
477
|
-
}
|
|
478
|
-
convertAttemptStatusToSnapshotStatus(attemptStatus) {
|
|
479
|
-
switch (attemptStatus) {
|
|
480
|
-
case "RUN_FINISHED":
|
|
481
|
-
return "FINISHED";
|
|
482
|
-
case "RUN_PENDING_CANCEL":
|
|
483
|
-
return "PENDING_CANCEL";
|
|
484
|
-
case "RETRY_QUEUED":
|
|
485
|
-
return "QUEUED";
|
|
486
|
-
case "RETRY_IMMEDIATELY":
|
|
487
|
-
return "EXECUTING";
|
|
488
|
-
default:
|
|
489
|
-
assertExhaustive(attemptStatus);
|
|
490
|
-
}
|
|
526
|
+
assertExhaustive(attemptStatus);
|
|
491
527
|
}
|
|
492
528
|
measureExecutionMetrics({ attemptCreatedAt, dequeuedAt, podScheduledAt, }) {
|
|
493
529
|
const metrics = [
|
|
@@ -517,7 +553,7 @@ export class RunExecution {
|
|
|
517
553
|
return metrics;
|
|
518
554
|
}
|
|
519
555
|
async retryImmediately({ retryOpts }) {
|
|
520
|
-
this.sendDebugLog("
|
|
556
|
+
this.sendDebugLog("Retrying run immediately", {
|
|
521
557
|
timestamp: retryOpts.timestamp,
|
|
522
558
|
delay: retryOpts.delay,
|
|
523
559
|
});
|
|
@@ -529,65 +565,52 @@ export class RunExecution {
|
|
|
529
565
|
// Start and execute next attempt
|
|
530
566
|
const [startError, start] = await tryCatch(this.startAttempt({ isWarmStart: true }));
|
|
531
567
|
if (startError) {
|
|
532
|
-
this.sendDebugLog("
|
|
533
|
-
this.
|
|
568
|
+
this.sendDebugLog("Failed to start attempt for retry", { error: startError.message });
|
|
569
|
+
this.stopServices();
|
|
534
570
|
return;
|
|
535
571
|
}
|
|
536
572
|
const [executeError] = await tryCatch(this.executeRunWrapper({ ...start, isWarmStart: true }));
|
|
537
573
|
if (executeError) {
|
|
538
|
-
this.sendDebugLog("
|
|
539
|
-
this.
|
|
574
|
+
this.sendDebugLog("Failed to execute run for retry", { error: executeError.message });
|
|
575
|
+
this.stopServices();
|
|
540
576
|
return;
|
|
541
577
|
}
|
|
578
|
+
this.stopServices();
|
|
542
579
|
}
|
|
543
580
|
/**
|
|
544
581
|
* Restores a suspended execution from PENDING_EXECUTING
|
|
545
582
|
*/
|
|
546
583
|
async restore() {
|
|
547
|
-
this.sendDebugLog("
|
|
548
|
-
if (!this.runFriendlyId || !this.
|
|
549
|
-
throw new Error("Cannot restore: missing run or snapshot
|
|
584
|
+
this.sendDebugLog("Restoring execution");
|
|
585
|
+
if (!this.runFriendlyId || !this.currentSnapshotId) {
|
|
586
|
+
throw new Error("Cannot restore: missing run or snapshot ID");
|
|
550
587
|
}
|
|
551
588
|
// Short delay to give websocket time to reconnect
|
|
552
589
|
await sleep(100);
|
|
553
590
|
// Process any env overrides
|
|
554
|
-
await this.processEnvOverrides(
|
|
555
|
-
const continuationResult = await this.httpClient.continueRunExecution(this.runFriendlyId, this.
|
|
591
|
+
await this.processEnvOverrides();
|
|
592
|
+
const continuationResult = await this.httpClient.continueRunExecution(this.runFriendlyId, this.currentSnapshotId);
|
|
556
593
|
if (!continuationResult.success) {
|
|
557
594
|
throw new Error(continuationResult.error);
|
|
558
595
|
}
|
|
559
596
|
// Track restore count
|
|
560
597
|
this.restoreCount++;
|
|
561
598
|
}
|
|
562
|
-
async exitTaskRunProcessWithoutFailingRun({ flush }) {
|
|
563
|
-
await this.taskRunProcess?.suspend({ flush });
|
|
564
|
-
// No services should be left running after this line - let's make sure of it
|
|
565
|
-
this.shutdown("exitTaskRunProcessWithoutFailingRun");
|
|
566
|
-
}
|
|
567
599
|
/**
|
|
568
600
|
* Processes env overrides from the metadata service. Generally called when we're resuming from a suspended state.
|
|
569
601
|
*/
|
|
570
|
-
async processEnvOverrides(
|
|
602
|
+
async processEnvOverrides() {
|
|
571
603
|
if (!this.env.TRIGGER_METADATA_URL) {
|
|
572
|
-
this.sendDebugLog("
|
|
604
|
+
this.sendDebugLog("No metadata URL, skipping env overrides");
|
|
573
605
|
return;
|
|
574
606
|
}
|
|
575
607
|
const metadataClient = new MetadataClient(this.env.TRIGGER_METADATA_URL);
|
|
576
608
|
const overrides = await metadataClient.getEnvOverrides();
|
|
577
609
|
if (!overrides) {
|
|
578
|
-
this.sendDebugLog("
|
|
610
|
+
this.sendDebugLog("No env overrides, skipping");
|
|
579
611
|
return;
|
|
580
612
|
}
|
|
581
|
-
this.sendDebugLog(
|
|
582
|
-
overrides,
|
|
583
|
-
currentEnv: this.env.raw,
|
|
584
|
-
});
|
|
585
|
-
if (this.env.TRIGGER_RUNNER_ID !== overrides.TRIGGER_RUNNER_ID) {
|
|
586
|
-
this.sendDebugLog("runner ID changed -> run was restored from a checkpoint", {
|
|
587
|
-
currentRunnerId: this.env.TRIGGER_RUNNER_ID,
|
|
588
|
-
newRunnerId: overrides.TRIGGER_RUNNER_ID,
|
|
589
|
-
});
|
|
590
|
-
}
|
|
613
|
+
this.sendDebugLog("Processing env overrides", overrides);
|
|
591
614
|
// Override the env with the new values
|
|
592
615
|
this.env.override(overrides);
|
|
593
616
|
// Update services with new values
|
|
@@ -605,17 +628,17 @@ export class RunExecution {
|
|
|
605
628
|
}
|
|
606
629
|
async onHeartbeat() {
|
|
607
630
|
if (!this.runFriendlyId) {
|
|
608
|
-
this.sendDebugLog("
|
|
631
|
+
this.sendDebugLog("Heartbeat: missing run ID");
|
|
609
632
|
return;
|
|
610
633
|
}
|
|
611
|
-
if (!this.
|
|
612
|
-
this.sendDebugLog("
|
|
634
|
+
if (!this.currentSnapshotId) {
|
|
635
|
+
this.sendDebugLog("Heartbeat: missing snapshot ID");
|
|
613
636
|
return;
|
|
614
637
|
}
|
|
615
|
-
this.sendDebugLog("
|
|
616
|
-
const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.
|
|
638
|
+
this.sendDebugLog("Heartbeat: started");
|
|
639
|
+
const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.currentSnapshotId);
|
|
617
640
|
if (!response.success) {
|
|
618
|
-
this.sendDebugLog("
|
|
641
|
+
this.sendDebugLog("Heartbeat: failed", { error: response.error });
|
|
619
642
|
}
|
|
620
643
|
this.lastHeartbeat = new Date();
|
|
621
644
|
}
|
|
@@ -626,33 +649,13 @@ export class RunExecution {
|
|
|
626
649
|
properties: {
|
|
627
650
|
...properties,
|
|
628
651
|
runId: this.runFriendlyId,
|
|
629
|
-
snapshotId: this.
|
|
630
|
-
executionId: this.id,
|
|
631
|
-
executionRestoreCount: this.restoreCount,
|
|
632
|
-
lastHeartbeat: this.lastHeartbeat?.toISOString(),
|
|
633
|
-
},
|
|
634
|
-
});
|
|
635
|
-
}
|
|
636
|
-
sendRuntimeDebugLog(message, properties, runIdOverride) {
|
|
637
|
-
this.logger.sendDebugLog({
|
|
638
|
-
runId: runIdOverride ?? this.runFriendlyId,
|
|
639
|
-
message: `[runtime] ${message}`,
|
|
640
|
-
print: false,
|
|
641
|
-
properties: {
|
|
642
|
-
...properties,
|
|
643
|
-
runId: this.runFriendlyId,
|
|
644
|
-
snapshotId: this.currentSnapshotFriendlyId,
|
|
652
|
+
snapshotId: this.currentSnapshotId,
|
|
645
653
|
executionId: this.id,
|
|
646
654
|
executionRestoreCount: this.restoreCount,
|
|
647
655
|
lastHeartbeat: this.lastHeartbeat?.toISOString(),
|
|
648
656
|
},
|
|
649
657
|
});
|
|
650
658
|
}
|
|
651
|
-
set suspendable(suspendable) {
|
|
652
|
-
this.snapshotManager?.setSuspendable(suspendable).catch((error) => {
|
|
653
|
-
this.sendDebugLog("failed to set suspendable", { error: error.message });
|
|
654
|
-
});
|
|
655
|
-
}
|
|
656
659
|
// Ensure we can only set this once
|
|
657
660
|
set runFriendlyId(id) {
|
|
658
661
|
if (this._runFriendlyId) {
|
|
@@ -664,7 +667,7 @@ export class RunExecution {
|
|
|
664
667
|
return this._runFriendlyId;
|
|
665
668
|
}
|
|
666
669
|
get currentSnapshotFriendlyId() {
|
|
667
|
-
return this.
|
|
670
|
+
return this.currentSnapshotId;
|
|
668
671
|
}
|
|
669
672
|
get taskRunEnv() {
|
|
670
673
|
return this.currentTaskRunEnv;
|
|
@@ -679,82 +682,19 @@ export class RunExecution {
|
|
|
679
682
|
}
|
|
680
683
|
abortExecution() {
|
|
681
684
|
if (this.isAborted) {
|
|
682
|
-
this.sendDebugLog("
|
|
685
|
+
this.sendDebugLog("Execution already aborted");
|
|
683
686
|
return;
|
|
684
687
|
}
|
|
685
688
|
this.executionAbortController.abort();
|
|
686
|
-
this.
|
|
689
|
+
this.stopServices();
|
|
687
690
|
}
|
|
688
|
-
|
|
691
|
+
stopServices() {
|
|
689
692
|
if (this.isShuttingDown) {
|
|
690
|
-
this.sendDebugLog(`[shutdown] ${reason} (already shutting down)`, {
|
|
691
|
-
firstShutdownReason: this.shutdownReason,
|
|
692
|
-
});
|
|
693
693
|
return;
|
|
694
694
|
}
|
|
695
|
-
this.sendDebugLog(`[shutdown] ${reason}`);
|
|
696
695
|
this.isShuttingDown = true;
|
|
697
|
-
this.shutdownReason = reason;
|
|
698
696
|
this.snapshotPoller?.stop();
|
|
699
|
-
this.
|
|
700
|
-
this.taskRunProcess?.unsafeDetachEvtHandlers();
|
|
701
|
-
}
|
|
702
|
-
async handleSuspendable(suspendableSnapshot) {
|
|
703
|
-
this.sendDebugLog("handleSuspendable", { suspendableSnapshot });
|
|
704
|
-
if (!this.snapshotManager) {
|
|
705
|
-
this.sendDebugLog("handleSuspendable: missing snapshot manager");
|
|
706
|
-
return;
|
|
707
|
-
}
|
|
708
|
-
// Ensure this is the current snapshot
|
|
709
|
-
if (suspendableSnapshot.id !== this.currentSnapshotFriendlyId) {
|
|
710
|
-
this.sendDebugLog("snapshot changed before cleanup, abort", {
|
|
711
|
-
suspendableSnapshot,
|
|
712
|
-
currentSnapshotId: this.currentSnapshotFriendlyId,
|
|
713
|
-
});
|
|
714
|
-
this.abortExecution();
|
|
715
|
-
return;
|
|
716
|
-
}
|
|
717
|
-
// First cleanup the task run process
|
|
718
|
-
const [error] = await tryCatch(this.taskRunProcess?.cleanup(false));
|
|
719
|
-
if (error) {
|
|
720
|
-
this.sendDebugLog("failed to cleanup task run process, carrying on", {
|
|
721
|
-
suspendableSnapshot,
|
|
722
|
-
error: error.message,
|
|
723
|
-
});
|
|
724
|
-
}
|
|
725
|
-
// Double check snapshot hasn't changed after cleanup
|
|
726
|
-
if (suspendableSnapshot.id !== this.currentSnapshotFriendlyId) {
|
|
727
|
-
this.sendDebugLog("snapshot changed after cleanup, abort", {
|
|
728
|
-
suspendableSnapshot,
|
|
729
|
-
currentSnapshotId: this.currentSnapshotFriendlyId,
|
|
730
|
-
});
|
|
731
|
-
this.abortExecution();
|
|
732
|
-
return;
|
|
733
|
-
}
|
|
734
|
-
if (!this.runFriendlyId) {
|
|
735
|
-
this.sendDebugLog("missing run ID for suspension, abort", { suspendableSnapshot });
|
|
736
|
-
this.abortExecution();
|
|
737
|
-
return;
|
|
738
|
-
}
|
|
739
|
-
// Call the suspend API with the current snapshot ID
|
|
740
|
-
const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, suspendableSnapshot.id);
|
|
741
|
-
if (!suspendResult.success) {
|
|
742
|
-
this.sendDebugLog("suspension request failed, staying alive 🎶", {
|
|
743
|
-
suspendableSnapshot,
|
|
744
|
-
error: suspendResult.error,
|
|
745
|
-
});
|
|
746
|
-
// This is fine, we'll wait for the next status change
|
|
747
|
-
return;
|
|
748
|
-
}
|
|
749
|
-
if (!suspendResult.data.ok) {
|
|
750
|
-
this.sendDebugLog("suspension request returned error, staying alive 🎶", {
|
|
751
|
-
suspendableSnapshot,
|
|
752
|
-
error: suspendResult.data.error,
|
|
753
|
-
});
|
|
754
|
-
// This is fine, we'll wait for the next status change
|
|
755
|
-
return;
|
|
756
|
-
}
|
|
757
|
-
this.sendDebugLog("suspending, any day now 🚬", { suspendableSnapshot });
|
|
697
|
+
this.taskRunProcess?.onTaskRunHeartbeat.detach();
|
|
758
698
|
}
|
|
759
699
|
}
|
|
760
700
|
//# sourceMappingURL=execution.js.map
|