trigger.dev 4.0.0-v4-beta.1 → 4.0.0-v4-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/dev/devSupervisor.js +14 -10
- package/dist/esm/dev/devSupervisor.js.map +1 -1
- package/dist/esm/entryPoints/dev-run-controller.js +6 -6
- package/dist/esm/entryPoints/dev-run-controller.js.map +1 -1
- package/dist/esm/entryPoints/managed/controller.d.ts +50 -0
- package/dist/esm/entryPoints/managed/controller.js +441 -0
- package/dist/esm/entryPoints/managed/controller.js.map +1 -0
- package/dist/esm/entryPoints/managed/env.d.ts +170 -0
- package/dist/esm/entryPoints/managed/env.js +191 -0
- package/dist/esm/entryPoints/managed/env.js.map +1 -0
- package/dist/esm/entryPoints/managed/execution.d.ts +94 -0
- package/dist/esm/entryPoints/managed/execution.js +638 -0
- package/dist/esm/entryPoints/managed/execution.js.map +1 -0
- package/dist/esm/entryPoints/managed/heartbeat.d.ts +23 -0
- package/dist/esm/entryPoints/managed/heartbeat.js +68 -0
- package/dist/esm/entryPoints/managed/heartbeat.js.map +1 -0
- package/dist/esm/entryPoints/managed/logger.d.ts +19 -0
- package/dist/esm/entryPoints/managed/logger.js +31 -0
- package/dist/esm/entryPoints/managed/logger.js.map +1 -0
- package/dist/esm/entryPoints/managed/overrides.d.ts +16 -0
- package/dist/esm/entryPoints/managed/overrides.js +17 -0
- package/dist/esm/entryPoints/managed/overrides.js.map +1 -0
- package/dist/esm/entryPoints/managed/poller.d.ts +26 -0
- package/dist/esm/entryPoints/managed/poller.js +94 -0
- package/dist/esm/entryPoints/managed/poller.js.map +1 -0
- package/dist/esm/entryPoints/managed-run-controller.js +8 -1347
- package/dist/esm/entryPoints/managed-run-controller.js.map +1 -1
- package/dist/esm/executions/taskRunProcess.d.ts +2 -2
- package/dist/esm/executions/taskRunProcess.js +1 -1
- package/dist/esm/executions/taskRunProcess.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +3 -3
|
@@ -1,1350 +1,11 @@
|
|
|
1
|
-
import { logger } from "../utilities/logger.js";
|
|
2
|
-
import { TaskRunProcess } from "../executions/taskRunProcess.js";
|
|
3
1
|
import { env as stdEnv } from "std-env";
|
|
4
|
-
import { z } from "zod";
|
|
5
|
-
import { randomUUID } from "crypto";
|
|
6
2
|
import { readJSONFile } from "../utilities/fileSystem.js";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
.pipe(z.date());
|
|
16
|
-
// All IDs are friendly IDs
|
|
17
|
-
const Env = z.object({
|
|
18
|
-
// Set at build time
|
|
19
|
-
TRIGGER_CONTENT_HASH: z.string(),
|
|
20
|
-
TRIGGER_DEPLOYMENT_ID: z.string(),
|
|
21
|
-
TRIGGER_DEPLOYMENT_VERSION: z.string(),
|
|
22
|
-
TRIGGER_PROJECT_ID: z.string(),
|
|
23
|
-
TRIGGER_PROJECT_REF: z.string(),
|
|
24
|
-
NODE_ENV: z.string().default("production"),
|
|
25
|
-
NODE_EXTRA_CA_CERTS: z.string().optional(),
|
|
26
|
-
// Set at runtime
|
|
27
|
-
TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`),
|
|
28
|
-
TRIGGER_ENV_ID: z.string(),
|
|
29
|
-
TRIGGER_RUN_ID: z.string().optional(), // This is only useful for cold starts
|
|
30
|
-
TRIGGER_SNAPSHOT_ID: z.string().optional(), // This is only useful for cold starts
|
|
31
|
-
OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
|
|
32
|
-
TRIGGER_WARM_START_URL: z.string().optional(),
|
|
33
|
-
TRIGGER_WARM_START_CONNECTION_TIMEOUT_MS: z.coerce.number().default(30_000),
|
|
34
|
-
TRIGGER_WARM_START_KEEPALIVE_MS: z.coerce.number().default(300_000),
|
|
35
|
-
TRIGGER_MACHINE_CPU: z.string().default("0"),
|
|
36
|
-
TRIGGER_MACHINE_MEMORY: z.string().default("0"),
|
|
37
|
-
TRIGGER_RUNNER_ID: z.string(),
|
|
38
|
-
TRIGGER_METADATA_URL: z.string().optional(),
|
|
39
|
-
TRIGGER_PRE_SUSPEND_WAIT_MS: z.coerce.number().default(200),
|
|
40
|
-
// Timeline metrics
|
|
41
|
-
TRIGGER_POD_SCHEDULED_AT_MS: DateEnv,
|
|
42
|
-
TRIGGER_DEQUEUED_AT_MS: DateEnv,
|
|
43
|
-
// May be overridden
|
|
44
|
-
TRIGGER_SUPERVISOR_API_PROTOCOL: z.enum(["http", "https"]),
|
|
45
|
-
TRIGGER_SUPERVISOR_API_DOMAIN: z.string(),
|
|
46
|
-
TRIGGER_SUPERVISOR_API_PORT: z.coerce.number(),
|
|
47
|
-
TRIGGER_WORKER_INSTANCE_NAME: z.string(),
|
|
48
|
-
TRIGGER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
|
|
49
|
-
TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().default(5),
|
|
50
|
-
TRIGGER_SUCCESS_EXIT_CODE: z.coerce.number().default(0),
|
|
51
|
-
TRIGGER_FAILURE_EXIT_CODE: z.coerce.number().default(1),
|
|
52
|
-
});
|
|
53
|
-
const env = Env.parse(stdEnv);
|
|
54
|
-
logger.loggerLevel = "debug";
|
|
55
|
-
class MetadataClient {
|
|
56
|
-
url;
|
|
57
|
-
constructor(url) {
|
|
58
|
-
this.url = new URL(url);
|
|
59
|
-
}
|
|
60
|
-
async getEnvOverrides() {
|
|
61
|
-
try {
|
|
62
|
-
const response = await fetch(new URL("/env", this.url));
|
|
63
|
-
return response.json();
|
|
64
|
-
}
|
|
65
|
-
catch (error) {
|
|
66
|
-
console.error("Failed to fetch metadata", { error });
|
|
67
|
-
return null;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
class ManagedRunController {
|
|
72
|
-
taskRunProcess;
|
|
73
|
-
workerManifest;
|
|
74
|
-
httpClient;
|
|
75
|
-
warmStartClient;
|
|
76
|
-
metadataClient;
|
|
77
|
-
socket;
|
|
78
|
-
runHeartbeat;
|
|
79
|
-
heartbeatIntervalSeconds;
|
|
80
|
-
snapshotPoller;
|
|
81
|
-
snapshotPollIntervalSeconds;
|
|
82
|
-
workerApiUrl;
|
|
83
|
-
workerInstanceName;
|
|
84
|
-
runnerId;
|
|
85
|
-
successExitCode = env.TRIGGER_SUCCESS_EXIT_CODE;
|
|
86
|
-
failureExitCode = env.TRIGGER_FAILURE_EXIT_CODE;
|
|
87
|
-
state = { phase: "IDLE" };
|
|
88
|
-
constructor(opts) {
|
|
89
|
-
this.workerManifest = opts.workerManifest;
|
|
90
|
-
this.runnerId = env.TRIGGER_RUNNER_ID;
|
|
91
|
-
this.workerApiUrl = `${env.TRIGGER_SUPERVISOR_API_PROTOCOL}://${env.TRIGGER_SUPERVISOR_API_DOMAIN}:${env.TRIGGER_SUPERVISOR_API_PORT}`;
|
|
92
|
-
this.workerInstanceName = env.TRIGGER_WORKER_INSTANCE_NAME;
|
|
93
|
-
this.httpClient = new WorkloadHttpClient({
|
|
94
|
-
workerApiUrl: this.workerApiUrl,
|
|
95
|
-
runnerId: this.runnerId,
|
|
96
|
-
deploymentId: env.TRIGGER_DEPLOYMENT_ID,
|
|
97
|
-
deploymentVersion: env.TRIGGER_DEPLOYMENT_VERSION,
|
|
98
|
-
projectRef: env.TRIGGER_PROJECT_REF,
|
|
99
|
-
});
|
|
100
|
-
const properties = {
|
|
101
|
-
...env,
|
|
102
|
-
TRIGGER_POD_SCHEDULED_AT_MS: env.TRIGGER_POD_SCHEDULED_AT_MS.toISOString(),
|
|
103
|
-
TRIGGER_DEQUEUED_AT_MS: env.TRIGGER_DEQUEUED_AT_MS.toISOString(),
|
|
104
|
-
};
|
|
105
|
-
this.sendDebugLog({
|
|
106
|
-
runId: env.TRIGGER_RUN_ID,
|
|
107
|
-
message: "Creating run controller",
|
|
108
|
-
properties,
|
|
109
|
-
});
|
|
110
|
-
this.heartbeatIntervalSeconds = env.TRIGGER_HEARTBEAT_INTERVAL_SECONDS;
|
|
111
|
-
this.snapshotPollIntervalSeconds = env.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS;
|
|
112
|
-
if (env.TRIGGER_METADATA_URL) {
|
|
113
|
-
this.metadataClient = new MetadataClient(env.TRIGGER_METADATA_URL);
|
|
114
|
-
}
|
|
115
|
-
if (env.TRIGGER_WARM_START_URL) {
|
|
116
|
-
this.warmStartClient = new WarmStartClient({
|
|
117
|
-
apiUrl: new URL(env.TRIGGER_WARM_START_URL),
|
|
118
|
-
controllerId: env.TRIGGER_WORKLOAD_CONTROLLER_ID,
|
|
119
|
-
deploymentId: env.TRIGGER_DEPLOYMENT_ID,
|
|
120
|
-
deploymentVersion: env.TRIGGER_DEPLOYMENT_VERSION,
|
|
121
|
-
machineCpu: env.TRIGGER_MACHINE_CPU,
|
|
122
|
-
machineMemory: env.TRIGGER_MACHINE_MEMORY,
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
this.snapshotPoller = new HeartbeatService({
|
|
126
|
-
heartbeat: async () => {
|
|
127
|
-
if (!this.runFriendlyId) {
|
|
128
|
-
this.sendDebugLog({
|
|
129
|
-
runId: env.TRIGGER_RUN_ID,
|
|
130
|
-
message: "Skipping snapshot poll, no run ID",
|
|
131
|
-
});
|
|
132
|
-
return;
|
|
133
|
-
}
|
|
134
|
-
this.sendDebugLog({
|
|
135
|
-
runId: env.TRIGGER_RUN_ID,
|
|
136
|
-
message: "Polling for latest snapshot",
|
|
137
|
-
});
|
|
138
|
-
this.sendDebugLog({
|
|
139
|
-
runId: this.runFriendlyId,
|
|
140
|
-
message: `snapshot poll: started`,
|
|
141
|
-
properties: {
|
|
142
|
-
snapshotId: this.snapshotFriendlyId,
|
|
143
|
-
},
|
|
144
|
-
});
|
|
145
|
-
const response = await this.httpClient.getRunExecutionData(this.runFriendlyId);
|
|
146
|
-
if (!response.success) {
|
|
147
|
-
this.sendDebugLog({
|
|
148
|
-
runId: this.runFriendlyId,
|
|
149
|
-
message: "Snapshot poll failed",
|
|
150
|
-
properties: {
|
|
151
|
-
error: response.error,
|
|
152
|
-
},
|
|
153
|
-
});
|
|
154
|
-
this.sendDebugLog({
|
|
155
|
-
runId: this.runFriendlyId,
|
|
156
|
-
message: `snapshot poll: failed`,
|
|
157
|
-
properties: {
|
|
158
|
-
snapshotId: this.snapshotFriendlyId,
|
|
159
|
-
error: response.error,
|
|
160
|
-
},
|
|
161
|
-
});
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
await this.handleSnapshotChange(response.data.execution);
|
|
165
|
-
},
|
|
166
|
-
intervalMs: this.snapshotPollIntervalSeconds * 1000,
|
|
167
|
-
leadingEdge: false,
|
|
168
|
-
onError: async (error) => {
|
|
169
|
-
this.sendDebugLog({
|
|
170
|
-
runId: this.runFriendlyId,
|
|
171
|
-
message: "Failed to poll for snapshot",
|
|
172
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
173
|
-
});
|
|
174
|
-
},
|
|
175
|
-
});
|
|
176
|
-
this.runHeartbeat = new HeartbeatService({
|
|
177
|
-
heartbeat: async () => {
|
|
178
|
-
if (!this.runFriendlyId || !this.snapshotFriendlyId) {
|
|
179
|
-
this.sendDebugLog({
|
|
180
|
-
runId: this.runFriendlyId,
|
|
181
|
-
message: "Skipping heartbeat, no run ID or snapshot ID",
|
|
182
|
-
});
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
this.sendDebugLog({
|
|
186
|
-
runId: this.runFriendlyId,
|
|
187
|
-
message: "heartbeat: started",
|
|
188
|
-
});
|
|
189
|
-
const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.snapshotFriendlyId);
|
|
190
|
-
if (!response.success) {
|
|
191
|
-
this.sendDebugLog({
|
|
192
|
-
runId: this.runFriendlyId,
|
|
193
|
-
message: "heartbeat: failed",
|
|
194
|
-
properties: {
|
|
195
|
-
error: response.error,
|
|
196
|
-
},
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
},
|
|
200
|
-
intervalMs: this.heartbeatIntervalSeconds * 1000,
|
|
201
|
-
leadingEdge: false,
|
|
202
|
-
onError: async (error) => {
|
|
203
|
-
this.sendDebugLog({
|
|
204
|
-
runId: this.runFriendlyId,
|
|
205
|
-
message: "Failed to send heartbeat",
|
|
206
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
207
|
-
});
|
|
208
|
-
},
|
|
209
|
-
});
|
|
210
|
-
process.on("SIGTERM", async () => {
|
|
211
|
-
this.sendDebugLog({
|
|
212
|
-
runId: this.runFriendlyId,
|
|
213
|
-
message: "Received SIGTERM, stopping worker",
|
|
214
|
-
});
|
|
215
|
-
await this.stop();
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
enterRunPhase(run, snapshot) {
|
|
219
|
-
this.onExitRunPhase(run);
|
|
220
|
-
this.state = { phase: "RUN", run, snapshot };
|
|
221
|
-
this.runHeartbeat.start();
|
|
222
|
-
this.snapshotPoller.start();
|
|
223
|
-
}
|
|
224
|
-
enterWarmStartPhase() {
|
|
225
|
-
this.onExitRunPhase();
|
|
226
|
-
this.state = { phase: "WARM_START" };
|
|
227
|
-
}
|
|
228
|
-
// This should only be used when we're already executing a run. Attempt number changes are not allowed.
|
|
229
|
-
updateRunPhase(run, snapshot) {
|
|
230
|
-
if (this.state.phase !== "RUN") {
|
|
231
|
-
this.sendDebugLog({
|
|
232
|
-
runId: run.friendlyId,
|
|
233
|
-
message: `updateRunPhase: Invalid phase for updating snapshot: ${this.state.phase}`,
|
|
234
|
-
properties: {
|
|
235
|
-
currentPhase: this.state.phase,
|
|
236
|
-
snapshotId: snapshot.friendlyId,
|
|
237
|
-
},
|
|
238
|
-
});
|
|
239
|
-
throw new Error(`Invalid phase for updating snapshot: ${this.state.phase}`);
|
|
240
|
-
}
|
|
241
|
-
if (this.state.run.friendlyId !== run.friendlyId) {
|
|
242
|
-
this.sendDebugLog({
|
|
243
|
-
runId: run.friendlyId,
|
|
244
|
-
message: `updateRunPhase: Mismatched run IDs`,
|
|
245
|
-
properties: {
|
|
246
|
-
currentRunId: this.state.run.friendlyId,
|
|
247
|
-
newRunId: run.friendlyId,
|
|
248
|
-
currentSnapshotId: this.state.snapshot.friendlyId,
|
|
249
|
-
newSnapshotId: snapshot.friendlyId,
|
|
250
|
-
},
|
|
251
|
-
});
|
|
252
|
-
throw new Error("Mismatched run IDs");
|
|
253
|
-
}
|
|
254
|
-
if (this.state.snapshot.friendlyId === snapshot.friendlyId) {
|
|
255
|
-
this.sendDebugLog({
|
|
256
|
-
runId: run.friendlyId,
|
|
257
|
-
message: "updateRunPhase: Snapshot not changed",
|
|
258
|
-
properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
|
|
259
|
-
});
|
|
260
|
-
this.sendDebugLog({
|
|
261
|
-
runId: run.friendlyId,
|
|
262
|
-
message: `updateRunPhase: Snapshot not changed`,
|
|
263
|
-
properties: {
|
|
264
|
-
snapshotId: snapshot.friendlyId,
|
|
265
|
-
},
|
|
266
|
-
});
|
|
267
|
-
return;
|
|
268
|
-
}
|
|
269
|
-
if (this.state.run.attemptNumber !== run.attemptNumber) {
|
|
270
|
-
this.sendDebugLog({
|
|
271
|
-
runId: run.friendlyId,
|
|
272
|
-
message: `updateRunPhase: Attempt number changed`,
|
|
273
|
-
properties: {
|
|
274
|
-
oldAttemptNumber: this.state.run.attemptNumber ?? undefined,
|
|
275
|
-
newAttemptNumber: run.attemptNumber ?? undefined,
|
|
276
|
-
},
|
|
277
|
-
});
|
|
278
|
-
throw new Error("Attempt number changed");
|
|
279
|
-
}
|
|
280
|
-
this.state = {
|
|
281
|
-
phase: "RUN",
|
|
282
|
-
run: {
|
|
283
|
-
friendlyId: run.friendlyId,
|
|
284
|
-
attemptNumber: run.attemptNumber,
|
|
285
|
-
},
|
|
286
|
-
snapshot: {
|
|
287
|
-
friendlyId: snapshot.friendlyId,
|
|
288
|
-
},
|
|
289
|
-
};
|
|
290
|
-
}
|
|
291
|
-
onExitRunPhase(newRun = undefined) {
|
|
292
|
-
// We're not in a run phase, nothing to do
|
|
293
|
-
if (this.state.phase !== "RUN") {
|
|
294
|
-
this.sendDebugLog({
|
|
295
|
-
runId: this.runFriendlyId,
|
|
296
|
-
message: "onExitRunPhase: Not in run phase, skipping",
|
|
297
|
-
properties: { phase: this.state.phase },
|
|
298
|
-
});
|
|
299
|
-
return;
|
|
300
|
-
}
|
|
301
|
-
// This is still the same run, so we're not exiting the phase
|
|
302
|
-
if (newRun?.friendlyId === this.state.run.friendlyId) {
|
|
303
|
-
this.sendDebugLog({
|
|
304
|
-
runId: this.runFriendlyId,
|
|
305
|
-
message: "onExitRunPhase: Same run, skipping",
|
|
306
|
-
properties: { newRun: newRun?.friendlyId },
|
|
307
|
-
});
|
|
308
|
-
return;
|
|
309
|
-
}
|
|
310
|
-
this.sendDebugLog({
|
|
311
|
-
runId: this.runFriendlyId,
|
|
312
|
-
message: "onExitRunPhase: Exiting run phase",
|
|
313
|
-
properties: { newRun: newRun?.friendlyId },
|
|
314
|
-
});
|
|
315
|
-
this.runHeartbeat.stop();
|
|
316
|
-
this.snapshotPoller.stop();
|
|
317
|
-
const { run, snapshot } = this.state;
|
|
318
|
-
this.unsubscribeFromRunNotifications({ run, snapshot });
|
|
319
|
-
}
|
|
320
|
-
subscribeToRunNotifications({ run, snapshot }) {
|
|
321
|
-
this.socket.emit("run:start", {
|
|
322
|
-
version: "1",
|
|
323
|
-
run: {
|
|
324
|
-
friendlyId: run.friendlyId,
|
|
325
|
-
},
|
|
326
|
-
snapshot: {
|
|
327
|
-
friendlyId: snapshot.friendlyId,
|
|
328
|
-
},
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
unsubscribeFromRunNotifications({ run, snapshot }) {
|
|
332
|
-
this.socket.emit("run:stop", {
|
|
333
|
-
version: "1",
|
|
334
|
-
run: {
|
|
335
|
-
friendlyId: run.friendlyId,
|
|
336
|
-
},
|
|
337
|
-
snapshot: {
|
|
338
|
-
friendlyId: snapshot.friendlyId,
|
|
339
|
-
},
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
get runFriendlyId() {
|
|
343
|
-
if (this.state.phase !== "RUN") {
|
|
344
|
-
return undefined;
|
|
345
|
-
}
|
|
346
|
-
return this.state.run.friendlyId;
|
|
347
|
-
}
|
|
348
|
-
get snapshotFriendlyId() {
|
|
349
|
-
if (this.state.phase !== "RUN") {
|
|
350
|
-
return;
|
|
351
|
-
}
|
|
352
|
-
return this.state.snapshot.friendlyId;
|
|
353
|
-
}
|
|
354
|
-
handleSnapshotChangeLock = false;
|
|
355
|
-
async handleSnapshotChange({ run, snapshot, completedWaitpoints, }) {
|
|
356
|
-
if (this.handleSnapshotChangeLock) {
|
|
357
|
-
this.sendDebugLog({
|
|
358
|
-
runId: run.friendlyId,
|
|
359
|
-
message: "handleSnapshotChange: already in progress",
|
|
360
|
-
});
|
|
361
|
-
return;
|
|
362
|
-
}
|
|
363
|
-
this.handleSnapshotChangeLock = true;
|
|
364
|
-
try {
|
|
365
|
-
if (!this.snapshotFriendlyId) {
|
|
366
|
-
this.sendDebugLog({
|
|
367
|
-
runId: run.friendlyId,
|
|
368
|
-
message: "handleSnapshotChange: Missing snapshot ID",
|
|
369
|
-
properties: {
|
|
370
|
-
newSnapshotId: snapshot.friendlyId,
|
|
371
|
-
newSnapshotStatus: snapshot.executionStatus,
|
|
372
|
-
},
|
|
373
|
-
});
|
|
374
|
-
this.sendDebugLog({
|
|
375
|
-
runId: run.friendlyId,
|
|
376
|
-
message: "snapshot change: missing snapshot ID",
|
|
377
|
-
properties: {
|
|
378
|
-
newSnapshotId: snapshot.friendlyId,
|
|
379
|
-
newSnapshotStatus: snapshot.executionStatus,
|
|
380
|
-
},
|
|
381
|
-
});
|
|
382
|
-
return;
|
|
383
|
-
}
|
|
384
|
-
if (this.snapshotFriendlyId === snapshot.friendlyId) {
|
|
385
|
-
this.sendDebugLog({
|
|
386
|
-
runId: run.friendlyId,
|
|
387
|
-
message: "handleSnapshotChange: snapshot not changed, skipping",
|
|
388
|
-
properties: { snapshot: snapshot.friendlyId },
|
|
389
|
-
});
|
|
390
|
-
this.sendDebugLog({
|
|
391
|
-
runId: run.friendlyId,
|
|
392
|
-
message: "snapshot change: skipping, no change",
|
|
393
|
-
properties: {
|
|
394
|
-
snapshotId: this.snapshotFriendlyId,
|
|
395
|
-
snapshotStatus: snapshot.executionStatus,
|
|
396
|
-
},
|
|
397
|
-
});
|
|
398
|
-
return;
|
|
399
|
-
}
|
|
400
|
-
this.sendDebugLog({
|
|
401
|
-
runId: run.friendlyId,
|
|
402
|
-
message: `snapshot change: ${snapshot.executionStatus}`,
|
|
403
|
-
properties: {
|
|
404
|
-
oldSnapshotId: this.snapshotFriendlyId,
|
|
405
|
-
newSnapshotId: snapshot.friendlyId,
|
|
406
|
-
completedWaitpoints: completedWaitpoints.length,
|
|
407
|
-
},
|
|
408
|
-
});
|
|
409
|
-
try {
|
|
410
|
-
this.updateRunPhase(run, snapshot);
|
|
411
|
-
}
|
|
412
|
-
catch (error) {
|
|
413
|
-
this.sendDebugLog({
|
|
414
|
-
runId: run.friendlyId,
|
|
415
|
-
message: "snapshot change: failed to update run phase",
|
|
416
|
-
properties: {
|
|
417
|
-
currentPhase: this.state.phase,
|
|
418
|
-
error: error instanceof Error ? error.message : String(error),
|
|
419
|
-
},
|
|
420
|
-
});
|
|
421
|
-
this.waitForNextRun();
|
|
422
|
-
return;
|
|
423
|
-
}
|
|
424
|
-
switch (snapshot.executionStatus) {
|
|
425
|
-
case "PENDING_CANCEL": {
|
|
426
|
-
try {
|
|
427
|
-
await this.cancelAttempt(run.friendlyId);
|
|
428
|
-
}
|
|
429
|
-
catch (error) {
|
|
430
|
-
this.sendDebugLog({
|
|
431
|
-
runId: run.friendlyId,
|
|
432
|
-
message: "snapshot change: failed to cancel attempt",
|
|
433
|
-
properties: {
|
|
434
|
-
error: error instanceof Error ? error.message : String(error),
|
|
435
|
-
},
|
|
436
|
-
});
|
|
437
|
-
this.waitForNextRun();
|
|
438
|
-
return;
|
|
439
|
-
}
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
case "FINISHED": {
|
|
443
|
-
this.sendDebugLog({
|
|
444
|
-
runId: run.friendlyId,
|
|
445
|
-
message: "Run is finished, will wait for next run",
|
|
446
|
-
});
|
|
447
|
-
if (this.activeRunExecution) {
|
|
448
|
-
// Let's pretend we've just suspended the run. This will kill the process and should automatically wait for the next run.
|
|
449
|
-
// We still explicitly call waitForNextRun() afterwards in case of race conditions. Locks will prevent this from causing issues.
|
|
450
|
-
await this.taskRunProcess?.suspend();
|
|
451
|
-
}
|
|
452
|
-
this.waitForNextRun();
|
|
453
|
-
return;
|
|
454
|
-
}
|
|
455
|
-
case "QUEUED_EXECUTING":
|
|
456
|
-
case "EXECUTING_WITH_WAITPOINTS": {
|
|
457
|
-
this.sendDebugLog({
|
|
458
|
-
runId: run.friendlyId,
|
|
459
|
-
message: "Run is executing with waitpoints",
|
|
460
|
-
properties: { snapshot: snapshot.friendlyId },
|
|
461
|
-
});
|
|
462
|
-
try {
|
|
463
|
-
// This should never throw. It should also never fail the run.
|
|
464
|
-
await this.taskRunProcess?.cleanup(false);
|
|
465
|
-
}
|
|
466
|
-
catch (error) {
|
|
467
|
-
this.sendDebugLog({
|
|
468
|
-
runId: run.friendlyId,
|
|
469
|
-
message: "Failed to cleanup task run process",
|
|
470
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
471
|
-
});
|
|
472
|
-
}
|
|
473
|
-
if (snapshot.friendlyId !== this.snapshotFriendlyId) {
|
|
474
|
-
this.sendDebugLog({
|
|
475
|
-
runId: run.friendlyId,
|
|
476
|
-
message: "Snapshot changed after cleanup, abort",
|
|
477
|
-
properties: {
|
|
478
|
-
oldSnapshotId: snapshot.friendlyId,
|
|
479
|
-
newSnapshotId: this.snapshotFriendlyId,
|
|
480
|
-
},
|
|
481
|
-
});
|
|
482
|
-
return;
|
|
483
|
-
}
|
|
484
|
-
await sleep(env.TRIGGER_PRE_SUSPEND_WAIT_MS);
|
|
485
|
-
if (snapshot.friendlyId !== this.snapshotFriendlyId) {
|
|
486
|
-
this.sendDebugLog({
|
|
487
|
-
runId: run.friendlyId,
|
|
488
|
-
message: "Snapshot changed after suspend threshold, abort",
|
|
489
|
-
properties: {
|
|
490
|
-
oldSnapshotId: snapshot.friendlyId,
|
|
491
|
-
newSnapshotId: this.snapshotFriendlyId,
|
|
492
|
-
},
|
|
493
|
-
});
|
|
494
|
-
return;
|
|
495
|
-
}
|
|
496
|
-
if (!this.runFriendlyId || !this.snapshotFriendlyId) {
|
|
497
|
-
this.sendDebugLog({
|
|
498
|
-
runId: run.friendlyId,
|
|
499
|
-
message: "handleSnapshotChange: Missing run ID or snapshot ID after suspension, abort",
|
|
500
|
-
properties: {
|
|
501
|
-
runId: this.runFriendlyId,
|
|
502
|
-
snapshotId: this.snapshotFriendlyId,
|
|
503
|
-
},
|
|
504
|
-
});
|
|
505
|
-
return;
|
|
506
|
-
}
|
|
507
|
-
const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, this.snapshotFriendlyId);
|
|
508
|
-
if (!suspendResult.success) {
|
|
509
|
-
this.sendDebugLog({
|
|
510
|
-
runId: run.friendlyId,
|
|
511
|
-
message: "Failed to suspend run, staying alive 🎶",
|
|
512
|
-
properties: {
|
|
513
|
-
error: suspendResult.error,
|
|
514
|
-
},
|
|
515
|
-
});
|
|
516
|
-
this.sendDebugLog({
|
|
517
|
-
runId: run.friendlyId,
|
|
518
|
-
message: "checkpoint: suspend request failed",
|
|
519
|
-
properties: {
|
|
520
|
-
snapshotId: snapshot.friendlyId,
|
|
521
|
-
error: suspendResult.error,
|
|
522
|
-
},
|
|
523
|
-
});
|
|
524
|
-
return;
|
|
525
|
-
}
|
|
526
|
-
if (!suspendResult.data.ok) {
|
|
527
|
-
this.sendDebugLog({
|
|
528
|
-
runId: run.friendlyId,
|
|
529
|
-
message: "checkpoint: failed to suspend run",
|
|
530
|
-
properties: {
|
|
531
|
-
snapshotId: snapshot.friendlyId,
|
|
532
|
-
error: suspendResult.data.error,
|
|
533
|
-
},
|
|
534
|
-
});
|
|
535
|
-
return;
|
|
536
|
-
}
|
|
537
|
-
this.sendDebugLog({
|
|
538
|
-
runId: run.friendlyId,
|
|
539
|
-
message: "Suspending, any day now 🚬",
|
|
540
|
-
properties: { ok: suspendResult.data.ok },
|
|
541
|
-
});
|
|
542
|
-
return;
|
|
543
|
-
}
|
|
544
|
-
case "SUSPENDED": {
|
|
545
|
-
this.sendDebugLog({
|
|
546
|
-
runId: run.friendlyId,
|
|
547
|
-
message: "Run was suspended, kill the process and wait for more runs",
|
|
548
|
-
properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
|
|
549
|
-
});
|
|
550
|
-
// This will kill the process and fail the execution with a SuspendedProcessError
|
|
551
|
-
await this.taskRunProcess?.suspend();
|
|
552
|
-
return;
|
|
553
|
-
}
|
|
554
|
-
case "PENDING_EXECUTING": {
|
|
555
|
-
this.sendDebugLog({
|
|
556
|
-
runId: run.friendlyId,
|
|
557
|
-
message: "Run is pending execution",
|
|
558
|
-
properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
|
|
559
|
-
});
|
|
560
|
-
if (completedWaitpoints.length === 0) {
|
|
561
|
-
this.sendDebugLog({
|
|
562
|
-
runId: run.friendlyId,
|
|
563
|
-
message: "No waitpoints to complete, nothing to do",
|
|
564
|
-
});
|
|
565
|
-
return;
|
|
566
|
-
}
|
|
567
|
-
// There are waitpoints to complete so we've been restored after being suspended
|
|
568
|
-
// Short delay to give websocket time to reconnect
|
|
569
|
-
await sleep(100);
|
|
570
|
-
// Env may have changed after restore
|
|
571
|
-
await this.processEnvOverrides();
|
|
572
|
-
// We need to let the platform know we're ready to continue
|
|
573
|
-
const continuationResult = await this.httpClient.continueRunExecution(run.friendlyId, snapshot.friendlyId);
|
|
574
|
-
if (!continuationResult.success) {
|
|
575
|
-
this.sendDebugLog({
|
|
576
|
-
runId: run.friendlyId,
|
|
577
|
-
message: "failed to continue execution",
|
|
578
|
-
properties: {
|
|
579
|
-
error: continuationResult.error,
|
|
580
|
-
},
|
|
581
|
-
});
|
|
582
|
-
this.waitForNextRun();
|
|
583
|
-
return;
|
|
584
|
-
}
|
|
585
|
-
return;
|
|
586
|
-
}
|
|
587
|
-
case "EXECUTING": {
|
|
588
|
-
this.sendDebugLog({
|
|
589
|
-
runId: run.friendlyId,
|
|
590
|
-
message: "Run is now executing",
|
|
591
|
-
properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
|
|
592
|
-
});
|
|
593
|
-
if (completedWaitpoints.length === 0) {
|
|
594
|
-
return;
|
|
595
|
-
}
|
|
596
|
-
this.sendDebugLog({
|
|
597
|
-
runId: run.friendlyId,
|
|
598
|
-
message: "Processing completed waitpoints",
|
|
599
|
-
properties: { completedWaitpoints: completedWaitpoints.length },
|
|
600
|
-
});
|
|
601
|
-
if (!this.taskRunProcess) {
|
|
602
|
-
this.sendDebugLog({
|
|
603
|
-
runId: run.friendlyId,
|
|
604
|
-
message: "No task run process, ignoring completed waitpoints",
|
|
605
|
-
properties: { completedWaitpoints: completedWaitpoints.length },
|
|
606
|
-
});
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
for (const waitpoint of completedWaitpoints) {
|
|
610
|
-
this.taskRunProcess.waitpointCompleted(waitpoint);
|
|
611
|
-
}
|
|
612
|
-
return;
|
|
613
|
-
}
|
|
614
|
-
case "RUN_CREATED":
|
|
615
|
-
case "QUEUED": {
|
|
616
|
-
this.sendDebugLog({
|
|
617
|
-
runId: run.friendlyId,
|
|
618
|
-
message: "Status change not handled",
|
|
619
|
-
properties: { status: snapshot.executionStatus },
|
|
620
|
-
});
|
|
621
|
-
return;
|
|
622
|
-
}
|
|
623
|
-
default: {
|
|
624
|
-
assertExhaustive(snapshot.executionStatus);
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
catch (error) {
|
|
629
|
-
this.sendDebugLog({
|
|
630
|
-
runId: run.friendlyId,
|
|
631
|
-
message: "snapshot change: unexpected error",
|
|
632
|
-
properties: {
|
|
633
|
-
snapshotId: snapshot.friendlyId,
|
|
634
|
-
error: error instanceof Error ? error.message : String(error),
|
|
635
|
-
},
|
|
636
|
-
});
|
|
637
|
-
}
|
|
638
|
-
finally {
|
|
639
|
-
this.handleSnapshotChangeLock = false;
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
async processEnvOverrides() {
|
|
643
|
-
if (!this.metadataClient) {
|
|
644
|
-
this.sendDebugLog({
|
|
645
|
-
runId: this.runFriendlyId,
|
|
646
|
-
message: "No metadata client, skipping env overrides",
|
|
647
|
-
});
|
|
648
|
-
return;
|
|
649
|
-
}
|
|
650
|
-
const overrides = await this.metadataClient.getEnvOverrides();
|
|
651
|
-
if (!overrides) {
|
|
652
|
-
this.sendDebugLog({
|
|
653
|
-
runId: this.runFriendlyId,
|
|
654
|
-
message: "No env overrides, skipping",
|
|
655
|
-
});
|
|
656
|
-
return;
|
|
657
|
-
}
|
|
658
|
-
this.sendDebugLog({
|
|
659
|
-
runId: this.runFriendlyId,
|
|
660
|
-
message: "Processing env overrides",
|
|
661
|
-
properties: { ...overrides },
|
|
662
|
-
});
|
|
663
|
-
if (overrides.TRIGGER_SUCCESS_EXIT_CODE) {
|
|
664
|
-
this.successExitCode = overrides.TRIGGER_SUCCESS_EXIT_CODE;
|
|
665
|
-
}
|
|
666
|
-
if (overrides.TRIGGER_FAILURE_EXIT_CODE) {
|
|
667
|
-
this.failureExitCode = overrides.TRIGGER_FAILURE_EXIT_CODE;
|
|
668
|
-
}
|
|
669
|
-
if (overrides.TRIGGER_HEARTBEAT_INTERVAL_SECONDS) {
|
|
670
|
-
this.heartbeatIntervalSeconds = overrides.TRIGGER_HEARTBEAT_INTERVAL_SECONDS;
|
|
671
|
-
this.runHeartbeat.updateInterval(this.heartbeatIntervalSeconds * 1000);
|
|
672
|
-
}
|
|
673
|
-
if (overrides.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS) {
|
|
674
|
-
this.snapshotPollIntervalSeconds = overrides.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS;
|
|
675
|
-
this.snapshotPoller.updateInterval(this.snapshotPollIntervalSeconds * 1000);
|
|
676
|
-
}
|
|
677
|
-
if (overrides.TRIGGER_WORKER_INSTANCE_NAME) {
|
|
678
|
-
this.workerInstanceName = overrides.TRIGGER_WORKER_INSTANCE_NAME;
|
|
679
|
-
}
|
|
680
|
-
if (overrides.TRIGGER_SUPERVISOR_API_PROTOCOL ||
|
|
681
|
-
overrides.TRIGGER_SUPERVISOR_API_DOMAIN ||
|
|
682
|
-
overrides.TRIGGER_SUPERVISOR_API_PORT) {
|
|
683
|
-
const protocol = overrides.TRIGGER_SUPERVISOR_API_PROTOCOL ?? env.TRIGGER_SUPERVISOR_API_PROTOCOL;
|
|
684
|
-
const domain = overrides.TRIGGER_SUPERVISOR_API_DOMAIN ?? env.TRIGGER_SUPERVISOR_API_DOMAIN;
|
|
685
|
-
const port = overrides.TRIGGER_SUPERVISOR_API_PORT ?? env.TRIGGER_SUPERVISOR_API_PORT;
|
|
686
|
-
this.workerApiUrl = `${protocol}://${domain}:${port}`;
|
|
687
|
-
this.httpClient.updateApiUrl(this.workerApiUrl);
|
|
688
|
-
}
|
|
689
|
-
if (overrides.TRIGGER_RUNNER_ID) {
|
|
690
|
-
this.runnerId = overrides.TRIGGER_RUNNER_ID;
|
|
691
|
-
this.httpClient.updateRunnerId(this.runnerId);
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
activeRunExecution = null;
|
|
695
|
-
async startAndExecuteRunAttempt({ runFriendlyId, snapshotFriendlyId, dequeuedAt, podScheduledAt, isWarmStart, skipLockCheckForImmediateRetry: skipLockCheck, }) {
|
|
696
|
-
if (!skipLockCheck && this.activeRunExecution) {
|
|
697
|
-
this.sendDebugLog({
|
|
698
|
-
runId: runFriendlyId,
|
|
699
|
-
message: "startAndExecuteRunAttempt: already in progress",
|
|
700
|
-
});
|
|
701
|
-
return;
|
|
702
|
-
}
|
|
703
|
-
const execution = async () => {
|
|
704
|
-
if (!this.socket) {
|
|
705
|
-
this.sendDebugLog({
|
|
706
|
-
runId: runFriendlyId,
|
|
707
|
-
message: "Starting run without socket connection",
|
|
708
|
-
});
|
|
709
|
-
}
|
|
710
|
-
this.subscribeToRunNotifications({
|
|
711
|
-
run: { friendlyId: runFriendlyId },
|
|
712
|
-
snapshot: { friendlyId: snapshotFriendlyId },
|
|
713
|
-
});
|
|
714
|
-
const attemptStartedAt = Date.now();
|
|
715
|
-
const start = await this.httpClient.startRunAttempt(runFriendlyId, snapshotFriendlyId, {
|
|
716
|
-
isWarmStart,
|
|
717
|
-
});
|
|
718
|
-
if (!start.success) {
|
|
719
|
-
this.sendDebugLog({
|
|
720
|
-
runId: runFriendlyId,
|
|
721
|
-
message: "Failed to start run",
|
|
722
|
-
properties: { error: start.error },
|
|
723
|
-
});
|
|
724
|
-
this.sendDebugLog({
|
|
725
|
-
runId: runFriendlyId,
|
|
726
|
-
message: "failed to start run attempt",
|
|
727
|
-
properties: {
|
|
728
|
-
error: start.error,
|
|
729
|
-
},
|
|
730
|
-
});
|
|
731
|
-
this.waitForNextRun();
|
|
732
|
-
return;
|
|
733
|
-
}
|
|
734
|
-
const attemptDuration = Date.now() - attemptStartedAt;
|
|
735
|
-
const { run, snapshot, execution, envVars } = start.data;
|
|
736
|
-
this.sendDebugLog({
|
|
737
|
-
runId: run.friendlyId,
|
|
738
|
-
message: "Started run",
|
|
739
|
-
properties: { snapshot: snapshot.friendlyId },
|
|
740
|
-
});
|
|
741
|
-
this.enterRunPhase(run, snapshot);
|
|
742
|
-
const metrics = [
|
|
743
|
-
{
|
|
744
|
-
name: "start",
|
|
745
|
-
event: "create_attempt",
|
|
746
|
-
timestamp: attemptStartedAt,
|
|
747
|
-
duration: attemptDuration,
|
|
748
|
-
},
|
|
749
|
-
]
|
|
750
|
-
.concat(dequeuedAt
|
|
751
|
-
? [
|
|
752
|
-
{
|
|
753
|
-
name: "start",
|
|
754
|
-
event: "dequeue",
|
|
755
|
-
timestamp: dequeuedAt.getTime(),
|
|
756
|
-
duration: 0,
|
|
757
|
-
},
|
|
758
|
-
]
|
|
759
|
-
: [])
|
|
760
|
-
.concat(podScheduledAt
|
|
761
|
-
? [
|
|
762
|
-
{
|
|
763
|
-
name: "start",
|
|
764
|
-
event: "pod_scheduled",
|
|
765
|
-
timestamp: podScheduledAt.getTime(),
|
|
766
|
-
duration: 0,
|
|
767
|
-
},
|
|
768
|
-
]
|
|
769
|
-
: []);
|
|
770
|
-
const taskRunEnv = {
|
|
771
|
-
...gatherProcessEnv(),
|
|
772
|
-
...envVars,
|
|
773
|
-
};
|
|
774
|
-
try {
|
|
775
|
-
return await this.executeRun({
|
|
776
|
-
run,
|
|
777
|
-
snapshot,
|
|
778
|
-
envVars: taskRunEnv,
|
|
779
|
-
execution,
|
|
780
|
-
metrics,
|
|
781
|
-
isWarmStart,
|
|
782
|
-
});
|
|
783
|
-
}
|
|
784
|
-
catch (error) {
|
|
785
|
-
if (error instanceof SuspendedProcessError) {
|
|
786
|
-
this.sendDebugLog({
|
|
787
|
-
runId: run.friendlyId,
|
|
788
|
-
message: "Run was suspended and task run process was killed, waiting for next run",
|
|
789
|
-
properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
|
|
790
|
-
});
|
|
791
|
-
this.waitForNextRun();
|
|
792
|
-
return;
|
|
793
|
-
}
|
|
794
|
-
this.sendDebugLog({
|
|
795
|
-
runId: run.friendlyId,
|
|
796
|
-
message: "Error while executing attempt",
|
|
797
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
798
|
-
});
|
|
799
|
-
this.sendDebugLog({
|
|
800
|
-
runId: run.friendlyId,
|
|
801
|
-
message: "Submitting attempt completion",
|
|
802
|
-
properties: {
|
|
803
|
-
snapshotId: snapshot.friendlyId,
|
|
804
|
-
updatedSnapshotId: this.snapshotFriendlyId,
|
|
805
|
-
},
|
|
806
|
-
});
|
|
807
|
-
const completion = {
|
|
808
|
-
id: execution.run.id,
|
|
809
|
-
ok: false,
|
|
810
|
-
retry: undefined,
|
|
811
|
-
error: TaskRunProcess.parseExecuteError(error),
|
|
812
|
-
};
|
|
813
|
-
const completionResult = await this.httpClient.completeRunAttempt(run.friendlyId,
|
|
814
|
-
// FIXME: if the snapshot has changed since starting the run, this won't be accurate
|
|
815
|
-
// ..but we probably shouldn't fetch the latest snapshot either because we may be in an "unhealthy" state while the next runner has already taken over
|
|
816
|
-
this.snapshotFriendlyId ?? snapshot.friendlyId, { completion });
|
|
817
|
-
if (!completionResult.success) {
|
|
818
|
-
this.sendDebugLog({
|
|
819
|
-
runId: run.friendlyId,
|
|
820
|
-
message: "Failed to submit completion after error",
|
|
821
|
-
properties: { error: completionResult.error },
|
|
822
|
-
});
|
|
823
|
-
this.sendDebugLog({
|
|
824
|
-
runId: run.friendlyId,
|
|
825
|
-
message: "completion: failed to submit after error",
|
|
826
|
-
properties: {
|
|
827
|
-
error: completionResult.error,
|
|
828
|
-
},
|
|
829
|
-
});
|
|
830
|
-
this.waitForNextRun();
|
|
831
|
-
return;
|
|
832
|
-
}
|
|
833
|
-
this.sendDebugLog({
|
|
834
|
-
runId: run.friendlyId,
|
|
835
|
-
message: "Attempt completion submitted after error",
|
|
836
|
-
properties: {
|
|
837
|
-
attemptStatus: completionResult.data.result.attemptStatus,
|
|
838
|
-
runId: completionResult.data.result.run.friendlyId,
|
|
839
|
-
snapshotId: completionResult.data.result.snapshot.friendlyId,
|
|
840
|
-
},
|
|
841
|
-
});
|
|
842
|
-
try {
|
|
843
|
-
await this.handleCompletionResult(completion, completionResult.data.result);
|
|
844
|
-
}
|
|
845
|
-
catch (error) {
|
|
846
|
-
this.sendDebugLog({
|
|
847
|
-
runId: run.friendlyId,
|
|
848
|
-
message: "Failed to handle completion result after error",
|
|
849
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
850
|
-
});
|
|
851
|
-
this.waitForNextRun();
|
|
852
|
-
return;
|
|
853
|
-
}
|
|
854
|
-
}
|
|
855
|
-
};
|
|
856
|
-
this.activeRunExecution = execution();
|
|
857
|
-
try {
|
|
858
|
-
await this.activeRunExecution;
|
|
859
|
-
}
|
|
860
|
-
catch (error) {
|
|
861
|
-
this.sendDebugLog({
|
|
862
|
-
runId: runFriendlyId,
|
|
863
|
-
message: "startAndExecuteRunAttempt: unexpected error",
|
|
864
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
865
|
-
});
|
|
866
|
-
}
|
|
867
|
-
finally {
|
|
868
|
-
this.activeRunExecution = null;
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
waitForNextRunLock = false;
|
|
872
|
-
/** This will kill the child process before spinning up a new one. It will never throw,
|
|
873
|
-
* but may exit the process on any errors or when no runs are available after the
|
|
874
|
-
* configured duration. */
|
|
875
|
-
async waitForNextRun() {
|
|
876
|
-
if (this.waitForNextRunLock) {
|
|
877
|
-
this.sendDebugLog({
|
|
878
|
-
runId: this.runFriendlyId,
|
|
879
|
-
message: "waitForNextRun: already in progress",
|
|
880
|
-
});
|
|
881
|
-
return;
|
|
882
|
-
}
|
|
883
|
-
this.waitForNextRunLock = true;
|
|
884
|
-
const previousRunId = this.runFriendlyId;
|
|
885
|
-
try {
|
|
886
|
-
// If there's a run execution in progress, we need to kill it and wait for it to finish
|
|
887
|
-
if (this.activeRunExecution) {
|
|
888
|
-
this.sendDebugLog({
|
|
889
|
-
runId: this.runFriendlyId,
|
|
890
|
-
message: "waitForNextRun: waiting for existing run execution to finish",
|
|
891
|
-
});
|
|
892
|
-
await this.activeRunExecution;
|
|
893
|
-
}
|
|
894
|
-
// Just for good measure
|
|
895
|
-
await this.taskRunProcess?.kill("SIGKILL");
|
|
896
|
-
this.sendDebugLog({
|
|
897
|
-
runId: this.runFriendlyId,
|
|
898
|
-
message: "waitForNextRun: waiting for next run",
|
|
899
|
-
});
|
|
900
|
-
this.enterWarmStartPhase();
|
|
901
|
-
if (!this.warmStartClient) {
|
|
902
|
-
this.sendDebugLog({
|
|
903
|
-
runId: this.runFriendlyId,
|
|
904
|
-
message: "waitForNextRun: warm starts disabled, shutting down",
|
|
905
|
-
});
|
|
906
|
-
this.exitProcess(this.successExitCode);
|
|
907
|
-
}
|
|
908
|
-
if (this.taskRunProcess) {
|
|
909
|
-
this.sendDebugLog({
|
|
910
|
-
runId: this.runFriendlyId,
|
|
911
|
-
message: "waitForNextRun: eagerly recreating task run process with options",
|
|
912
|
-
});
|
|
913
|
-
this.taskRunProcess = new TaskRunProcess({
|
|
914
|
-
...this.taskRunProcess.options,
|
|
915
|
-
isWarmStart: true,
|
|
916
|
-
}).initialize();
|
|
917
|
-
}
|
|
918
|
-
else {
|
|
919
|
-
this.sendDebugLog({
|
|
920
|
-
runId: this.runFriendlyId,
|
|
921
|
-
message: "waitForNextRun: no existing task run process, so we can't eagerly recreate it",
|
|
922
|
-
});
|
|
923
|
-
}
|
|
924
|
-
// Check the service is up and get additional warm start config
|
|
925
|
-
const connect = await this.warmStartClient.connect();
|
|
926
|
-
if (!connect.success) {
|
|
927
|
-
this.sendDebugLog({
|
|
928
|
-
runId: this.runFriendlyId,
|
|
929
|
-
message: "waitForNextRun: failed to connect to warm start service",
|
|
930
|
-
properties: {
|
|
931
|
-
warmStartUrl: env.TRIGGER_WARM_START_URL,
|
|
932
|
-
error: connect.error,
|
|
933
|
-
},
|
|
934
|
-
});
|
|
935
|
-
this.exitProcess(this.successExitCode);
|
|
936
|
-
}
|
|
937
|
-
const connectionTimeoutMs = connect.data.connectionTimeoutMs ?? env.TRIGGER_WARM_START_CONNECTION_TIMEOUT_MS;
|
|
938
|
-
const keepaliveMs = connect.data.keepaliveMs ?? env.TRIGGER_WARM_START_KEEPALIVE_MS;
|
|
939
|
-
this.sendDebugLog({
|
|
940
|
-
runId: this.runFriendlyId,
|
|
941
|
-
message: "waitForNextRun: connected to warm start service",
|
|
942
|
-
properties: {
|
|
943
|
-
connectionTimeoutMs,
|
|
944
|
-
keepaliveMs,
|
|
945
|
-
},
|
|
946
|
-
});
|
|
947
|
-
if (previousRunId) {
|
|
948
|
-
this.sendDebugLog({
|
|
949
|
-
runId: previousRunId,
|
|
950
|
-
message: "warm start: received config",
|
|
951
|
-
properties: {
|
|
952
|
-
connectionTimeoutMs,
|
|
953
|
-
keepaliveMs,
|
|
954
|
-
},
|
|
955
|
-
});
|
|
956
|
-
}
|
|
957
|
-
if (!connectionTimeoutMs || !keepaliveMs) {
|
|
958
|
-
this.sendDebugLog({
|
|
959
|
-
runId: this.runFriendlyId,
|
|
960
|
-
message: "waitForNextRun: warm starts disabled after connect",
|
|
961
|
-
properties: {
|
|
962
|
-
connectionTimeoutMs,
|
|
963
|
-
keepaliveMs,
|
|
964
|
-
},
|
|
965
|
-
});
|
|
966
|
-
this.exitProcess(this.successExitCode);
|
|
967
|
-
}
|
|
968
|
-
const nextRun = await this.warmStartClient.warmStart({
|
|
969
|
-
workerInstanceName: this.workerInstanceName,
|
|
970
|
-
connectionTimeoutMs,
|
|
971
|
-
keepaliveMs,
|
|
972
|
-
});
|
|
973
|
-
if (!nextRun) {
|
|
974
|
-
this.sendDebugLog({
|
|
975
|
-
runId: this.runFriendlyId,
|
|
976
|
-
message: "waitForNextRun: warm start failed, shutting down",
|
|
977
|
-
});
|
|
978
|
-
this.exitProcess(this.successExitCode);
|
|
979
|
-
}
|
|
980
|
-
this.sendDebugLog({
|
|
981
|
-
runId: this.runFriendlyId,
|
|
982
|
-
message: "waitForNextRun: got next run",
|
|
983
|
-
properties: { nextRun: nextRun.run.friendlyId },
|
|
984
|
-
});
|
|
985
|
-
this.startAndExecuteRunAttempt({
|
|
986
|
-
runFriendlyId: nextRun.run.friendlyId,
|
|
987
|
-
snapshotFriendlyId: nextRun.snapshot.friendlyId,
|
|
988
|
-
dequeuedAt: nextRun.dequeuedAt,
|
|
989
|
-
isWarmStart: true,
|
|
990
|
-
}).finally(() => { });
|
|
991
|
-
return;
|
|
992
|
-
}
|
|
993
|
-
catch (error) {
|
|
994
|
-
this.sendDebugLog({
|
|
995
|
-
runId: this.runFriendlyId,
|
|
996
|
-
message: "waitForNextRun: unexpected error",
|
|
997
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
998
|
-
});
|
|
999
|
-
this.exitProcess(this.failureExitCode);
|
|
1000
|
-
}
|
|
1001
|
-
finally {
|
|
1002
|
-
this.waitForNextRunLock = false;
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
exitProcess(code) {
|
|
1006
|
-
this.sendDebugLog({
|
|
1007
|
-
runId: this.runFriendlyId,
|
|
1008
|
-
message: "Exiting process",
|
|
1009
|
-
properties: { code },
|
|
1010
|
-
});
|
|
1011
|
-
if (this.taskRunProcess?.isPreparedForNextRun) {
|
|
1012
|
-
this.taskRunProcess.forceExit();
|
|
1013
|
-
}
|
|
1014
|
-
process.exit(code);
|
|
1015
|
-
}
|
|
1016
|
-
createSocket() {
|
|
1017
|
-
const wsUrl = new URL("/workload", this.workerApiUrl);
|
|
1018
|
-
this.socket = io(wsUrl.href, {
|
|
1019
|
-
transports: ["websocket"],
|
|
1020
|
-
extraHeaders: {
|
|
1021
|
-
[WORKLOAD_HEADERS.DEPLOYMENT_ID]: env.TRIGGER_DEPLOYMENT_ID,
|
|
1022
|
-
[WORKLOAD_HEADERS.RUNNER_ID]: env.TRIGGER_RUNNER_ID,
|
|
1023
|
-
},
|
|
1024
|
-
});
|
|
1025
|
-
this.socket.on("run:notify", async ({ version, run }) => {
|
|
1026
|
-
this.sendDebugLog({
|
|
1027
|
-
runId: run.friendlyId,
|
|
1028
|
-
message: "run:notify received by runner",
|
|
1029
|
-
properties: { version, runId: run.friendlyId },
|
|
1030
|
-
});
|
|
1031
|
-
if (!this.runFriendlyId) {
|
|
1032
|
-
this.sendDebugLog({
|
|
1033
|
-
runId: run.friendlyId,
|
|
1034
|
-
message: "run:notify: ignoring notification, no local run ID",
|
|
1035
|
-
properties: {
|
|
1036
|
-
currentRunId: this.runFriendlyId,
|
|
1037
|
-
currentSnapshotId: this.snapshotFriendlyId,
|
|
1038
|
-
},
|
|
1039
|
-
});
|
|
1040
|
-
return;
|
|
1041
|
-
}
|
|
1042
|
-
if (run.friendlyId !== this.runFriendlyId) {
|
|
1043
|
-
this.sendDebugLog({
|
|
1044
|
-
runId: run.friendlyId,
|
|
1045
|
-
message: "run:notify: ignoring notification for different run",
|
|
1046
|
-
properties: {
|
|
1047
|
-
currentRunId: this.runFriendlyId,
|
|
1048
|
-
currentSnapshotId: this.snapshotFriendlyId,
|
|
1049
|
-
notificationRunId: run.friendlyId,
|
|
1050
|
-
},
|
|
1051
|
-
});
|
|
1052
|
-
return;
|
|
1053
|
-
}
|
|
1054
|
-
// Reset the (fallback) snapshot poll interval so we don't do unnecessary work
|
|
1055
|
-
this.snapshotPoller.resetCurrentInterval();
|
|
1056
|
-
const latestSnapshot = await this.httpClient.getRunExecutionData(this.runFriendlyId);
|
|
1057
|
-
if (!latestSnapshot.success) {
|
|
1058
|
-
this.sendDebugLog({
|
|
1059
|
-
runId: this.runFriendlyId,
|
|
1060
|
-
message: "run:notify: failed to get latest snapshot data",
|
|
1061
|
-
properties: {
|
|
1062
|
-
currentRunId: this.runFriendlyId,
|
|
1063
|
-
currentSnapshotId: this.snapshotFriendlyId,
|
|
1064
|
-
error: latestSnapshot.error,
|
|
1065
|
-
},
|
|
1066
|
-
});
|
|
1067
|
-
return;
|
|
1068
|
-
}
|
|
1069
|
-
await this.handleSnapshotChange(latestSnapshot.data.execution);
|
|
1070
|
-
});
|
|
1071
|
-
this.socket.on("connect", () => {
|
|
1072
|
-
this.sendDebugLog({
|
|
1073
|
-
runId: this.runFriendlyId,
|
|
1074
|
-
message: "Connected to supervisor",
|
|
1075
|
-
});
|
|
1076
|
-
// This should handle the case where we reconnect after being restored
|
|
1077
|
-
if (this.state.phase === "RUN") {
|
|
1078
|
-
const { run, snapshot } = this.state;
|
|
1079
|
-
this.subscribeToRunNotifications({ run, snapshot });
|
|
1080
|
-
}
|
|
1081
|
-
});
|
|
1082
|
-
this.socket.on("connect_error", (error) => {
|
|
1083
|
-
this.sendDebugLog({
|
|
1084
|
-
runId: this.runFriendlyId,
|
|
1085
|
-
message: "Connection error",
|
|
1086
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
1087
|
-
});
|
|
1088
|
-
});
|
|
1089
|
-
this.socket.on("disconnect", (reason, description) => {
|
|
1090
|
-
this.sendDebugLog({
|
|
1091
|
-
runId: this.runFriendlyId,
|
|
1092
|
-
message: "Disconnected from supervisor",
|
|
1093
|
-
properties: { reason, description: description?.toString() },
|
|
1094
|
-
});
|
|
1095
|
-
});
|
|
1096
|
-
}
|
|
1097
|
-
async executeRun({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
|
|
1098
|
-
this.snapshotPoller.start();
|
|
1099
|
-
if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
|
|
1100
|
-
this.taskRunProcess = new TaskRunProcess({
|
|
1101
|
-
workerManifest: this.workerManifest,
|
|
1102
|
-
env: envVars,
|
|
1103
|
-
serverWorker: {
|
|
1104
|
-
id: "unmanaged",
|
|
1105
|
-
contentHash: env.TRIGGER_CONTENT_HASH,
|
|
1106
|
-
version: env.TRIGGER_DEPLOYMENT_VERSION,
|
|
1107
|
-
engine: "V2",
|
|
1108
|
-
},
|
|
1109
|
-
machine: execution.machine,
|
|
1110
|
-
isWarmStart,
|
|
1111
|
-
}).initialize();
|
|
1112
|
-
}
|
|
1113
|
-
this.sendDebugLog({
|
|
1114
|
-
runId: this.runFriendlyId,
|
|
1115
|
-
message: "executing task run process",
|
|
1116
|
-
properties: {
|
|
1117
|
-
attemptId: execution.attempt.id,
|
|
1118
|
-
runId: execution.run.id,
|
|
1119
|
-
},
|
|
1120
|
-
});
|
|
1121
|
-
const completion = await this.taskRunProcess.execute({
|
|
1122
|
-
payload: {
|
|
1123
|
-
execution,
|
|
1124
|
-
traceContext: execution.run.traceContext ?? {},
|
|
1125
|
-
metrics,
|
|
1126
|
-
},
|
|
1127
|
-
messageId: run.friendlyId,
|
|
1128
|
-
env: envVars,
|
|
1129
|
-
}, isWarmStart);
|
|
1130
|
-
this.sendDebugLog({
|
|
1131
|
-
runId: this.runFriendlyId,
|
|
1132
|
-
message: "Completed run",
|
|
1133
|
-
properties: { completion: completion.ok },
|
|
1134
|
-
});
|
|
1135
|
-
try {
|
|
1136
|
-
// The execution has finished, so we can cleanup the task run process. Killing it should be safe.
|
|
1137
|
-
await this.taskRunProcess.cleanup(true);
|
|
1138
|
-
}
|
|
1139
|
-
catch (error) {
|
|
1140
|
-
this.sendDebugLog({
|
|
1141
|
-
runId: this.runFriendlyId,
|
|
1142
|
-
message: "Failed to cleanup task run process, submitting completion anyway",
|
|
1143
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
1144
|
-
});
|
|
1145
|
-
}
|
|
1146
|
-
if (!this.runFriendlyId || !this.snapshotFriendlyId) {
|
|
1147
|
-
this.sendDebugLog({
|
|
1148
|
-
runId: this.runFriendlyId,
|
|
1149
|
-
message: "executeRun: Missing run ID or snapshot ID after execution",
|
|
1150
|
-
properties: {
|
|
1151
|
-
runId: this.runFriendlyId,
|
|
1152
|
-
snapshotId: this.snapshotFriendlyId,
|
|
1153
|
-
},
|
|
1154
|
-
});
|
|
1155
|
-
this.waitForNextRun();
|
|
1156
|
-
return;
|
|
1157
|
-
}
|
|
1158
|
-
const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.snapshotFriendlyId, {
|
|
1159
|
-
completion,
|
|
1160
|
-
});
|
|
1161
|
-
if (!completionResult.success) {
|
|
1162
|
-
this.sendDebugLog({
|
|
1163
|
-
runId: run.friendlyId,
|
|
1164
|
-
message: "completion: failed to submit",
|
|
1165
|
-
properties: {
|
|
1166
|
-
error: completionResult.error,
|
|
1167
|
-
},
|
|
1168
|
-
});
|
|
1169
|
-
this.sendDebugLog({
|
|
1170
|
-
runId: run.friendlyId,
|
|
1171
|
-
message: "completion: failed to submit",
|
|
1172
|
-
properties: {
|
|
1173
|
-
error: completionResult.error,
|
|
1174
|
-
},
|
|
1175
|
-
});
|
|
1176
|
-
this.waitForNextRun();
|
|
1177
|
-
return;
|
|
1178
|
-
}
|
|
1179
|
-
this.sendDebugLog({
|
|
1180
|
-
runId: run.friendlyId,
|
|
1181
|
-
message: "Attempt completion submitted",
|
|
1182
|
-
properties: {
|
|
1183
|
-
attemptStatus: completionResult.data.result.attemptStatus,
|
|
1184
|
-
runId: completionResult.data.result.run.friendlyId,
|
|
1185
|
-
snapshotId: completionResult.data.result.snapshot.friendlyId,
|
|
1186
|
-
},
|
|
1187
|
-
});
|
|
1188
|
-
try {
|
|
1189
|
-
await this.handleCompletionResult(completion, completionResult.data.result);
|
|
1190
|
-
}
|
|
1191
|
-
catch (error) {
|
|
1192
|
-
this.sendDebugLog({
|
|
1193
|
-
runId: run.friendlyId,
|
|
1194
|
-
message: "Failed to handle completion result",
|
|
1195
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
1196
|
-
});
|
|
1197
|
-
this.waitForNextRun();
|
|
1198
|
-
return;
|
|
1199
|
-
}
|
|
1200
|
-
}
|
|
1201
|
-
async handleCompletionResult(completion, result) {
|
|
1202
|
-
this.sendDebugLog({
|
|
1203
|
-
runId: this.runFriendlyId,
|
|
1204
|
-
message: "Handling completion result",
|
|
1205
|
-
properties: {
|
|
1206
|
-
completion: completion.ok,
|
|
1207
|
-
attemptStatus: result.attemptStatus,
|
|
1208
|
-
snapshotId: result.snapshot.friendlyId,
|
|
1209
|
-
runId: result.run.friendlyId,
|
|
1210
|
-
},
|
|
1211
|
-
});
|
|
1212
|
-
const { attemptStatus, snapshot: completionSnapshot, run } = result;
|
|
1213
|
-
try {
|
|
1214
|
-
this.updateRunPhase(run, completionSnapshot);
|
|
1215
|
-
}
|
|
1216
|
-
catch (error) {
|
|
1217
|
-
this.sendDebugLog({
|
|
1218
|
-
runId: run.friendlyId,
|
|
1219
|
-
message: "Failed to update run phase after completion",
|
|
1220
|
-
properties: { error: error instanceof Error ? error.message : String(error) },
|
|
1221
|
-
});
|
|
1222
|
-
this.waitForNextRun();
|
|
1223
|
-
return;
|
|
1224
|
-
}
|
|
1225
|
-
if (attemptStatus === "RUN_FINISHED") {
|
|
1226
|
-
this.sendDebugLog({
|
|
1227
|
-
runId: run.friendlyId,
|
|
1228
|
-
message: "Run finished",
|
|
1229
|
-
});
|
|
1230
|
-
this.waitForNextRun();
|
|
1231
|
-
return;
|
|
1232
|
-
}
|
|
1233
|
-
if (attemptStatus === "RUN_PENDING_CANCEL") {
|
|
1234
|
-
this.sendDebugLog({
|
|
1235
|
-
runId: run.friendlyId,
|
|
1236
|
-
message: "Run pending cancel",
|
|
1237
|
-
});
|
|
1238
|
-
return;
|
|
1239
|
-
}
|
|
1240
|
-
if (attemptStatus === "RETRY_QUEUED") {
|
|
1241
|
-
this.sendDebugLog({
|
|
1242
|
-
runId: run.friendlyId,
|
|
1243
|
-
message: "Retry queued",
|
|
1244
|
-
});
|
|
1245
|
-
this.waitForNextRun();
|
|
1246
|
-
return;
|
|
1247
|
-
}
|
|
1248
|
-
if (attemptStatus === "RETRY_IMMEDIATELY") {
|
|
1249
|
-
if (completion.ok) {
|
|
1250
|
-
throw new Error("Should retry but completion OK.");
|
|
1251
|
-
}
|
|
1252
|
-
if (!completion.retry) {
|
|
1253
|
-
throw new Error("Should retry but missing retry params.");
|
|
1254
|
-
}
|
|
1255
|
-
await sleep(completion.retry.delay);
|
|
1256
|
-
if (!this.snapshotFriendlyId) {
|
|
1257
|
-
throw new Error("Missing snapshot ID after retry");
|
|
1258
|
-
}
|
|
1259
|
-
this.startAndExecuteRunAttempt({
|
|
1260
|
-
runFriendlyId: run.friendlyId,
|
|
1261
|
-
snapshotFriendlyId: this.snapshotFriendlyId,
|
|
1262
|
-
skipLockCheckForImmediateRetry: true,
|
|
1263
|
-
isWarmStart: true,
|
|
1264
|
-
}).finally(() => { });
|
|
1265
|
-
return;
|
|
1266
|
-
}
|
|
1267
|
-
assertExhaustive(attemptStatus);
|
|
1268
|
-
}
|
|
1269
|
-
sendDebugLog({ runId, message, date, properties, }) {
|
|
1270
|
-
if (!runId) {
|
|
1271
|
-
runId = this.runFriendlyId;
|
|
1272
|
-
}
|
|
1273
|
-
if (!runId) {
|
|
1274
|
-
runId = env.TRIGGER_RUN_ID;
|
|
1275
|
-
}
|
|
1276
|
-
if (!runId) {
|
|
1277
|
-
return;
|
|
1278
|
-
}
|
|
1279
|
-
const mergedProperties = {
|
|
1280
|
-
...properties,
|
|
1281
|
-
runId,
|
|
1282
|
-
runnerId: this.runnerId,
|
|
1283
|
-
workerName: this.workerInstanceName,
|
|
1284
|
-
};
|
|
1285
|
-
console.log(message, mergedProperties);
|
|
1286
|
-
this.httpClient.sendDebugLog(runId, {
|
|
1287
|
-
message,
|
|
1288
|
-
time: date ?? new Date(),
|
|
1289
|
-
properties: mergedProperties,
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
async cancelAttempt(runId) {
|
|
1293
|
-
this.sendDebugLog({
|
|
1294
|
-
runId,
|
|
1295
|
-
message: "cancelling attempt",
|
|
1296
|
-
properties: { runId },
|
|
1297
|
-
});
|
|
1298
|
-
await this.taskRunProcess?.cancel();
|
|
1299
|
-
}
|
|
1300
|
-
async start() {
|
|
1301
|
-
this.sendDebugLog({
|
|
1302
|
-
runId: this.runFriendlyId,
|
|
1303
|
-
message: "Starting up",
|
|
1304
|
-
});
|
|
1305
|
-
// Websocket notifications are only an optimisation so we don't need to wait for a successful connection
|
|
1306
|
-
this.createSocket();
|
|
1307
|
-
// If we have run and snapshot IDs, we can start an attempt immediately
|
|
1308
|
-
if (env.TRIGGER_RUN_ID && env.TRIGGER_SNAPSHOT_ID) {
|
|
1309
|
-
this.startAndExecuteRunAttempt({
|
|
1310
|
-
runFriendlyId: env.TRIGGER_RUN_ID,
|
|
1311
|
-
snapshotFriendlyId: env.TRIGGER_SNAPSHOT_ID,
|
|
1312
|
-
dequeuedAt: env.TRIGGER_DEQUEUED_AT_MS,
|
|
1313
|
-
podScheduledAt: env.TRIGGER_POD_SCHEDULED_AT_MS,
|
|
1314
|
-
}).finally(() => { });
|
|
1315
|
-
return;
|
|
1316
|
-
}
|
|
1317
|
-
// ..otherwise we need to wait for a run
|
|
1318
|
-
this.waitForNextRun();
|
|
1319
|
-
return;
|
|
1320
|
-
}
|
|
1321
|
-
async stop() {
|
|
1322
|
-
this.sendDebugLog({
|
|
1323
|
-
runId: this.runFriendlyId,
|
|
1324
|
-
message: "Shutting down",
|
|
1325
|
-
});
|
|
1326
|
-
if (this.taskRunProcess) {
|
|
1327
|
-
await this.taskRunProcess.cleanup(true);
|
|
1328
|
-
}
|
|
1329
|
-
this.runHeartbeat.stop();
|
|
1330
|
-
this.snapshotPoller.stop();
|
|
1331
|
-
this.socket.close();
|
|
1332
|
-
}
|
|
1333
|
-
}
|
|
1334
|
-
const workerManifest = await loadWorkerManifest();
|
|
1335
|
-
const prodWorker = new ManagedRunController({ workerManifest });
|
|
1336
|
-
await prodWorker.start();
|
|
1337
|
-
function gatherProcessEnv() {
|
|
1338
|
-
const $env = {
|
|
1339
|
-
NODE_ENV: env.NODE_ENV,
|
|
1340
|
-
NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS,
|
|
1341
|
-
OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT,
|
|
1342
|
-
};
|
|
1343
|
-
// Filter out undefined values
|
|
1344
|
-
return Object.fromEntries(Object.entries($env).filter(([key, value]) => value !== undefined));
|
|
1345
|
-
}
|
|
1346
|
-
async function loadWorkerManifest() {
|
|
1347
|
-
const manifest = await readJSONFile("./index.json");
|
|
1348
|
-
return WorkerManifest.parse(manifest);
|
|
1349
|
-
}
|
|
3
|
+
import { WorkerManifest } from "@trigger.dev/core/v3";
|
|
4
|
+
import { ManagedRunController } from "./managed/controller.js";
|
|
5
|
+
const manifest = await readJSONFile("./index.json");
|
|
6
|
+
const workerManifest = WorkerManifest.parse(manifest);
|
|
7
|
+
new ManagedRunController({
|
|
8
|
+
workerManifest,
|
|
9
|
+
env: stdEnv,
|
|
10
|
+
}).start();
|
|
1350
11
|
//# sourceMappingURL=managed-run-controller.js.map
|