trigger.dev 4.0.0-v4-beta.1 → 4.0.0-v4-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/esm/dev/devSupervisor.js +14 -10
  2. package/dist/esm/dev/devSupervisor.js.map +1 -1
  3. package/dist/esm/entryPoints/dev-run-controller.js +6 -6
  4. package/dist/esm/entryPoints/dev-run-controller.js.map +1 -1
  5. package/dist/esm/entryPoints/managed/controller.d.ts +50 -0
  6. package/dist/esm/entryPoints/managed/controller.js +441 -0
  7. package/dist/esm/entryPoints/managed/controller.js.map +1 -0
  8. package/dist/esm/entryPoints/managed/env.d.ts +170 -0
  9. package/dist/esm/entryPoints/managed/env.js +191 -0
  10. package/dist/esm/entryPoints/managed/env.js.map +1 -0
  11. package/dist/esm/entryPoints/managed/execution.d.ts +95 -0
  12. package/dist/esm/entryPoints/managed/execution.js +661 -0
  13. package/dist/esm/entryPoints/managed/execution.js.map +1 -0
  14. package/dist/esm/entryPoints/managed/logger.d.ts +19 -0
  15. package/dist/esm/entryPoints/managed/logger.js +31 -0
  16. package/dist/esm/entryPoints/managed/logger.js.map +1 -0
  17. package/dist/esm/entryPoints/managed/overrides.d.ts +16 -0
  18. package/dist/esm/entryPoints/managed/overrides.js +17 -0
  19. package/dist/esm/entryPoints/managed/overrides.js.map +1 -0
  20. package/dist/esm/entryPoints/managed/poller.d.ts +26 -0
  21. package/dist/esm/entryPoints/managed/poller.js +94 -0
  22. package/dist/esm/entryPoints/managed/poller.js.map +1 -0
  23. package/dist/esm/entryPoints/managed-run-controller.js +9 -1346
  24. package/dist/esm/entryPoints/managed-run-controller.js.map +1 -1
  25. package/dist/esm/entryPoints/managed-run-worker.js +17 -13
  26. package/dist/esm/entryPoints/managed-run-worker.js.map +1 -1
  27. package/dist/esm/executions/taskRunProcess.d.ts +2 -2
  28. package/dist/esm/executions/taskRunProcess.js +1 -1
  29. package/dist/esm/executions/taskRunProcess.js.map +1 -1
  30. package/dist/esm/version.js +1 -1
  31. package/package.json +3 -3
@@ -1,1350 +1,13 @@
1
- import { logger } from "../utilities/logger.js";
2
- import { TaskRunProcess } from "../executions/taskRunProcess.js";
3
1
  import { env as stdEnv } from "std-env";
4
- import { z } from "zod";
5
- import { randomUUID } from "crypto";
6
2
  import { readJSONFile } from "../utilities/fileSystem.js";
7
- import { HeartbeatService, SuspendedProcessError, WorkerManifest, } from "@trigger.dev/core/v3";
8
- import { WarmStartClient, WORKLOAD_HEADERS, WorkloadHttpClient, } from "@trigger.dev/core/v3/workers";
9
- import { assertExhaustive } from "../utilities/assertExhaustive.js";
10
- import { setTimeout as sleep } from "timers/promises";
11
- import { io } from "socket.io-client";
12
- const DateEnv = z
13
- .string()
14
- .transform((val) => new Date(parseInt(val, 10)))
15
- .pipe(z.date());
16
- // All IDs are friendly IDs
17
- const Env = z.object({
18
- // Set at build time
19
- TRIGGER_CONTENT_HASH: z.string(),
20
- TRIGGER_DEPLOYMENT_ID: z.string(),
21
- TRIGGER_DEPLOYMENT_VERSION: z.string(),
22
- TRIGGER_PROJECT_ID: z.string(),
23
- TRIGGER_PROJECT_REF: z.string(),
24
- NODE_ENV: z.string().default("production"),
25
- NODE_EXTRA_CA_CERTS: z.string().optional(),
26
- // Set at runtime
27
- TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`),
28
- TRIGGER_ENV_ID: z.string(),
29
- TRIGGER_RUN_ID: z.string().optional(), // This is only useful for cold starts
30
- TRIGGER_SNAPSHOT_ID: z.string().optional(), // This is only useful for cold starts
31
- OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
32
- TRIGGER_WARM_START_URL: z.string().optional(),
33
- TRIGGER_WARM_START_CONNECTION_TIMEOUT_MS: z.coerce.number().default(30_000),
34
- TRIGGER_WARM_START_KEEPALIVE_MS: z.coerce.number().default(300_000),
35
- TRIGGER_MACHINE_CPU: z.string().default("0"),
36
- TRIGGER_MACHINE_MEMORY: z.string().default("0"),
37
- TRIGGER_RUNNER_ID: z.string(),
38
- TRIGGER_METADATA_URL: z.string().optional(),
39
- TRIGGER_PRE_SUSPEND_WAIT_MS: z.coerce.number().default(200),
40
- // Timeline metrics
41
- TRIGGER_POD_SCHEDULED_AT_MS: DateEnv,
42
- TRIGGER_DEQUEUED_AT_MS: DateEnv,
43
- // May be overridden
44
- TRIGGER_SUPERVISOR_API_PROTOCOL: z.enum(["http", "https"]),
45
- TRIGGER_SUPERVISOR_API_DOMAIN: z.string(),
46
- TRIGGER_SUPERVISOR_API_PORT: z.coerce.number(),
47
- TRIGGER_WORKER_INSTANCE_NAME: z.string(),
48
- TRIGGER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
49
- TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().default(5),
50
- TRIGGER_SUCCESS_EXIT_CODE: z.coerce.number().default(0),
51
- TRIGGER_FAILURE_EXIT_CODE: z.coerce.number().default(1),
52
- });
53
- const env = Env.parse(stdEnv);
3
+ import { WorkerManifest } from "@trigger.dev/core/v3";
4
+ import { ManagedRunController } from "./managed/controller.js";
5
+ import { logger } from "../utilities/logger.js";
54
6
  logger.loggerLevel = "debug";
55
- class MetadataClient {
56
- url;
57
- constructor(url) {
58
- this.url = new URL(url);
59
- }
60
- async getEnvOverrides() {
61
- try {
62
- const response = await fetch(new URL("/env", this.url));
63
- return response.json();
64
- }
65
- catch (error) {
66
- console.error("Failed to fetch metadata", { error });
67
- return null;
68
- }
69
- }
70
- }
71
- class ManagedRunController {
72
- taskRunProcess;
73
- workerManifest;
74
- httpClient;
75
- warmStartClient;
76
- metadataClient;
77
- socket;
78
- runHeartbeat;
79
- heartbeatIntervalSeconds;
80
- snapshotPoller;
81
- snapshotPollIntervalSeconds;
82
- workerApiUrl;
83
- workerInstanceName;
84
- runnerId;
85
- successExitCode = env.TRIGGER_SUCCESS_EXIT_CODE;
86
- failureExitCode = env.TRIGGER_FAILURE_EXIT_CODE;
87
- state = { phase: "IDLE" };
88
- constructor(opts) {
89
- this.workerManifest = opts.workerManifest;
90
- this.runnerId = env.TRIGGER_RUNNER_ID;
91
- this.workerApiUrl = `${env.TRIGGER_SUPERVISOR_API_PROTOCOL}://${env.TRIGGER_SUPERVISOR_API_DOMAIN}:${env.TRIGGER_SUPERVISOR_API_PORT}`;
92
- this.workerInstanceName = env.TRIGGER_WORKER_INSTANCE_NAME;
93
- this.httpClient = new WorkloadHttpClient({
94
- workerApiUrl: this.workerApiUrl,
95
- runnerId: this.runnerId,
96
- deploymentId: env.TRIGGER_DEPLOYMENT_ID,
97
- deploymentVersion: env.TRIGGER_DEPLOYMENT_VERSION,
98
- projectRef: env.TRIGGER_PROJECT_REF,
99
- });
100
- const properties = {
101
- ...env,
102
- TRIGGER_POD_SCHEDULED_AT_MS: env.TRIGGER_POD_SCHEDULED_AT_MS.toISOString(),
103
- TRIGGER_DEQUEUED_AT_MS: env.TRIGGER_DEQUEUED_AT_MS.toISOString(),
104
- };
105
- this.sendDebugLog({
106
- runId: env.TRIGGER_RUN_ID,
107
- message: "Creating run controller",
108
- properties,
109
- });
110
- this.heartbeatIntervalSeconds = env.TRIGGER_HEARTBEAT_INTERVAL_SECONDS;
111
- this.snapshotPollIntervalSeconds = env.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS;
112
- if (env.TRIGGER_METADATA_URL) {
113
- this.metadataClient = new MetadataClient(env.TRIGGER_METADATA_URL);
114
- }
115
- if (env.TRIGGER_WARM_START_URL) {
116
- this.warmStartClient = new WarmStartClient({
117
- apiUrl: new URL(env.TRIGGER_WARM_START_URL),
118
- controllerId: env.TRIGGER_WORKLOAD_CONTROLLER_ID,
119
- deploymentId: env.TRIGGER_DEPLOYMENT_ID,
120
- deploymentVersion: env.TRIGGER_DEPLOYMENT_VERSION,
121
- machineCpu: env.TRIGGER_MACHINE_CPU,
122
- machineMemory: env.TRIGGER_MACHINE_MEMORY,
123
- });
124
- }
125
- this.snapshotPoller = new HeartbeatService({
126
- heartbeat: async () => {
127
- if (!this.runFriendlyId) {
128
- this.sendDebugLog({
129
- runId: env.TRIGGER_RUN_ID,
130
- message: "Skipping snapshot poll, no run ID",
131
- });
132
- return;
133
- }
134
- this.sendDebugLog({
135
- runId: env.TRIGGER_RUN_ID,
136
- message: "Polling for latest snapshot",
137
- });
138
- this.sendDebugLog({
139
- runId: this.runFriendlyId,
140
- message: `snapshot poll: started`,
141
- properties: {
142
- snapshotId: this.snapshotFriendlyId,
143
- },
144
- });
145
- const response = await this.httpClient.getRunExecutionData(this.runFriendlyId);
146
- if (!response.success) {
147
- this.sendDebugLog({
148
- runId: this.runFriendlyId,
149
- message: "Snapshot poll failed",
150
- properties: {
151
- error: response.error,
152
- },
153
- });
154
- this.sendDebugLog({
155
- runId: this.runFriendlyId,
156
- message: `snapshot poll: failed`,
157
- properties: {
158
- snapshotId: this.snapshotFriendlyId,
159
- error: response.error,
160
- },
161
- });
162
- return;
163
- }
164
- await this.handleSnapshotChange(response.data.execution);
165
- },
166
- intervalMs: this.snapshotPollIntervalSeconds * 1000,
167
- leadingEdge: false,
168
- onError: async (error) => {
169
- this.sendDebugLog({
170
- runId: this.runFriendlyId,
171
- message: "Failed to poll for snapshot",
172
- properties: { error: error instanceof Error ? error.message : String(error) },
173
- });
174
- },
175
- });
176
- this.runHeartbeat = new HeartbeatService({
177
- heartbeat: async () => {
178
- if (!this.runFriendlyId || !this.snapshotFriendlyId) {
179
- this.sendDebugLog({
180
- runId: this.runFriendlyId,
181
- message: "Skipping heartbeat, no run ID or snapshot ID",
182
- });
183
- return;
184
- }
185
- this.sendDebugLog({
186
- runId: this.runFriendlyId,
187
- message: "heartbeat: started",
188
- });
189
- const response = await this.httpClient.heartbeatRun(this.runFriendlyId, this.snapshotFriendlyId);
190
- if (!response.success) {
191
- this.sendDebugLog({
192
- runId: this.runFriendlyId,
193
- message: "heartbeat: failed",
194
- properties: {
195
- error: response.error,
196
- },
197
- });
198
- }
199
- },
200
- intervalMs: this.heartbeatIntervalSeconds * 1000,
201
- leadingEdge: false,
202
- onError: async (error) => {
203
- this.sendDebugLog({
204
- runId: this.runFriendlyId,
205
- message: "Failed to send heartbeat",
206
- properties: { error: error instanceof Error ? error.message : String(error) },
207
- });
208
- },
209
- });
210
- process.on("SIGTERM", async () => {
211
- this.sendDebugLog({
212
- runId: this.runFriendlyId,
213
- message: "Received SIGTERM, stopping worker",
214
- });
215
- await this.stop();
216
- });
217
- }
218
- enterRunPhase(run, snapshot) {
219
- this.onExitRunPhase(run);
220
- this.state = { phase: "RUN", run, snapshot };
221
- this.runHeartbeat.start();
222
- this.snapshotPoller.start();
223
- }
224
- enterWarmStartPhase() {
225
- this.onExitRunPhase();
226
- this.state = { phase: "WARM_START" };
227
- }
228
- // This should only be used when we're already executing a run. Attempt number changes are not allowed.
229
- updateRunPhase(run, snapshot) {
230
- if (this.state.phase !== "RUN") {
231
- this.sendDebugLog({
232
- runId: run.friendlyId,
233
- message: `updateRunPhase: Invalid phase for updating snapshot: ${this.state.phase}`,
234
- properties: {
235
- currentPhase: this.state.phase,
236
- snapshotId: snapshot.friendlyId,
237
- },
238
- });
239
- throw new Error(`Invalid phase for updating snapshot: ${this.state.phase}`);
240
- }
241
- if (this.state.run.friendlyId !== run.friendlyId) {
242
- this.sendDebugLog({
243
- runId: run.friendlyId,
244
- message: `updateRunPhase: Mismatched run IDs`,
245
- properties: {
246
- currentRunId: this.state.run.friendlyId,
247
- newRunId: run.friendlyId,
248
- currentSnapshotId: this.state.snapshot.friendlyId,
249
- newSnapshotId: snapshot.friendlyId,
250
- },
251
- });
252
- throw new Error("Mismatched run IDs");
253
- }
254
- if (this.state.snapshot.friendlyId === snapshot.friendlyId) {
255
- this.sendDebugLog({
256
- runId: run.friendlyId,
257
- message: "updateRunPhase: Snapshot not changed",
258
- properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
259
- });
260
- this.sendDebugLog({
261
- runId: run.friendlyId,
262
- message: `updateRunPhase: Snapshot not changed`,
263
- properties: {
264
- snapshotId: snapshot.friendlyId,
265
- },
266
- });
267
- return;
268
- }
269
- if (this.state.run.attemptNumber !== run.attemptNumber) {
270
- this.sendDebugLog({
271
- runId: run.friendlyId,
272
- message: `updateRunPhase: Attempt number changed`,
273
- properties: {
274
- oldAttemptNumber: this.state.run.attemptNumber ?? undefined,
275
- newAttemptNumber: run.attemptNumber ?? undefined,
276
- },
277
- });
278
- throw new Error("Attempt number changed");
279
- }
280
- this.state = {
281
- phase: "RUN",
282
- run: {
283
- friendlyId: run.friendlyId,
284
- attemptNumber: run.attemptNumber,
285
- },
286
- snapshot: {
287
- friendlyId: snapshot.friendlyId,
288
- },
289
- };
290
- }
291
- onExitRunPhase(newRun = undefined) {
292
- // We're not in a run phase, nothing to do
293
- if (this.state.phase !== "RUN") {
294
- this.sendDebugLog({
295
- runId: this.runFriendlyId,
296
- message: "onExitRunPhase: Not in run phase, skipping",
297
- properties: { phase: this.state.phase },
298
- });
299
- return;
300
- }
301
- // This is still the same run, so we're not exiting the phase
302
- if (newRun?.friendlyId === this.state.run.friendlyId) {
303
- this.sendDebugLog({
304
- runId: this.runFriendlyId,
305
- message: "onExitRunPhase: Same run, skipping",
306
- properties: { newRun: newRun?.friendlyId },
307
- });
308
- return;
309
- }
310
- this.sendDebugLog({
311
- runId: this.runFriendlyId,
312
- message: "onExitRunPhase: Exiting run phase",
313
- properties: { newRun: newRun?.friendlyId },
314
- });
315
- this.runHeartbeat.stop();
316
- this.snapshotPoller.stop();
317
- const { run, snapshot } = this.state;
318
- this.unsubscribeFromRunNotifications({ run, snapshot });
319
- }
320
- subscribeToRunNotifications({ run, snapshot }) {
321
- this.socket.emit("run:start", {
322
- version: "1",
323
- run: {
324
- friendlyId: run.friendlyId,
325
- },
326
- snapshot: {
327
- friendlyId: snapshot.friendlyId,
328
- },
329
- });
330
- }
331
- unsubscribeFromRunNotifications({ run, snapshot }) {
332
- this.socket.emit("run:stop", {
333
- version: "1",
334
- run: {
335
- friendlyId: run.friendlyId,
336
- },
337
- snapshot: {
338
- friendlyId: snapshot.friendlyId,
339
- },
340
- });
341
- }
342
- get runFriendlyId() {
343
- if (this.state.phase !== "RUN") {
344
- return undefined;
345
- }
346
- return this.state.run.friendlyId;
347
- }
348
- get snapshotFriendlyId() {
349
- if (this.state.phase !== "RUN") {
350
- return;
351
- }
352
- return this.state.snapshot.friendlyId;
353
- }
354
- handleSnapshotChangeLock = false;
355
- async handleSnapshotChange({ run, snapshot, completedWaitpoints, }) {
356
- if (this.handleSnapshotChangeLock) {
357
- this.sendDebugLog({
358
- runId: run.friendlyId,
359
- message: "handleSnapshotChange: already in progress",
360
- });
361
- return;
362
- }
363
- this.handleSnapshotChangeLock = true;
364
- try {
365
- if (!this.snapshotFriendlyId) {
366
- this.sendDebugLog({
367
- runId: run.friendlyId,
368
- message: "handleSnapshotChange: Missing snapshot ID",
369
- properties: {
370
- newSnapshotId: snapshot.friendlyId,
371
- newSnapshotStatus: snapshot.executionStatus,
372
- },
373
- });
374
- this.sendDebugLog({
375
- runId: run.friendlyId,
376
- message: "snapshot change: missing snapshot ID",
377
- properties: {
378
- newSnapshotId: snapshot.friendlyId,
379
- newSnapshotStatus: snapshot.executionStatus,
380
- },
381
- });
382
- return;
383
- }
384
- if (this.snapshotFriendlyId === snapshot.friendlyId) {
385
- this.sendDebugLog({
386
- runId: run.friendlyId,
387
- message: "handleSnapshotChange: snapshot not changed, skipping",
388
- properties: { snapshot: snapshot.friendlyId },
389
- });
390
- this.sendDebugLog({
391
- runId: run.friendlyId,
392
- message: "snapshot change: skipping, no change",
393
- properties: {
394
- snapshotId: this.snapshotFriendlyId,
395
- snapshotStatus: snapshot.executionStatus,
396
- },
397
- });
398
- return;
399
- }
400
- this.sendDebugLog({
401
- runId: run.friendlyId,
402
- message: `snapshot change: ${snapshot.executionStatus}`,
403
- properties: {
404
- oldSnapshotId: this.snapshotFriendlyId,
405
- newSnapshotId: snapshot.friendlyId,
406
- completedWaitpoints: completedWaitpoints.length,
407
- },
408
- });
409
- try {
410
- this.updateRunPhase(run, snapshot);
411
- }
412
- catch (error) {
413
- this.sendDebugLog({
414
- runId: run.friendlyId,
415
- message: "snapshot change: failed to update run phase",
416
- properties: {
417
- currentPhase: this.state.phase,
418
- error: error instanceof Error ? error.message : String(error),
419
- },
420
- });
421
- this.waitForNextRun();
422
- return;
423
- }
424
- switch (snapshot.executionStatus) {
425
- case "PENDING_CANCEL": {
426
- try {
427
- await this.cancelAttempt(run.friendlyId);
428
- }
429
- catch (error) {
430
- this.sendDebugLog({
431
- runId: run.friendlyId,
432
- message: "snapshot change: failed to cancel attempt",
433
- properties: {
434
- error: error instanceof Error ? error.message : String(error),
435
- },
436
- });
437
- this.waitForNextRun();
438
- return;
439
- }
440
- return;
441
- }
442
- case "FINISHED": {
443
- this.sendDebugLog({
444
- runId: run.friendlyId,
445
- message: "Run is finished, will wait for next run",
446
- });
447
- if (this.activeRunExecution) {
448
- // Let's pretend we've just suspended the run. This will kill the process and should automatically wait for the next run.
449
- // We still explicitly call waitForNextRun() afterwards in case of race conditions. Locks will prevent this from causing issues.
450
- await this.taskRunProcess?.suspend();
451
- }
452
- this.waitForNextRun();
453
- return;
454
- }
455
- case "QUEUED_EXECUTING":
456
- case "EXECUTING_WITH_WAITPOINTS": {
457
- this.sendDebugLog({
458
- runId: run.friendlyId,
459
- message: "Run is executing with waitpoints",
460
- properties: { snapshot: snapshot.friendlyId },
461
- });
462
- try {
463
- // This should never throw. It should also never fail the run.
464
- await this.taskRunProcess?.cleanup(false);
465
- }
466
- catch (error) {
467
- this.sendDebugLog({
468
- runId: run.friendlyId,
469
- message: "Failed to cleanup task run process",
470
- properties: { error: error instanceof Error ? error.message : String(error) },
471
- });
472
- }
473
- if (snapshot.friendlyId !== this.snapshotFriendlyId) {
474
- this.sendDebugLog({
475
- runId: run.friendlyId,
476
- message: "Snapshot changed after cleanup, abort",
477
- properties: {
478
- oldSnapshotId: snapshot.friendlyId,
479
- newSnapshotId: this.snapshotFriendlyId,
480
- },
481
- });
482
- return;
483
- }
484
- await sleep(env.TRIGGER_PRE_SUSPEND_WAIT_MS);
485
- if (snapshot.friendlyId !== this.snapshotFriendlyId) {
486
- this.sendDebugLog({
487
- runId: run.friendlyId,
488
- message: "Snapshot changed after suspend threshold, abort",
489
- properties: {
490
- oldSnapshotId: snapshot.friendlyId,
491
- newSnapshotId: this.snapshotFriendlyId,
492
- },
493
- });
494
- return;
495
- }
496
- if (!this.runFriendlyId || !this.snapshotFriendlyId) {
497
- this.sendDebugLog({
498
- runId: run.friendlyId,
499
- message: "handleSnapshotChange: Missing run ID or snapshot ID after suspension, abort",
500
- properties: {
501
- runId: this.runFriendlyId,
502
- snapshotId: this.snapshotFriendlyId,
503
- },
504
- });
505
- return;
506
- }
507
- const suspendResult = await this.httpClient.suspendRun(this.runFriendlyId, this.snapshotFriendlyId);
508
- if (!suspendResult.success) {
509
- this.sendDebugLog({
510
- runId: run.friendlyId,
511
- message: "Failed to suspend run, staying alive 🎶",
512
- properties: {
513
- error: suspendResult.error,
514
- },
515
- });
516
- this.sendDebugLog({
517
- runId: run.friendlyId,
518
- message: "checkpoint: suspend request failed",
519
- properties: {
520
- snapshotId: snapshot.friendlyId,
521
- error: suspendResult.error,
522
- },
523
- });
524
- return;
525
- }
526
- if (!suspendResult.data.ok) {
527
- this.sendDebugLog({
528
- runId: run.friendlyId,
529
- message: "checkpoint: failed to suspend run",
530
- properties: {
531
- snapshotId: snapshot.friendlyId,
532
- error: suspendResult.data.error,
533
- },
534
- });
535
- return;
536
- }
537
- this.sendDebugLog({
538
- runId: run.friendlyId,
539
- message: "Suspending, any day now 🚬",
540
- properties: { ok: suspendResult.data.ok },
541
- });
542
- return;
543
- }
544
- case "SUSPENDED": {
545
- this.sendDebugLog({
546
- runId: run.friendlyId,
547
- message: "Run was suspended, kill the process and wait for more runs",
548
- properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
549
- });
550
- // This will kill the process and fail the execution with a SuspendedProcessError
551
- await this.taskRunProcess?.suspend();
552
- return;
553
- }
554
- case "PENDING_EXECUTING": {
555
- this.sendDebugLog({
556
- runId: run.friendlyId,
557
- message: "Run is pending execution",
558
- properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
559
- });
560
- if (completedWaitpoints.length === 0) {
561
- this.sendDebugLog({
562
- runId: run.friendlyId,
563
- message: "No waitpoints to complete, nothing to do",
564
- });
565
- return;
566
- }
567
- // There are waitpoints to complete so we've been restored after being suspended
568
- // Short delay to give websocket time to reconnect
569
- await sleep(100);
570
- // Env may have changed after restore
571
- await this.processEnvOverrides();
572
- // We need to let the platform know we're ready to continue
573
- const continuationResult = await this.httpClient.continueRunExecution(run.friendlyId, snapshot.friendlyId);
574
- if (!continuationResult.success) {
575
- this.sendDebugLog({
576
- runId: run.friendlyId,
577
- message: "failed to continue execution",
578
- properties: {
579
- error: continuationResult.error,
580
- },
581
- });
582
- this.waitForNextRun();
583
- return;
584
- }
585
- return;
586
- }
587
- case "EXECUTING": {
588
- this.sendDebugLog({
589
- runId: run.friendlyId,
590
- message: "Run is now executing",
591
- properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
592
- });
593
- if (completedWaitpoints.length === 0) {
594
- return;
595
- }
596
- this.sendDebugLog({
597
- runId: run.friendlyId,
598
- message: "Processing completed waitpoints",
599
- properties: { completedWaitpoints: completedWaitpoints.length },
600
- });
601
- if (!this.taskRunProcess) {
602
- this.sendDebugLog({
603
- runId: run.friendlyId,
604
- message: "No task run process, ignoring completed waitpoints",
605
- properties: { completedWaitpoints: completedWaitpoints.length },
606
- });
607
- return;
608
- }
609
- for (const waitpoint of completedWaitpoints) {
610
- this.taskRunProcess.waitpointCompleted(waitpoint);
611
- }
612
- return;
613
- }
614
- case "RUN_CREATED":
615
- case "QUEUED": {
616
- this.sendDebugLog({
617
- runId: run.friendlyId,
618
- message: "Status change not handled",
619
- properties: { status: snapshot.executionStatus },
620
- });
621
- return;
622
- }
623
- default: {
624
- assertExhaustive(snapshot.executionStatus);
625
- }
626
- }
627
- }
628
- catch (error) {
629
- this.sendDebugLog({
630
- runId: run.friendlyId,
631
- message: "snapshot change: unexpected error",
632
- properties: {
633
- snapshotId: snapshot.friendlyId,
634
- error: error instanceof Error ? error.message : String(error),
635
- },
636
- });
637
- }
638
- finally {
639
- this.handleSnapshotChangeLock = false;
640
- }
641
- }
642
- async processEnvOverrides() {
643
- if (!this.metadataClient) {
644
- this.sendDebugLog({
645
- runId: this.runFriendlyId,
646
- message: "No metadata client, skipping env overrides",
647
- });
648
- return;
649
- }
650
- const overrides = await this.metadataClient.getEnvOverrides();
651
- if (!overrides) {
652
- this.sendDebugLog({
653
- runId: this.runFriendlyId,
654
- message: "No env overrides, skipping",
655
- });
656
- return;
657
- }
658
- this.sendDebugLog({
659
- runId: this.runFriendlyId,
660
- message: "Processing env overrides",
661
- properties: { ...overrides },
662
- });
663
- if (overrides.TRIGGER_SUCCESS_EXIT_CODE) {
664
- this.successExitCode = overrides.TRIGGER_SUCCESS_EXIT_CODE;
665
- }
666
- if (overrides.TRIGGER_FAILURE_EXIT_CODE) {
667
- this.failureExitCode = overrides.TRIGGER_FAILURE_EXIT_CODE;
668
- }
669
- if (overrides.TRIGGER_HEARTBEAT_INTERVAL_SECONDS) {
670
- this.heartbeatIntervalSeconds = overrides.TRIGGER_HEARTBEAT_INTERVAL_SECONDS;
671
- this.runHeartbeat.updateInterval(this.heartbeatIntervalSeconds * 1000);
672
- }
673
- if (overrides.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS) {
674
- this.snapshotPollIntervalSeconds = overrides.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS;
675
- this.snapshotPoller.updateInterval(this.snapshotPollIntervalSeconds * 1000);
676
- }
677
- if (overrides.TRIGGER_WORKER_INSTANCE_NAME) {
678
- this.workerInstanceName = overrides.TRIGGER_WORKER_INSTANCE_NAME;
679
- }
680
- if (overrides.TRIGGER_SUPERVISOR_API_PROTOCOL ||
681
- overrides.TRIGGER_SUPERVISOR_API_DOMAIN ||
682
- overrides.TRIGGER_SUPERVISOR_API_PORT) {
683
- const protocol = overrides.TRIGGER_SUPERVISOR_API_PROTOCOL ?? env.TRIGGER_SUPERVISOR_API_PROTOCOL;
684
- const domain = overrides.TRIGGER_SUPERVISOR_API_DOMAIN ?? env.TRIGGER_SUPERVISOR_API_DOMAIN;
685
- const port = overrides.TRIGGER_SUPERVISOR_API_PORT ?? env.TRIGGER_SUPERVISOR_API_PORT;
686
- this.workerApiUrl = `${protocol}://${domain}:${port}`;
687
- this.httpClient.updateApiUrl(this.workerApiUrl);
688
- }
689
- if (overrides.TRIGGER_RUNNER_ID) {
690
- this.runnerId = overrides.TRIGGER_RUNNER_ID;
691
- this.httpClient.updateRunnerId(this.runnerId);
692
- }
693
- }
694
- activeRunExecution = null;
695
- async startAndExecuteRunAttempt({ runFriendlyId, snapshotFriendlyId, dequeuedAt, podScheduledAt, isWarmStart, skipLockCheckForImmediateRetry: skipLockCheck, }) {
696
- if (!skipLockCheck && this.activeRunExecution) {
697
- this.sendDebugLog({
698
- runId: runFriendlyId,
699
- message: "startAndExecuteRunAttempt: already in progress",
700
- });
701
- return;
702
- }
703
- const execution = async () => {
704
- if (!this.socket) {
705
- this.sendDebugLog({
706
- runId: runFriendlyId,
707
- message: "Starting run without socket connection",
708
- });
709
- }
710
- this.subscribeToRunNotifications({
711
- run: { friendlyId: runFriendlyId },
712
- snapshot: { friendlyId: snapshotFriendlyId },
713
- });
714
- const attemptStartedAt = Date.now();
715
- const start = await this.httpClient.startRunAttempt(runFriendlyId, snapshotFriendlyId, {
716
- isWarmStart,
717
- });
718
- if (!start.success) {
719
- this.sendDebugLog({
720
- runId: runFriendlyId,
721
- message: "Failed to start run",
722
- properties: { error: start.error },
723
- });
724
- this.sendDebugLog({
725
- runId: runFriendlyId,
726
- message: "failed to start run attempt",
727
- properties: {
728
- error: start.error,
729
- },
730
- });
731
- this.waitForNextRun();
732
- return;
733
- }
734
- const attemptDuration = Date.now() - attemptStartedAt;
735
- const { run, snapshot, execution, envVars } = start.data;
736
- this.sendDebugLog({
737
- runId: run.friendlyId,
738
- message: "Started run",
739
- properties: { snapshot: snapshot.friendlyId },
740
- });
741
- this.enterRunPhase(run, snapshot);
742
- const metrics = [
743
- {
744
- name: "start",
745
- event: "create_attempt",
746
- timestamp: attemptStartedAt,
747
- duration: attemptDuration,
748
- },
749
- ]
750
- .concat(dequeuedAt
751
- ? [
752
- {
753
- name: "start",
754
- event: "dequeue",
755
- timestamp: dequeuedAt.getTime(),
756
- duration: 0,
757
- },
758
- ]
759
- : [])
760
- .concat(podScheduledAt
761
- ? [
762
- {
763
- name: "start",
764
- event: "pod_scheduled",
765
- timestamp: podScheduledAt.getTime(),
766
- duration: 0,
767
- },
768
- ]
769
- : []);
770
- const taskRunEnv = {
771
- ...gatherProcessEnv(),
772
- ...envVars,
773
- };
774
- try {
775
- return await this.executeRun({
776
- run,
777
- snapshot,
778
- envVars: taskRunEnv,
779
- execution,
780
- metrics,
781
- isWarmStart,
782
- });
783
- }
784
- catch (error) {
785
- if (error instanceof SuspendedProcessError) {
786
- this.sendDebugLog({
787
- runId: run.friendlyId,
788
- message: "Run was suspended and task run process was killed, waiting for next run",
789
- properties: { run: run.friendlyId, snapshot: snapshot.friendlyId },
790
- });
791
- this.waitForNextRun();
792
- return;
793
- }
794
- this.sendDebugLog({
795
- runId: run.friendlyId,
796
- message: "Error while executing attempt",
797
- properties: { error: error instanceof Error ? error.message : String(error) },
798
- });
799
- this.sendDebugLog({
800
- runId: run.friendlyId,
801
- message: "Submitting attempt completion",
802
- properties: {
803
- snapshotId: snapshot.friendlyId,
804
- updatedSnapshotId: this.snapshotFriendlyId,
805
- },
806
- });
807
- const completion = {
808
- id: execution.run.id,
809
- ok: false,
810
- retry: undefined,
811
- error: TaskRunProcess.parseExecuteError(error),
812
- };
813
- const completionResult = await this.httpClient.completeRunAttempt(run.friendlyId,
814
- // FIXME: if the snapshot has changed since starting the run, this won't be accurate
815
- // ..but we probably shouldn't fetch the latest snapshot either because we may be in an "unhealthy" state while the next runner has already taken over
816
- this.snapshotFriendlyId ?? snapshot.friendlyId, { completion });
817
- if (!completionResult.success) {
818
- this.sendDebugLog({
819
- runId: run.friendlyId,
820
- message: "Failed to submit completion after error",
821
- properties: { error: completionResult.error },
822
- });
823
- this.sendDebugLog({
824
- runId: run.friendlyId,
825
- message: "completion: failed to submit after error",
826
- properties: {
827
- error: completionResult.error,
828
- },
829
- });
830
- this.waitForNextRun();
831
- return;
832
- }
833
- this.sendDebugLog({
834
- runId: run.friendlyId,
835
- message: "Attempt completion submitted after error",
836
- properties: {
837
- attemptStatus: completionResult.data.result.attemptStatus,
838
- runId: completionResult.data.result.run.friendlyId,
839
- snapshotId: completionResult.data.result.snapshot.friendlyId,
840
- },
841
- });
842
- try {
843
- await this.handleCompletionResult(completion, completionResult.data.result);
844
- }
845
- catch (error) {
846
- this.sendDebugLog({
847
- runId: run.friendlyId,
848
- message: "Failed to handle completion result after error",
849
- properties: { error: error instanceof Error ? error.message : String(error) },
850
- });
851
- this.waitForNextRun();
852
- return;
853
- }
854
- }
855
- };
856
- this.activeRunExecution = execution();
857
- try {
858
- await this.activeRunExecution;
859
- }
860
- catch (error) {
861
- this.sendDebugLog({
862
- runId: runFriendlyId,
863
- message: "startAndExecuteRunAttempt: unexpected error",
864
- properties: { error: error instanceof Error ? error.message : String(error) },
865
- });
866
- }
867
- finally {
868
- this.activeRunExecution = null;
869
- }
870
- }
871
- waitForNextRunLock = false;
872
- /** This will kill the child process before spinning up a new one. It will never throw,
873
- * but may exit the process on any errors or when no runs are available after the
874
- * configured duration. */
875
- async waitForNextRun() {
876
- if (this.waitForNextRunLock) {
877
- this.sendDebugLog({
878
- runId: this.runFriendlyId,
879
- message: "waitForNextRun: already in progress",
880
- });
881
- return;
882
- }
883
- this.waitForNextRunLock = true;
884
- const previousRunId = this.runFriendlyId;
885
- try {
886
- // If there's a run execution in progress, we need to kill it and wait for it to finish
887
- if (this.activeRunExecution) {
888
- this.sendDebugLog({
889
- runId: this.runFriendlyId,
890
- message: "waitForNextRun: waiting for existing run execution to finish",
891
- });
892
- await this.activeRunExecution;
893
- }
894
- // Just for good measure
895
- await this.taskRunProcess?.kill("SIGKILL");
896
- this.sendDebugLog({
897
- runId: this.runFriendlyId,
898
- message: "waitForNextRun: waiting for next run",
899
- });
900
- this.enterWarmStartPhase();
901
- if (!this.warmStartClient) {
902
- this.sendDebugLog({
903
- runId: this.runFriendlyId,
904
- message: "waitForNextRun: warm starts disabled, shutting down",
905
- });
906
- this.exitProcess(this.successExitCode);
907
- }
908
- if (this.taskRunProcess) {
909
- this.sendDebugLog({
910
- runId: this.runFriendlyId,
911
- message: "waitForNextRun: eagerly recreating task run process with options",
912
- });
913
- this.taskRunProcess = new TaskRunProcess({
914
- ...this.taskRunProcess.options,
915
- isWarmStart: true,
916
- }).initialize();
917
- }
918
- else {
919
- this.sendDebugLog({
920
- runId: this.runFriendlyId,
921
- message: "waitForNextRun: no existing task run process, so we can't eagerly recreate it",
922
- });
923
- }
924
- // Check the service is up and get additional warm start config
925
- const connect = await this.warmStartClient.connect();
926
- if (!connect.success) {
927
- this.sendDebugLog({
928
- runId: this.runFriendlyId,
929
- message: "waitForNextRun: failed to connect to warm start service",
930
- properties: {
931
- warmStartUrl: env.TRIGGER_WARM_START_URL,
932
- error: connect.error,
933
- },
934
- });
935
- this.exitProcess(this.successExitCode);
936
- }
937
- const connectionTimeoutMs = connect.data.connectionTimeoutMs ?? env.TRIGGER_WARM_START_CONNECTION_TIMEOUT_MS;
938
- const keepaliveMs = connect.data.keepaliveMs ?? env.TRIGGER_WARM_START_KEEPALIVE_MS;
939
- this.sendDebugLog({
940
- runId: this.runFriendlyId,
941
- message: "waitForNextRun: connected to warm start service",
942
- properties: {
943
- connectionTimeoutMs,
944
- keepaliveMs,
945
- },
946
- });
947
- if (previousRunId) {
948
- this.sendDebugLog({
949
- runId: previousRunId,
950
- message: "warm start: received config",
951
- properties: {
952
- connectionTimeoutMs,
953
- keepaliveMs,
954
- },
955
- });
956
- }
957
- if (!connectionTimeoutMs || !keepaliveMs) {
958
- this.sendDebugLog({
959
- runId: this.runFriendlyId,
960
- message: "waitForNextRun: warm starts disabled after connect",
961
- properties: {
962
- connectionTimeoutMs,
963
- keepaliveMs,
964
- },
965
- });
966
- this.exitProcess(this.successExitCode);
967
- }
968
- const nextRun = await this.warmStartClient.warmStart({
969
- workerInstanceName: this.workerInstanceName,
970
- connectionTimeoutMs,
971
- keepaliveMs,
972
- });
973
- if (!nextRun) {
974
- this.sendDebugLog({
975
- runId: this.runFriendlyId,
976
- message: "waitForNextRun: warm start failed, shutting down",
977
- });
978
- this.exitProcess(this.successExitCode);
979
- }
980
- this.sendDebugLog({
981
- runId: this.runFriendlyId,
982
- message: "waitForNextRun: got next run",
983
- properties: { nextRun: nextRun.run.friendlyId },
984
- });
985
- this.startAndExecuteRunAttempt({
986
- runFriendlyId: nextRun.run.friendlyId,
987
- snapshotFriendlyId: nextRun.snapshot.friendlyId,
988
- dequeuedAt: nextRun.dequeuedAt,
989
- isWarmStart: true,
990
- }).finally(() => { });
991
- return;
992
- }
993
- catch (error) {
994
- this.sendDebugLog({
995
- runId: this.runFriendlyId,
996
- message: "waitForNextRun: unexpected error",
997
- properties: { error: error instanceof Error ? error.message : String(error) },
998
- });
999
- this.exitProcess(this.failureExitCode);
1000
- }
1001
- finally {
1002
- this.waitForNextRunLock = false;
1003
- }
1004
- }
1005
- exitProcess(code) {
1006
- this.sendDebugLog({
1007
- runId: this.runFriendlyId,
1008
- message: "Exiting process",
1009
- properties: { code },
1010
- });
1011
- if (this.taskRunProcess?.isPreparedForNextRun) {
1012
- this.taskRunProcess.forceExit();
1013
- }
1014
- process.exit(code);
1015
- }
1016
- createSocket() {
1017
- const wsUrl = new URL("/workload", this.workerApiUrl);
1018
- this.socket = io(wsUrl.href, {
1019
- transports: ["websocket"],
1020
- extraHeaders: {
1021
- [WORKLOAD_HEADERS.DEPLOYMENT_ID]: env.TRIGGER_DEPLOYMENT_ID,
1022
- [WORKLOAD_HEADERS.RUNNER_ID]: env.TRIGGER_RUNNER_ID,
1023
- },
1024
- });
1025
- this.socket.on("run:notify", async ({ version, run }) => {
1026
- this.sendDebugLog({
1027
- runId: run.friendlyId,
1028
- message: "run:notify received by runner",
1029
- properties: { version, runId: run.friendlyId },
1030
- });
1031
- if (!this.runFriendlyId) {
1032
- this.sendDebugLog({
1033
- runId: run.friendlyId,
1034
- message: "run:notify: ignoring notification, no local run ID",
1035
- properties: {
1036
- currentRunId: this.runFriendlyId,
1037
- currentSnapshotId: this.snapshotFriendlyId,
1038
- },
1039
- });
1040
- return;
1041
- }
1042
- if (run.friendlyId !== this.runFriendlyId) {
1043
- this.sendDebugLog({
1044
- runId: run.friendlyId,
1045
- message: "run:notify: ignoring notification for different run",
1046
- properties: {
1047
- currentRunId: this.runFriendlyId,
1048
- currentSnapshotId: this.snapshotFriendlyId,
1049
- notificationRunId: run.friendlyId,
1050
- },
1051
- });
1052
- return;
1053
- }
1054
- // Reset the (fallback) snapshot poll interval so we don't do unnecessary work
1055
- this.snapshotPoller.resetCurrentInterval();
1056
- const latestSnapshot = await this.httpClient.getRunExecutionData(this.runFriendlyId);
1057
- if (!latestSnapshot.success) {
1058
- this.sendDebugLog({
1059
- runId: this.runFriendlyId,
1060
- message: "run:notify: failed to get latest snapshot data",
1061
- properties: {
1062
- currentRunId: this.runFriendlyId,
1063
- currentSnapshotId: this.snapshotFriendlyId,
1064
- error: latestSnapshot.error,
1065
- },
1066
- });
1067
- return;
1068
- }
1069
- await this.handleSnapshotChange(latestSnapshot.data.execution);
1070
- });
1071
- this.socket.on("connect", () => {
1072
- this.sendDebugLog({
1073
- runId: this.runFriendlyId,
1074
- message: "Connected to supervisor",
1075
- });
1076
- // This should handle the case where we reconnect after being restored
1077
- if (this.state.phase === "RUN") {
1078
- const { run, snapshot } = this.state;
1079
- this.subscribeToRunNotifications({ run, snapshot });
1080
- }
1081
- });
1082
- this.socket.on("connect_error", (error) => {
1083
- this.sendDebugLog({
1084
- runId: this.runFriendlyId,
1085
- message: "Connection error",
1086
- properties: { error: error instanceof Error ? error.message : String(error) },
1087
- });
1088
- });
1089
- this.socket.on("disconnect", (reason, description) => {
1090
- this.sendDebugLog({
1091
- runId: this.runFriendlyId,
1092
- message: "Disconnected from supervisor",
1093
- properties: { reason, description: description?.toString() },
1094
- });
1095
- });
1096
- }
1097
- async executeRun({ run, snapshot, envVars, execution, metrics, isWarmStart, }) {
1098
- this.snapshotPoller.start();
1099
- if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
1100
- this.taskRunProcess = new TaskRunProcess({
1101
- workerManifest: this.workerManifest,
1102
- env: envVars,
1103
- serverWorker: {
1104
- id: "unmanaged",
1105
- contentHash: env.TRIGGER_CONTENT_HASH,
1106
- version: env.TRIGGER_DEPLOYMENT_VERSION,
1107
- engine: "V2",
1108
- },
1109
- machine: execution.machine,
1110
- isWarmStart,
1111
- }).initialize();
1112
- }
1113
- this.sendDebugLog({
1114
- runId: this.runFriendlyId,
1115
- message: "executing task run process",
1116
- properties: {
1117
- attemptId: execution.attempt.id,
1118
- runId: execution.run.id,
1119
- },
1120
- });
1121
- const completion = await this.taskRunProcess.execute({
1122
- payload: {
1123
- execution,
1124
- traceContext: execution.run.traceContext ?? {},
1125
- metrics,
1126
- },
1127
- messageId: run.friendlyId,
1128
- env: envVars,
1129
- }, isWarmStart);
1130
- this.sendDebugLog({
1131
- runId: this.runFriendlyId,
1132
- message: "Completed run",
1133
- properties: { completion: completion.ok },
1134
- });
1135
- try {
1136
- // The execution has finished, so we can cleanup the task run process. Killing it should be safe.
1137
- await this.taskRunProcess.cleanup(true);
1138
- }
1139
- catch (error) {
1140
- this.sendDebugLog({
1141
- runId: this.runFriendlyId,
1142
- message: "Failed to cleanup task run process, submitting completion anyway",
1143
- properties: { error: error instanceof Error ? error.message : String(error) },
1144
- });
1145
- }
1146
- if (!this.runFriendlyId || !this.snapshotFriendlyId) {
1147
- this.sendDebugLog({
1148
- runId: this.runFriendlyId,
1149
- message: "executeRun: Missing run ID or snapshot ID after execution",
1150
- properties: {
1151
- runId: this.runFriendlyId,
1152
- snapshotId: this.snapshotFriendlyId,
1153
- },
1154
- });
1155
- this.waitForNextRun();
1156
- return;
1157
- }
1158
- const completionResult = await this.httpClient.completeRunAttempt(this.runFriendlyId, this.snapshotFriendlyId, {
1159
- completion,
1160
- });
1161
- if (!completionResult.success) {
1162
- this.sendDebugLog({
1163
- runId: run.friendlyId,
1164
- message: "completion: failed to submit",
1165
- properties: {
1166
- error: completionResult.error,
1167
- },
1168
- });
1169
- this.sendDebugLog({
1170
- runId: run.friendlyId,
1171
- message: "completion: failed to submit",
1172
- properties: {
1173
- error: completionResult.error,
1174
- },
1175
- });
1176
- this.waitForNextRun();
1177
- return;
1178
- }
1179
- this.sendDebugLog({
1180
- runId: run.friendlyId,
1181
- message: "Attempt completion submitted",
1182
- properties: {
1183
- attemptStatus: completionResult.data.result.attemptStatus,
1184
- runId: completionResult.data.result.run.friendlyId,
1185
- snapshotId: completionResult.data.result.snapshot.friendlyId,
1186
- },
1187
- });
1188
- try {
1189
- await this.handleCompletionResult(completion, completionResult.data.result);
1190
- }
1191
- catch (error) {
1192
- this.sendDebugLog({
1193
- runId: run.friendlyId,
1194
- message: "Failed to handle completion result",
1195
- properties: { error: error instanceof Error ? error.message : String(error) },
1196
- });
1197
- this.waitForNextRun();
1198
- return;
1199
- }
1200
- }
1201
- async handleCompletionResult(completion, result) {
1202
- this.sendDebugLog({
1203
- runId: this.runFriendlyId,
1204
- message: "Handling completion result",
1205
- properties: {
1206
- completion: completion.ok,
1207
- attemptStatus: result.attemptStatus,
1208
- snapshotId: result.snapshot.friendlyId,
1209
- runId: result.run.friendlyId,
1210
- },
1211
- });
1212
- const { attemptStatus, snapshot: completionSnapshot, run } = result;
1213
- try {
1214
- this.updateRunPhase(run, completionSnapshot);
1215
- }
1216
- catch (error) {
1217
- this.sendDebugLog({
1218
- runId: run.friendlyId,
1219
- message: "Failed to update run phase after completion",
1220
- properties: { error: error instanceof Error ? error.message : String(error) },
1221
- });
1222
- this.waitForNextRun();
1223
- return;
1224
- }
1225
- if (attemptStatus === "RUN_FINISHED") {
1226
- this.sendDebugLog({
1227
- runId: run.friendlyId,
1228
- message: "Run finished",
1229
- });
1230
- this.waitForNextRun();
1231
- return;
1232
- }
1233
- if (attemptStatus === "RUN_PENDING_CANCEL") {
1234
- this.sendDebugLog({
1235
- runId: run.friendlyId,
1236
- message: "Run pending cancel",
1237
- });
1238
- return;
1239
- }
1240
- if (attemptStatus === "RETRY_QUEUED") {
1241
- this.sendDebugLog({
1242
- runId: run.friendlyId,
1243
- message: "Retry queued",
1244
- });
1245
- this.waitForNextRun();
1246
- return;
1247
- }
1248
- if (attemptStatus === "RETRY_IMMEDIATELY") {
1249
- if (completion.ok) {
1250
- throw new Error("Should retry but completion OK.");
1251
- }
1252
- if (!completion.retry) {
1253
- throw new Error("Should retry but missing retry params.");
1254
- }
1255
- await sleep(completion.retry.delay);
1256
- if (!this.snapshotFriendlyId) {
1257
- throw new Error("Missing snapshot ID after retry");
1258
- }
1259
- this.startAndExecuteRunAttempt({
1260
- runFriendlyId: run.friendlyId,
1261
- snapshotFriendlyId: this.snapshotFriendlyId,
1262
- skipLockCheckForImmediateRetry: true,
1263
- isWarmStart: true,
1264
- }).finally(() => { });
1265
- return;
1266
- }
1267
- assertExhaustive(attemptStatus);
1268
- }
1269
- sendDebugLog({ runId, message, date, properties, }) {
1270
- if (!runId) {
1271
- runId = this.runFriendlyId;
1272
- }
1273
- if (!runId) {
1274
- runId = env.TRIGGER_RUN_ID;
1275
- }
1276
- if (!runId) {
1277
- return;
1278
- }
1279
- const mergedProperties = {
1280
- ...properties,
1281
- runId,
1282
- runnerId: this.runnerId,
1283
- workerName: this.workerInstanceName,
1284
- };
1285
- console.log(message, mergedProperties);
1286
- this.httpClient.sendDebugLog(runId, {
1287
- message,
1288
- time: date ?? new Date(),
1289
- properties: mergedProperties,
1290
- });
1291
- }
1292
- async cancelAttempt(runId) {
1293
- this.sendDebugLog({
1294
- runId,
1295
- message: "cancelling attempt",
1296
- properties: { runId },
1297
- });
1298
- await this.taskRunProcess?.cancel();
1299
- }
1300
- async start() {
1301
- this.sendDebugLog({
1302
- runId: this.runFriendlyId,
1303
- message: "Starting up",
1304
- });
1305
- // Websocket notifications are only an optimisation so we don't need to wait for a successful connection
1306
- this.createSocket();
1307
- // If we have run and snapshot IDs, we can start an attempt immediately
1308
- if (env.TRIGGER_RUN_ID && env.TRIGGER_SNAPSHOT_ID) {
1309
- this.startAndExecuteRunAttempt({
1310
- runFriendlyId: env.TRIGGER_RUN_ID,
1311
- snapshotFriendlyId: env.TRIGGER_SNAPSHOT_ID,
1312
- dequeuedAt: env.TRIGGER_DEQUEUED_AT_MS,
1313
- podScheduledAt: env.TRIGGER_POD_SCHEDULED_AT_MS,
1314
- }).finally(() => { });
1315
- return;
1316
- }
1317
- // ..otherwise we need to wait for a run
1318
- this.waitForNextRun();
1319
- return;
1320
- }
1321
- async stop() {
1322
- this.sendDebugLog({
1323
- runId: this.runFriendlyId,
1324
- message: "Shutting down",
1325
- });
1326
- if (this.taskRunProcess) {
1327
- await this.taskRunProcess.cleanup(true);
1328
- }
1329
- this.runHeartbeat.stop();
1330
- this.snapshotPoller.stop();
1331
- this.socket.close();
1332
- }
1333
- }
1334
- const workerManifest = await loadWorkerManifest();
1335
- const prodWorker = new ManagedRunController({ workerManifest });
1336
- await prodWorker.start();
1337
- function gatherProcessEnv() {
1338
- const $env = {
1339
- NODE_ENV: env.NODE_ENV,
1340
- NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS,
1341
- OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT,
1342
- };
1343
- // Filter out undefined values
1344
- return Object.fromEntries(Object.entries($env).filter(([key, value]) => value !== undefined));
1345
- }
1346
- async function loadWorkerManifest() {
1347
- const manifest = await readJSONFile("./index.json");
1348
- return WorkerManifest.parse(manifest);
1349
- }
7
+ const manifest = await readJSONFile("./index.json");
8
+ const workerManifest = WorkerManifest.parse(manifest);
9
+ new ManagedRunController({
10
+ workerManifest,
11
+ env: stdEnv,
12
+ }).start();
1350
13
  //# sourceMappingURL=managed-run-controller.js.map