trigger.dev 3.0.0-beta.3 → 3.0.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Containerfile.prod +17 -4
- package/dist/index.js +2008 -671
- package/dist/index.js.map +1 -1
- package/dist/templates/trigger.config.ts.template +2 -1
- package/dist/workers/dev/worker-facade.js +51 -54
- package/dist/workers/dev/worker-setup.js +8 -15
- package/dist/workers/prod/entry-point.js +177 -78
- package/dist/workers/prod/worker-facade.js +63 -55
- package/dist/workers/prod/worker-setup.js +5 -20
- package/package.json +9 -14
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
CoordinatorToProdWorkerMessages,
|
|
4
4
|
PostStartCauses,
|
|
5
5
|
PreStopCauses,
|
|
6
|
-
ProdWorkerToCoordinatorMessages
|
|
7
|
-
ZodSocketConnection as ZodSocketConnection2
|
|
6
|
+
ProdWorkerToCoordinatorMessages
|
|
8
7
|
} from "@trigger.dev/core/v3";
|
|
8
|
+
import { ZodSocketConnection as ZodSocketConnection2 } from "@trigger.dev/core/v3/zodSocket";
|
|
9
9
|
|
|
10
10
|
// ../core-apps/src/http.ts
|
|
11
11
|
var HttpReply = class {
|
|
@@ -71,10 +71,10 @@ import {
|
|
|
71
71
|
clientWebsocketMessages,
|
|
72
72
|
PlatformToProviderMessages,
|
|
73
73
|
ProviderToPlatformMessages,
|
|
74
|
-
SharedQueueToClientMessages
|
|
75
|
-
ZodMessageSender,
|
|
76
|
-
ZodSocketConnection
|
|
74
|
+
SharedQueueToClientMessages
|
|
77
75
|
} from "@trigger.dev/core/v3";
|
|
76
|
+
import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
|
|
77
|
+
import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
|
|
78
78
|
var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
|
79
79
|
var MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
|
80
80
|
var PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
|
|
@@ -93,9 +93,9 @@ import {
|
|
|
93
93
|
ProdWorkerToChildMessages,
|
|
94
94
|
SemanticInternalAttributes,
|
|
95
95
|
TaskRunErrorCodes,
|
|
96
|
-
ZodIpcConnection,
|
|
97
96
|
correctErrorStackTrace
|
|
98
97
|
} from "@trigger.dev/core/v3";
|
|
98
|
+
import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
|
|
99
99
|
import { Evt } from "evt";
|
|
100
100
|
import { fork } from "node:child_process";
|
|
101
101
|
|
|
@@ -108,6 +108,14 @@ var UncaughtExceptionError = class extends Error {
|
|
|
108
108
|
this.name = "UncaughtExceptionError";
|
|
109
109
|
}
|
|
110
110
|
};
|
|
111
|
+
var TaskMetadataParseError = class extends Error {
|
|
112
|
+
constructor(zodIssues, tasks) {
|
|
113
|
+
super(`Failed to parse task metadata`);
|
|
114
|
+
this.zodIssues = zodIssues;
|
|
115
|
+
this.tasks = tasks;
|
|
116
|
+
this.name = "TaskMetadataParseError";
|
|
117
|
+
}
|
|
118
|
+
};
|
|
111
119
|
|
|
112
120
|
// src/workers/prod/backgroundWorker.ts
|
|
113
121
|
var UnexpectedExitError = class extends Error {
|
|
@@ -140,6 +148,7 @@ var ProdBackgroundWorker = class {
|
|
|
140
148
|
onWaitForDuration = new Evt();
|
|
141
149
|
onWaitForTask = new Evt();
|
|
142
150
|
preCheckpointNotification = Evt.create();
|
|
151
|
+
checkpointCanceledNotification = Evt.create();
|
|
143
152
|
onReadyForCheckpoint = Evt.create();
|
|
144
153
|
onCancelCheckpoint = Evt.create();
|
|
145
154
|
_onClose = new Evt();
|
|
@@ -206,6 +215,14 @@ var ProdBackgroundWorker = class {
|
|
|
206
215
|
reject(new UncaughtExceptionError(message.error, message.origin));
|
|
207
216
|
child.kill();
|
|
208
217
|
}
|
|
218
|
+
},
|
|
219
|
+
TASKS_FAILED_TO_PARSE: async (message) => {
|
|
220
|
+
if (!resolved) {
|
|
221
|
+
clearTimeout(timeout);
|
|
222
|
+
resolved = true;
|
|
223
|
+
reject(new TaskMetadataParseError(message.zodIssues, message.tasks));
|
|
224
|
+
child.kill();
|
|
225
|
+
}
|
|
209
226
|
}
|
|
210
227
|
}
|
|
211
228
|
});
|
|
@@ -247,6 +264,7 @@ var ProdBackgroundWorker = class {
|
|
|
247
264
|
);
|
|
248
265
|
if (!this._taskRunProcess) {
|
|
249
266
|
const taskRunProcess = new TaskRunProcess(
|
|
267
|
+
payload.execution,
|
|
250
268
|
this.path,
|
|
251
269
|
{
|
|
252
270
|
...this.params.env,
|
|
@@ -279,6 +297,9 @@ var ProdBackgroundWorker = class {
|
|
|
279
297
|
this.preCheckpointNotification.attach((message) => {
|
|
280
298
|
taskRunProcess.preCheckpointNotification.post(message);
|
|
281
299
|
});
|
|
300
|
+
this.checkpointCanceledNotification.attach((message) => {
|
|
301
|
+
taskRunProcess.checkpointCanceledNotification.post(message);
|
|
302
|
+
});
|
|
282
303
|
await taskRunProcess.initialize();
|
|
283
304
|
this._taskRunProcess = taskRunProcess;
|
|
284
305
|
}
|
|
@@ -289,7 +310,6 @@ var ProdBackgroundWorker = class {
|
|
|
289
310
|
try {
|
|
290
311
|
const taskRunProcess = await this.#initializeTaskRunProcess(payload);
|
|
291
312
|
const result = await taskRunProcess.executeTaskRun(payload);
|
|
292
|
-
await taskRunProcess.cleanup(result.ok || result.retry === void 0);
|
|
293
313
|
if (result.ok) {
|
|
294
314
|
return result;
|
|
295
315
|
}
|
|
@@ -358,7 +378,8 @@ var ProdBackgroundWorker = class {
|
|
|
358
378
|
}
|
|
359
379
|
};
|
|
360
380
|
var TaskRunProcess = class {
|
|
361
|
-
constructor(path, env, metadata, worker) {
|
|
381
|
+
constructor(execution, path, env, metadata, worker) {
|
|
382
|
+
this.execution = execution;
|
|
362
383
|
this.path = path;
|
|
363
384
|
this.env = env;
|
|
364
385
|
this.metadata = metadata;
|
|
@@ -377,6 +398,7 @@ var TaskRunProcess = class {
|
|
|
377
398
|
onWaitForDuration = new Evt();
|
|
378
399
|
onWaitForTask = new Evt();
|
|
379
400
|
preCheckpointNotification = Evt.create();
|
|
401
|
+
checkpointCanceledNotification = Evt.create();
|
|
380
402
|
onReadyForCheckpoint = Evt.create();
|
|
381
403
|
onCancelCheckpoint = Evt.create();
|
|
382
404
|
async initialize() {
|
|
@@ -391,6 +413,7 @@ var TaskRunProcess = class {
|
|
|
391
413
|
"ipc"
|
|
392
414
|
],
|
|
393
415
|
env: {
|
|
416
|
+
...this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
|
|
394
417
|
...this.env,
|
|
395
418
|
OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
|
|
396
419
|
[SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
|
|
@@ -418,28 +441,56 @@ var TaskRunProcess = class {
|
|
|
418
441
|
resolver(result);
|
|
419
442
|
},
|
|
420
443
|
READY_TO_DISPOSE: async (message) => {
|
|
444
|
+
process.exit(0);
|
|
421
445
|
},
|
|
422
446
|
TASK_HEARTBEAT: async (message) => {
|
|
423
447
|
this.onTaskHeartbeat.post(message.id);
|
|
424
448
|
},
|
|
425
449
|
TASKS_READY: async (message) => {
|
|
426
450
|
},
|
|
451
|
+
WAIT_FOR_TASK: async (message) => {
|
|
452
|
+
this.onWaitForTask.post(message);
|
|
453
|
+
},
|
|
427
454
|
WAIT_FOR_BATCH: async (message) => {
|
|
428
455
|
this.onWaitForBatch.post(message);
|
|
429
456
|
},
|
|
430
457
|
WAIT_FOR_DURATION: async (message) => {
|
|
431
458
|
this.onWaitForDuration.post(message);
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
459
|
+
try {
|
|
460
|
+
const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor(
|
|
461
|
+
3e4
|
|
462
|
+
);
|
|
463
|
+
return {
|
|
464
|
+
willCheckpointAndRestore
|
|
465
|
+
};
|
|
466
|
+
} catch (error) {
|
|
467
|
+
console.error("Error while waiting for pre-checkpoint notification", error);
|
|
468
|
+
return {
|
|
469
|
+
willCheckpointAndRestore: false
|
|
470
|
+
};
|
|
471
|
+
}
|
|
437
472
|
},
|
|
438
473
|
READY_FOR_CHECKPOINT: async (message) => {
|
|
439
474
|
this.onReadyForCheckpoint.post(message);
|
|
440
475
|
},
|
|
441
476
|
CANCEL_CHECKPOINT: async (message) => {
|
|
477
|
+
const version = "v2";
|
|
442
478
|
this.onCancelCheckpoint.post(message);
|
|
479
|
+
try {
|
|
480
|
+
const { checkpointCanceled } = await this.checkpointCanceledNotification.waitFor(
|
|
481
|
+
3e4
|
|
482
|
+
);
|
|
483
|
+
return {
|
|
484
|
+
version,
|
|
485
|
+
checkpointCanceled
|
|
486
|
+
};
|
|
487
|
+
} catch (error) {
|
|
488
|
+
console.error("Error while waiting for checkpoint cancellation", error);
|
|
489
|
+
return {
|
|
490
|
+
version,
|
|
491
|
+
checkpointCanceled: true
|
|
492
|
+
};
|
|
493
|
+
}
|
|
443
494
|
}
|
|
444
495
|
}
|
|
445
496
|
});
|
|
@@ -558,6 +609,7 @@ var logger2 = new SimpleLogger(`[${MACHINE_NAME2}][${SHORT_HASH}]`);
|
|
|
558
609
|
var ProdWorker = class {
|
|
559
610
|
constructor(port, host = "0.0.0.0") {
|
|
560
611
|
this.host = host;
|
|
612
|
+
process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
|
|
561
613
|
this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
|
|
562
614
|
this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
|
563
615
|
projectConfig: __PROJECT_CONFIG__,
|
|
@@ -573,18 +625,26 @@ var ProdWorker = class {
|
|
|
573
625
|
this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
|
|
574
626
|
});
|
|
575
627
|
this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
|
|
628
|
+
await this.#backgroundWorker.flushTelemetry();
|
|
576
629
|
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
|
577
630
|
});
|
|
578
631
|
this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
|
|
579
|
-
logger2.log("onCancelCheckpoint
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
632
|
+
logger2.log("onCancelCheckpoint", { message });
|
|
633
|
+
const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
|
|
634
|
+
"CANCEL_CHECKPOINT",
|
|
635
|
+
{
|
|
636
|
+
version: "v2",
|
|
637
|
+
reason: message.reason
|
|
638
|
+
}
|
|
639
|
+
);
|
|
640
|
+
if (checkpointCanceled) {
|
|
641
|
+
if (message.reason === "WAIT_FOR_DURATION") {
|
|
642
|
+
this.paused = false;
|
|
643
|
+
this.nextResumeAfter = void 0;
|
|
644
|
+
this.waitForPostStart = false;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
this.#backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
|
|
588
648
|
});
|
|
589
649
|
this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
|
|
590
650
|
if (!this.attemptFriendlyId) {
|
|
@@ -650,40 +710,66 @@ var ProdWorker = class {
|
|
|
650
710
|
#backgroundWorker;
|
|
651
711
|
#httpServer;
|
|
652
712
|
#coordinatorSocket;
|
|
653
|
-
async #
|
|
713
|
+
async #handleSignal(signal) {
|
|
714
|
+
logger2.log("Received signal", { signal });
|
|
715
|
+
if (signal === "SIGTERM") {
|
|
716
|
+
if (this.executing) {
|
|
717
|
+
const terminationGracePeriodSeconds = 60 * 60;
|
|
718
|
+
logger2.log("Waiting for attempt to complete before exiting", {
|
|
719
|
+
terminationGracePeriodSeconds
|
|
720
|
+
});
|
|
721
|
+
await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
|
|
722
|
+
logger2.log("Termination timeout reached, exiting gracefully.");
|
|
723
|
+
} else {
|
|
724
|
+
logger2.log("Not executing, exiting immediately.");
|
|
725
|
+
}
|
|
726
|
+
await this.#exitGracefully();
|
|
727
|
+
}
|
|
728
|
+
logger2.log("Unhandled signal", { signal });
|
|
729
|
+
}
|
|
730
|
+
async #exitGracefully() {
|
|
731
|
+
await this.#backgroundWorker.close();
|
|
732
|
+
process.exit(0);
|
|
733
|
+
}
|
|
734
|
+
async #reconnect(isPostStart = false, reconnectImmediately = false) {
|
|
654
735
|
if (isPostStart) {
|
|
655
736
|
this.waitForPostStart = false;
|
|
656
737
|
}
|
|
657
738
|
this.#coordinatorSocket.close();
|
|
658
|
-
if (!
|
|
659
|
-
|
|
660
|
-
return;
|
|
739
|
+
if (!reconnectImmediately) {
|
|
740
|
+
await setTimeout2(1e3);
|
|
661
741
|
}
|
|
742
|
+
let coordinatorHost = COORDINATOR_HOST;
|
|
662
743
|
try {
|
|
663
|
-
|
|
664
|
-
"
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
744
|
+
if (this.runningInKubernetes) {
|
|
745
|
+
coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
|
|
746
|
+
"\n",
|
|
747
|
+
""
|
|
748
|
+
);
|
|
749
|
+
logger2.log("reconnecting", {
|
|
750
|
+
coordinatorHost: {
|
|
751
|
+
fromEnv: COORDINATOR_HOST,
|
|
752
|
+
fromVolume: coordinatorHost,
|
|
753
|
+
current: this.#coordinatorSocket.socket.io.opts.hostname
|
|
754
|
+
}
|
|
755
|
+
});
|
|
756
|
+
}
|
|
675
757
|
} catch (error) {
|
|
676
758
|
logger2.error("taskinfo read error during reconnect", { error });
|
|
677
|
-
|
|
759
|
+
} finally {
|
|
760
|
+
this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
|
|
678
761
|
}
|
|
679
762
|
}
|
|
680
|
-
#prepareForWait(reason, willCheckpointAndRestore) {
|
|
763
|
+
async #prepareForWait(reason, willCheckpointAndRestore) {
|
|
681
764
|
logger2.log(`prepare for ${reason}`, { willCheckpointAndRestore });
|
|
682
765
|
this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
|
|
683
766
|
if (willCheckpointAndRestore) {
|
|
684
767
|
this.paused = true;
|
|
685
768
|
this.nextResumeAfter = reason;
|
|
686
769
|
this.waitForPostStart = true;
|
|
770
|
+
if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
|
|
771
|
+
await this.#backgroundWorker.flushTelemetry();
|
|
772
|
+
}
|
|
687
773
|
}
|
|
688
774
|
}
|
|
689
775
|
async #prepareForRetry(willCheckpointAndRestore, shouldExit) {
|
|
@@ -692,8 +778,7 @@ var ProdWorker = class {
|
|
|
692
778
|
if (willCheckpointAndRestore) {
|
|
693
779
|
logger2.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
|
|
694
780
|
}
|
|
695
|
-
await this.#
|
|
696
|
-
process.exit(0);
|
|
781
|
+
await this.#exitGracefully();
|
|
697
782
|
}
|
|
698
783
|
this.executing = false;
|
|
699
784
|
this.attemptFriendlyId = void 0;
|
|
@@ -706,6 +791,7 @@ var ProdWorker = class {
|
|
|
706
791
|
#resumeAfterDuration() {
|
|
707
792
|
this.paused = false;
|
|
708
793
|
this.nextResumeAfter = void 0;
|
|
794
|
+
this.waitForPostStart = false;
|
|
709
795
|
this.#backgroundWorker.waitCompletedNotification();
|
|
710
796
|
}
|
|
711
797
|
#returnValidatedExtraHeaders(headers) {
|
|
@@ -716,6 +802,7 @@ var ProdWorker = class {
|
|
|
716
802
|
}
|
|
717
803
|
return headers;
|
|
718
804
|
}
|
|
805
|
+
// FIXME: If the the worker can't connect for a while, this runs MANY times - it should only run once
|
|
719
806
|
#createCoordinatorSocket(host) {
|
|
720
807
|
const extraHeaders = this.#returnValidatedExtraHeaders({
|
|
721
808
|
"x-machine-name": MACHINE_NAME2,
|
|
@@ -780,6 +867,7 @@ var ProdWorker = class {
|
|
|
780
867
|
}
|
|
781
868
|
this.paused = false;
|
|
782
869
|
this.nextResumeAfter = void 0;
|
|
870
|
+
this.waitForPostStart = false;
|
|
783
871
|
for (let i = 0; i < message.completions.length; i++) {
|
|
784
872
|
const completion = message.completions[i];
|
|
785
873
|
const execution = message.executions[i];
|
|
@@ -817,7 +905,6 @@ var ProdWorker = class {
|
|
|
817
905
|
const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
|
|
818
906
|
logger2.log("completed", completion);
|
|
819
907
|
this.completed.add(executionPayload.execution.attempt.id);
|
|
820
|
-
await this.#backgroundWorker.flushTelemetry();
|
|
821
908
|
const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
|
822
909
|
version: "v1",
|
|
823
910
|
execution: executionPayload.execution,
|
|
@@ -852,6 +939,21 @@ var ProdWorker = class {
|
|
|
852
939
|
logger3.log("skip connection handler, waiting for post start hook");
|
|
853
940
|
return;
|
|
854
941
|
}
|
|
942
|
+
if (this.paused) {
|
|
943
|
+
if (!this.nextResumeAfter) {
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
if (!this.attemptFriendlyId) {
|
|
947
|
+
logger3.error("Missing friendly ID");
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
socket.emit("READY_FOR_RESUME", {
|
|
951
|
+
version: "v1",
|
|
952
|
+
attemptFriendlyId: this.attemptFriendlyId,
|
|
953
|
+
type: this.nextResumeAfter
|
|
954
|
+
});
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
855
957
|
if (process.env.INDEX_TASKS === "true") {
|
|
856
958
|
try {
|
|
857
959
|
const taskResources = await this.#initializeWorker();
|
|
@@ -868,30 +970,46 @@ var ProdWorker = class {
|
|
|
868
970
|
process.exit(1);
|
|
869
971
|
}
|
|
870
972
|
} catch (e) {
|
|
871
|
-
if (e instanceof
|
|
872
|
-
logger3.error("
|
|
973
|
+
if (e instanceof TaskMetadataParseError) {
|
|
974
|
+
logger3.error("tasks metadata parse error", {
|
|
975
|
+
zodIssues: e.zodIssues,
|
|
976
|
+
tasks: e.tasks
|
|
977
|
+
});
|
|
873
978
|
socket.emit("INDEXING_FAILED", {
|
|
874
979
|
version: "v1",
|
|
875
980
|
deploymentId: this.deploymentId,
|
|
876
981
|
error: {
|
|
877
|
-
name:
|
|
878
|
-
message:
|
|
879
|
-
stack: e.
|
|
982
|
+
name: "TaskMetadataParseError",
|
|
983
|
+
message: "There was an error parsing the task metadata",
|
|
984
|
+
stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks })
|
|
880
985
|
}
|
|
881
986
|
});
|
|
987
|
+
} else if (e instanceof UncaughtExceptionError) {
|
|
988
|
+
const error = {
|
|
989
|
+
name: e.originalError.name,
|
|
990
|
+
message: e.originalError.message,
|
|
991
|
+
stack: e.originalError.stack
|
|
992
|
+
};
|
|
993
|
+
logger3.error("uncaught exception", { originalError: error });
|
|
994
|
+
socket.emit("INDEXING_FAILED", {
|
|
995
|
+
version: "v1",
|
|
996
|
+
deploymentId: this.deploymentId,
|
|
997
|
+
error
|
|
998
|
+
});
|
|
882
999
|
} else if (e instanceof Error) {
|
|
883
|
-
|
|
1000
|
+
const error = {
|
|
1001
|
+
name: e.name,
|
|
1002
|
+
message: e.message,
|
|
1003
|
+
stack: e.stack
|
|
1004
|
+
};
|
|
1005
|
+
logger3.error("error", { error });
|
|
884
1006
|
socket.emit("INDEXING_FAILED", {
|
|
885
1007
|
version: "v1",
|
|
886
1008
|
deploymentId: this.deploymentId,
|
|
887
|
-
error
|
|
888
|
-
name: e.name,
|
|
889
|
-
message: e.message,
|
|
890
|
-
stack: e.stack
|
|
891
|
-
}
|
|
1009
|
+
error
|
|
892
1010
|
});
|
|
893
1011
|
} else if (typeof e === "string") {
|
|
894
|
-
logger3.error("string error", { message: e });
|
|
1012
|
+
logger3.error("string error", { error: { message: e } });
|
|
895
1013
|
socket.emit("INDEXING_FAILED", {
|
|
896
1014
|
version: "v1",
|
|
897
1015
|
deploymentId: this.deploymentId,
|
|
@@ -912,27 +1030,8 @@ var ProdWorker = class {
|
|
|
912
1030
|
});
|
|
913
1031
|
}
|
|
914
1032
|
await setTimeout2(200);
|
|
915
|
-
process.exit(
|
|
916
|
-
}
|
|
917
|
-
}
|
|
918
|
-
if (this.paused) {
|
|
919
|
-
if (!this.nextResumeAfter) {
|
|
920
|
-
return;
|
|
921
|
-
}
|
|
922
|
-
if (!this.attemptFriendlyId) {
|
|
923
|
-
logger3.error("Missing friendly ID");
|
|
924
|
-
return;
|
|
1033
|
+
process.exit(111);
|
|
925
1034
|
}
|
|
926
|
-
if (this.nextResumeAfter === "WAIT_FOR_DURATION") {
|
|
927
|
-
this.#resumeAfterDuration();
|
|
928
|
-
return;
|
|
929
|
-
}
|
|
930
|
-
socket.emit("READY_FOR_RESUME", {
|
|
931
|
-
version: "v1",
|
|
932
|
-
attemptFriendlyId: this.attemptFriendlyId,
|
|
933
|
-
type: this.nextResumeAfter
|
|
934
|
-
});
|
|
935
|
-
return;
|
|
936
1035
|
}
|
|
937
1036
|
if (this.executing) {
|
|
938
1037
|
return;
|
|
@@ -970,7 +1069,8 @@ var ProdWorker = class {
|
|
|
970
1069
|
case "/status": {
|
|
971
1070
|
return reply.json({
|
|
972
1071
|
executing: this.executing,
|
|
973
|
-
|
|
1072
|
+
paused: this.paused,
|
|
1073
|
+
completed: this.completed.size,
|
|
974
1074
|
nextResumeAfter: this.nextResumeAfter
|
|
975
1075
|
});
|
|
976
1076
|
}
|
|
@@ -1008,7 +1108,6 @@ var ProdWorker = class {
|
|
|
1008
1108
|
break;
|
|
1009
1109
|
}
|
|
1010
1110
|
}
|
|
1011
|
-
logger2.log("preStop", { url: req.url });
|
|
1012
1111
|
return reply.text("preStop ok");
|
|
1013
1112
|
}
|
|
1014
1113
|
case "/postStart": {
|
|
@@ -1025,7 +1124,7 @@ var ProdWorker = class {
|
|
|
1025
1124
|
break;
|
|
1026
1125
|
}
|
|
1027
1126
|
case "restore": {
|
|
1028
|
-
await this.#reconnect(true);
|
|
1127
|
+
await this.#reconnect(true, true);
|
|
1029
1128
|
break;
|
|
1030
1129
|
}
|
|
1031
1130
|
default: {
|
|
@@ -2,24 +2,30 @@
|
|
|
2
2
|
import {
|
|
3
3
|
ProdChildToWorkerMessages,
|
|
4
4
|
ProdWorkerToChildMessages,
|
|
5
|
+
clock,
|
|
6
|
+
taskCatalog
|
|
7
|
+
} from "@trigger.dev/core/v3";
|
|
8
|
+
import {
|
|
5
9
|
TaskExecutor,
|
|
6
|
-
ZodIpcConnection,
|
|
7
10
|
DurableClock,
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
getEnvVar,
|
|
12
|
+
logLevels,
|
|
13
|
+
OtelTaskLogger,
|
|
14
|
+
ConsoleInterceptor
|
|
15
|
+
} from "@trigger.dev/core/v3/workers";
|
|
16
|
+
import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
|
|
17
|
+
import { ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler";
|
|
10
18
|
import "source-map-support/register.js";
|
|
11
19
|
import {
|
|
12
|
-
ConsoleInterceptor,
|
|
13
|
-
OtelTaskLogger,
|
|
14
|
-
ProdRuntimeManager,
|
|
15
20
|
TaskRunErrorCodes,
|
|
16
21
|
TriggerTracer,
|
|
17
22
|
logger,
|
|
18
23
|
runtime
|
|
19
24
|
} from "@trigger.dev/core/v3";
|
|
25
|
+
import { ProdRuntimeManager } from "@trigger.dev/core/v3/prod";
|
|
20
26
|
|
|
21
27
|
// package.json
|
|
22
|
-
var version = "3.0.0-beta.
|
|
28
|
+
var version = "3.0.0-beta.30";
|
|
23
29
|
|
|
24
30
|
// src/workers/prod/worker-facade.ts
|
|
25
31
|
__WORKER_SETUP__;
|
|
@@ -29,60 +35,33 @@ var otelLogger = tracingSDK.getLogger("trigger-prod-worker", version);
|
|
|
29
35
|
var durableClock = new DurableClock();
|
|
30
36
|
clock.setGlobalClock(durableClock);
|
|
31
37
|
var tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
|
|
32
|
-
var consoleInterceptor = new ConsoleInterceptor(otelLogger);
|
|
38
|
+
var consoleInterceptor = new ConsoleInterceptor(otelLogger, true);
|
|
39
|
+
var triggerLogLevel = getEnvVar("TRIGGER_LOG_LEVEL");
|
|
40
|
+
var configLogLevel = triggerLogLevel ? triggerLogLevel : importedConfig ? importedConfig.logLevel : __PROJECT_CONFIG__.logLevel;
|
|
33
41
|
var otelTaskLogger = new OtelTaskLogger({
|
|
34
42
|
logger: otelLogger,
|
|
35
43
|
tracer,
|
|
36
|
-
level: "info"
|
|
44
|
+
level: logLevels.includes(configLogLevel) ? configLogLevel : "info"
|
|
37
45
|
});
|
|
38
46
|
logger.setGlobalTaskLogger(otelTaskLogger);
|
|
39
47
|
var TaskFileImports = {};
|
|
40
48
|
var TaskFiles = {};
|
|
41
49
|
__TASKS__;
|
|
42
|
-
|
|
43
|
-
const result = [];
|
|
50
|
+
(() => {
|
|
44
51
|
for (const [importName, taskFile] of Object.entries(TaskFiles)) {
|
|
45
52
|
const fileImports = TaskFileImports[importName];
|
|
46
53
|
for (const [exportName, task] of Object.entries(fileImports ?? {})) {
|
|
47
|
-
if (task.
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
retry: task.__trigger.retry,
|
|
55
|
-
machine: task.__trigger.machine,
|
|
56
|
-
fns: task.__trigger.fns
|
|
57
|
-
});
|
|
54
|
+
if (typeof task === "object" && task !== null && "id" in task && typeof task.id === "string") {
|
|
55
|
+
if (taskCatalog.taskExists(task.id)) {
|
|
56
|
+
taskCatalog.registerTaskFileMetadata(task.id, {
|
|
57
|
+
exportName,
|
|
58
|
+
filePath: taskFile.filePath
|
|
59
|
+
});
|
|
60
|
+
}
|
|
58
61
|
}
|
|
59
62
|
}
|
|
60
63
|
}
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
function getTaskMetadata() {
|
|
64
|
-
const result = getTasks();
|
|
65
|
-
return result.map((task) => {
|
|
66
|
-
const { fns, ...metadata } = task;
|
|
67
|
-
return metadata;
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
var tasks = getTasks();
|
|
71
|
-
runtime.registerTasks(tasks);
|
|
72
|
-
var taskExecutors = /* @__PURE__ */ new Map();
|
|
73
|
-
for (const task of tasks) {
|
|
74
|
-
taskExecutors.set(
|
|
75
|
-
task.id,
|
|
76
|
-
new TaskExecutor(task, {
|
|
77
|
-
tracer,
|
|
78
|
-
tracingSDK,
|
|
79
|
-
consoleInterceptor,
|
|
80
|
-
projectConfig: __PROJECT_CONFIG__,
|
|
81
|
-
importedConfig,
|
|
82
|
-
handleErrorFn: handleError
|
|
83
|
-
})
|
|
84
|
-
);
|
|
85
|
-
}
|
|
64
|
+
})();
|
|
86
65
|
var _execution;
|
|
87
66
|
var _isRunning = false;
|
|
88
67
|
var zodIpc = new ZodIpcConnection({
|
|
@@ -97,7 +76,7 @@ var zodIpc = new ZodIpcConnection({
|
|
|
97
76
|
execution,
|
|
98
77
|
result: {
|
|
99
78
|
ok: false,
|
|
100
|
-
id: execution.
|
|
79
|
+
id: execution.run.id,
|
|
101
80
|
error: {
|
|
102
81
|
type: "INTERNAL_ERROR",
|
|
103
82
|
code: TaskRunErrorCodes.TASK_ALREADY_RUNNING
|
|
@@ -107,14 +86,14 @@ var zodIpc = new ZodIpcConnection({
|
|
|
107
86
|
return;
|
|
108
87
|
}
|
|
109
88
|
process.title = `trigger-prod-worker: ${execution.task.id} ${execution.run.id}`;
|
|
110
|
-
const
|
|
111
|
-
if (!
|
|
112
|
-
console.error(`Could not find
|
|
89
|
+
const task = taskCatalog.getTask(execution.task.id);
|
|
90
|
+
if (!task) {
|
|
91
|
+
console.error(`Could not find task ${execution.task.id}`);
|
|
113
92
|
await sender.send("TASK_RUN_COMPLETED", {
|
|
114
93
|
execution,
|
|
115
94
|
result: {
|
|
116
95
|
ok: false,
|
|
117
|
-
id: execution.
|
|
96
|
+
id: execution.run.id,
|
|
118
97
|
error: {
|
|
119
98
|
type: "INTERNAL_ERROR",
|
|
120
99
|
code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR
|
|
@@ -123,6 +102,14 @@ var zodIpc = new ZodIpcConnection({
|
|
|
123
102
|
});
|
|
124
103
|
return;
|
|
125
104
|
}
|
|
105
|
+
const executor = new TaskExecutor(task, {
|
|
106
|
+
tracer,
|
|
107
|
+
tracingSDK,
|
|
108
|
+
consoleInterceptor,
|
|
109
|
+
projectConfig: __PROJECT_CONFIG__,
|
|
110
|
+
importedConfig,
|
|
111
|
+
handleErrorFn: handleError
|
|
112
|
+
});
|
|
126
113
|
try {
|
|
127
114
|
_execution = execution;
|
|
128
115
|
_isRunning = true;
|
|
@@ -140,11 +127,25 @@ var zodIpc = new ZodIpcConnection({
|
|
|
140
127
|
prodRuntimeManager.resumeTask(completion, execution);
|
|
141
128
|
},
|
|
142
129
|
WAIT_COMPLETED_NOTIFICATION: async () => {
|
|
143
|
-
prodRuntimeManager.
|
|
130
|
+
prodRuntimeManager.resumeAfterDuration();
|
|
144
131
|
},
|
|
145
132
|
CLEANUP: async ({ flush, kill }, sender) => {
|
|
146
133
|
if (kill) {
|
|
147
134
|
await tracingSDK.flush();
|
|
135
|
+
if (_execution) {
|
|
136
|
+
await sender.send("TASK_RUN_COMPLETED", {
|
|
137
|
+
execution: _execution,
|
|
138
|
+
result: {
|
|
139
|
+
ok: false,
|
|
140
|
+
id: _execution.run.id,
|
|
141
|
+
error: {
|
|
142
|
+
type: "INTERNAL_ERROR",
|
|
143
|
+
code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
|
|
144
|
+
message: "Worker process killed while attempt in progress."
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
148
149
|
await sender.send("READY_TO_DISPOSE", void 0);
|
|
149
150
|
} else {
|
|
150
151
|
if (flush) {
|
|
@@ -154,12 +155,19 @@ var zodIpc = new ZodIpcConnection({
|
|
|
154
155
|
}
|
|
155
156
|
}
|
|
156
157
|
});
|
|
158
|
+
process.on("SIGTERM", async () => {
|
|
159
|
+
});
|
|
157
160
|
var prodRuntimeManager = new ProdRuntimeManager(zodIpc, {
|
|
158
161
|
waitThresholdInMs: parseInt(process.env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10)
|
|
159
162
|
});
|
|
160
163
|
runtime.setGlobalRuntimeManager(prodRuntimeManager);
|
|
161
|
-
|
|
162
|
-
|
|
164
|
+
var TASK_METADATA = taskCatalog.getAllTaskMetadata();
|
|
165
|
+
zodIpc.send("TASKS_READY", { tasks: TASK_METADATA }).catch((err) => {
|
|
166
|
+
if (err instanceof ZodSchemaParsedError) {
|
|
167
|
+
zodIpc.send("TASKS_FAILED_TO_PARSE", { zodIssues: err.error.issues, tasks: TASK_METADATA });
|
|
168
|
+
} else {
|
|
169
|
+
console.error("Failed to send TASKS_READY message", err);
|
|
170
|
+
}
|
|
163
171
|
});
|
|
164
172
|
process.title = "trigger-prod-worker";
|
|
165
173
|
async function asyncHeartbeat(initialDelayInSeconds = 30, intervalInSeconds = 5) {
|