trigger.dev 3.0.0-beta.36 → 3.0.0-beta.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +91 -68
- package/dist/index.js.map +1 -1
- package/dist/workers/dev/worker-facade.js +19 -3
- package/dist/workers/dev/worker-setup.js +1 -1
- package/dist/workers/prod/entry-point.js +94 -101
- package/dist/workers/prod/worker-facade.js +43 -11
- package/dist/workers/prod/worker-setup.js +1 -1
- package/package.json +7 -6
|
@@ -9,7 +9,9 @@ import {
|
|
|
9
9
|
getEnvVar,
|
|
10
10
|
logLevels,
|
|
11
11
|
OtelTaskLogger,
|
|
12
|
-
ConsoleInterceptor
|
|
12
|
+
ConsoleInterceptor,
|
|
13
|
+
usage,
|
|
14
|
+
DevUsageManager
|
|
13
15
|
} from "@trigger.dev/core/v3/workers";
|
|
14
16
|
import {
|
|
15
17
|
TaskRunErrorCodes,
|
|
@@ -27,6 +29,7 @@ __WORKER_SETUP__;
|
|
|
27
29
|
__IMPORTED_PROJECT_CONFIG__;
|
|
28
30
|
var durableClock = new DurableClock();
|
|
29
31
|
clock.setGlobalClock(durableClock);
|
|
32
|
+
usage.setGlobalUsageManager(new DevUsageManager());
|
|
30
33
|
var tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
|
|
31
34
|
var consoleInterceptor = new ConsoleInterceptor(
|
|
32
35
|
otelLogger,
|
|
@@ -76,6 +79,9 @@ var handler = new ZodMessageHandler({
|
|
|
76
79
|
error: {
|
|
77
80
|
type: "INTERNAL_ERROR",
|
|
78
81
|
code: TaskRunErrorCodes.TASK_ALREADY_RUNNING
|
|
82
|
+
},
|
|
83
|
+
usage: {
|
|
84
|
+
durationMs: 0
|
|
79
85
|
}
|
|
80
86
|
}
|
|
81
87
|
});
|
|
@@ -93,6 +99,9 @@ var handler = new ZodMessageHandler({
|
|
|
93
99
|
error: {
|
|
94
100
|
type: "INTERNAL_ERROR",
|
|
95
101
|
code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR
|
|
102
|
+
},
|
|
103
|
+
usage: {
|
|
104
|
+
durationMs: 0
|
|
96
105
|
}
|
|
97
106
|
}
|
|
98
107
|
});
|
|
@@ -109,10 +118,17 @@ var handler = new ZodMessageHandler({
|
|
|
109
118
|
try {
|
|
110
119
|
_execution = execution;
|
|
111
120
|
_isRunning = true;
|
|
112
|
-
const
|
|
121
|
+
const measurement = usage.start();
|
|
122
|
+
const { result } = await executor.execute(execution, metadata, traceContext, measurement);
|
|
123
|
+
const usageSample = usage.stop(measurement);
|
|
113
124
|
return sender.send("TASK_RUN_COMPLETED", {
|
|
114
125
|
execution,
|
|
115
|
-
result
|
|
126
|
+
result: {
|
|
127
|
+
...result,
|
|
128
|
+
usage: {
|
|
129
|
+
durationMs: usageSample.cpuTime
|
|
130
|
+
}
|
|
131
|
+
}
|
|
116
132
|
});
|
|
117
133
|
} finally {
|
|
118
134
|
_execution = void 0;
|
|
@@ -8,7 +8,7 @@ import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
|
|
|
8
8
|
import "source-map-support/register.js";
|
|
9
9
|
|
|
10
10
|
// package.json
|
|
11
|
-
var version = "3.0.0-beta.
|
|
11
|
+
var version = "3.0.0-beta.38";
|
|
12
12
|
|
|
13
13
|
// src/workers/dev/worker-setup.ts
|
|
14
14
|
__SETUP_IMPORTED_PROJECT_CONFIG__;
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
PreStopCauses,
|
|
6
6
|
ProdWorkerToCoordinatorMessages
|
|
7
7
|
} from "@trigger.dev/core/v3";
|
|
8
|
-
import { ZodSocketConnection
|
|
8
|
+
import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
|
|
9
9
|
|
|
10
10
|
// ../core-apps/src/http.ts
|
|
11
11
|
var HttpReply = class {
|
|
@@ -65,24 +65,6 @@ var SimpleLogger = class {
|
|
|
65
65
|
}
|
|
66
66
|
};
|
|
67
67
|
|
|
68
|
-
// ../core-apps/src/provider.ts
|
|
69
|
-
import {
|
|
70
|
-
ClientToSharedQueueMessages,
|
|
71
|
-
clientWebsocketMessages,
|
|
72
|
-
PlatformToProviderMessages,
|
|
73
|
-
ProviderToPlatformMessages,
|
|
74
|
-
SharedQueueToClientMessages
|
|
75
|
-
} from "@trigger.dev/core/v3";
|
|
76
|
-
import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
|
|
77
|
-
import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
|
|
78
|
-
var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
|
79
|
-
var MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
|
80
|
-
var PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
|
|
81
|
-
var PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
|
|
82
|
-
var PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
|
|
83
|
-
var SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
|
|
84
|
-
var logger = new SimpleLogger(`[${MACHINE_NAME}]`);
|
|
85
|
-
|
|
86
68
|
// src/workers/prod/entry-point.ts
|
|
87
69
|
import { readFile } from "node:fs/promises";
|
|
88
70
|
import { createServer } from "node:http";
|
|
@@ -171,6 +153,7 @@ var ProdBackgroundWorker = class {
|
|
|
171
153
|
attemptCreatedNotification = Evt.create();
|
|
172
154
|
_onClose = new Evt();
|
|
173
155
|
tasks = [];
|
|
156
|
+
stderr = [];
|
|
174
157
|
_taskRunProcess;
|
|
175
158
|
_taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
|
|
176
159
|
_closed = false;
|
|
@@ -234,6 +217,20 @@ var ProdBackgroundWorker = class {
|
|
|
234
217
|
child.kill();
|
|
235
218
|
reject(new Error("Worker timed out"));
|
|
236
219
|
}, 1e4);
|
|
220
|
+
child.stdout?.on("data", (data) => {
|
|
221
|
+
console.log(data.toString());
|
|
222
|
+
});
|
|
223
|
+
child.stderr?.on("data", (data) => {
|
|
224
|
+
console.error(data.toString());
|
|
225
|
+
this.stderr.push(data.toString());
|
|
226
|
+
});
|
|
227
|
+
child.on("exit", (code) => {
|
|
228
|
+
if (!resolved) {
|
|
229
|
+
clearTimeout(timeout);
|
|
230
|
+
resolved = true;
|
|
231
|
+
reject(new Error(`Worker exited with code ${code}`));
|
|
232
|
+
}
|
|
233
|
+
});
|
|
237
234
|
new ZodIpcConnection({
|
|
238
235
|
listenSchema: ProdChildToWorkerMessages,
|
|
239
236
|
emitSchema: ProdWorkerToChildMessages,
|
|
@@ -265,19 +262,6 @@ var ProdBackgroundWorker = class {
|
|
|
265
262
|
}
|
|
266
263
|
}
|
|
267
264
|
});
|
|
268
|
-
child.stdout?.on("data", (data) => {
|
|
269
|
-
console.log(data.toString());
|
|
270
|
-
});
|
|
271
|
-
child.stderr?.on("data", (data) => {
|
|
272
|
-
console.error(data.toString());
|
|
273
|
-
});
|
|
274
|
-
child.on("exit", (code) => {
|
|
275
|
-
if (!resolved) {
|
|
276
|
-
clearTimeout(timeout);
|
|
277
|
-
resolved = true;
|
|
278
|
-
reject(new Error(`Worker exited with code ${code}`));
|
|
279
|
-
}
|
|
280
|
-
});
|
|
281
265
|
});
|
|
282
266
|
this._initialized = true;
|
|
283
267
|
}
|
|
@@ -702,7 +686,10 @@ var TaskRunProcess = class {
|
|
|
702
686
|
const killParentProcess = kill && !killChildProcess;
|
|
703
687
|
console.log("Cleaning up task run process", {
|
|
704
688
|
killChildProcess,
|
|
705
|
-
killParentProcess
|
|
689
|
+
killParentProcess,
|
|
690
|
+
ipc: this._ipc,
|
|
691
|
+
childPid: this._childPid,
|
|
692
|
+
realChildPid: this._child?.pid
|
|
706
693
|
});
|
|
707
694
|
await this._ipc?.sendWithAck(
|
|
708
695
|
"CLEANUP",
|
|
@@ -818,13 +805,13 @@ var TaskRunProcess = class {
|
|
|
818
805
|
|
|
819
806
|
// src/workers/prod/entry-point.ts
|
|
820
807
|
import { setTimeout as setTimeout2 } from "node:timers/promises";
|
|
821
|
-
var
|
|
808
|
+
var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
|
822
809
|
var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
|
823
810
|
var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
|
|
824
|
-
var
|
|
811
|
+
var MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
|
825
812
|
var POD_NAME = process.env.POD_NAME || "some-pod";
|
|
826
813
|
var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
|
|
827
|
-
var
|
|
814
|
+
var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
|
|
828
815
|
var ProdWorker = class {
|
|
829
816
|
constructor(port, host = "0.0.0.0") {
|
|
830
817
|
this.host = host;
|
|
@@ -854,24 +841,24 @@ var ProdWorker = class {
|
|
|
854
841
|
#httpServer;
|
|
855
842
|
#coordinatorSocket;
|
|
856
843
|
async #handleSignal(signal) {
|
|
857
|
-
|
|
844
|
+
logger.log("Received signal", { signal });
|
|
858
845
|
if (signal === "SIGTERM") {
|
|
859
846
|
let gracefulExitTimeoutElapsed = false;
|
|
860
847
|
if (this.executing) {
|
|
861
848
|
const terminationGracePeriodSeconds = 60 * 60;
|
|
862
|
-
|
|
849
|
+
logger.log("Waiting for attempt to complete before exiting", {
|
|
863
850
|
terminationGracePeriodSeconds
|
|
864
851
|
});
|
|
865
852
|
await setTimeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
|
|
866
853
|
gracefulExitTimeoutElapsed = true;
|
|
867
|
-
|
|
854
|
+
logger.log("Termination timeout reached, exiting gracefully.");
|
|
868
855
|
} else {
|
|
869
|
-
|
|
856
|
+
logger.log("Not executing, exiting immediately.");
|
|
870
857
|
}
|
|
871
858
|
await this.#exitGracefully(gracefulExitTimeoutElapsed);
|
|
872
859
|
return;
|
|
873
860
|
}
|
|
874
|
-
|
|
861
|
+
logger.log("Unhandled signal", { signal });
|
|
875
862
|
}
|
|
876
863
|
async #exitGracefully(gracefulExitTimeoutElapsed = false) {
|
|
877
864
|
await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
|
|
@@ -894,7 +881,7 @@ var ProdWorker = class {
|
|
|
894
881
|
"\n",
|
|
895
882
|
""
|
|
896
883
|
);
|
|
897
|
-
|
|
884
|
+
logger.log("reconnecting", {
|
|
898
885
|
coordinatorHost: {
|
|
899
886
|
fromEnv: COORDINATOR_HOST,
|
|
900
887
|
fromVolume: coordinatorHost,
|
|
@@ -903,7 +890,7 @@ var ProdWorker = class {
|
|
|
903
890
|
});
|
|
904
891
|
}
|
|
905
892
|
} catch (error) {
|
|
906
|
-
|
|
893
|
+
logger.error("taskinfo read error during reconnect", {
|
|
907
894
|
error: error instanceof Error ? error.message : error
|
|
908
895
|
});
|
|
909
896
|
} finally {
|
|
@@ -932,7 +919,7 @@ var ProdWorker = class {
|
|
|
932
919
|
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
|
933
920
|
});
|
|
934
921
|
backgroundWorker.onCancelCheckpoint.attach(async (message) => {
|
|
935
|
-
|
|
922
|
+
logger.log("onCancelCheckpoint", { message });
|
|
936
923
|
const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
|
|
937
924
|
"CANCEL_CHECKPOINT",
|
|
938
925
|
{
|
|
@@ -940,7 +927,7 @@ var ProdWorker = class {
|
|
|
940
927
|
reason: message.reason
|
|
941
928
|
}
|
|
942
929
|
);
|
|
943
|
-
|
|
930
|
+
logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
|
|
944
931
|
if (checkpointCanceled) {
|
|
945
932
|
if (message.reason === "WAIT_FOR_DURATION") {
|
|
946
933
|
this.paused = false;
|
|
@@ -951,7 +938,7 @@ var ProdWorker = class {
|
|
|
951
938
|
backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
|
|
952
939
|
});
|
|
953
940
|
backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
|
|
954
|
-
|
|
941
|
+
logger.log("onCreateTaskRunAttempt()", { message });
|
|
955
942
|
const createAttempt = await this.#coordinatorSocket.socket.emitWithAck(
|
|
956
943
|
"CREATE_TASK_RUN_ATTEMPT",
|
|
957
944
|
{
|
|
@@ -979,7 +966,7 @@ var ProdWorker = class {
|
|
|
979
966
|
});
|
|
980
967
|
backgroundWorker.onWaitForDuration.attach(async (message) => {
|
|
981
968
|
if (!this.attemptFriendlyId) {
|
|
982
|
-
|
|
969
|
+
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
983
970
|
this.#emitUnrecoverableError(
|
|
984
971
|
"NoAttemptId",
|
|
985
972
|
"Attempt ID not set before waiting for duration"
|
|
@@ -997,7 +984,7 @@ var ProdWorker = class {
|
|
|
997
984
|
});
|
|
998
985
|
backgroundWorker.onWaitForTask.attach(async (message) => {
|
|
999
986
|
if (!this.attemptFriendlyId) {
|
|
1000
|
-
|
|
987
|
+
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1001
988
|
this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for task");
|
|
1002
989
|
return;
|
|
1003
990
|
}
|
|
@@ -1012,7 +999,7 @@ var ProdWorker = class {
|
|
|
1012
999
|
});
|
|
1013
1000
|
backgroundWorker.onWaitForBatch.attach(async (message) => {
|
|
1014
1001
|
if (!this.attemptFriendlyId) {
|
|
1015
|
-
|
|
1002
|
+
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1016
1003
|
this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for batch");
|
|
1017
1004
|
return;
|
|
1018
1005
|
}
|
|
@@ -1028,7 +1015,7 @@ var ProdWorker = class {
|
|
|
1028
1015
|
return backgroundWorker;
|
|
1029
1016
|
}
|
|
1030
1017
|
async #prepareForWait(reason, willCheckpointAndRestore) {
|
|
1031
|
-
|
|
1018
|
+
logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
|
|
1032
1019
|
this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
|
|
1033
1020
|
if (willCheckpointAndRestore) {
|
|
1034
1021
|
this.paused = true;
|
|
@@ -1040,10 +1027,10 @@ var ProdWorker = class {
|
|
|
1040
1027
|
}
|
|
1041
1028
|
}
|
|
1042
1029
|
async #prepareForRetry(willCheckpointAndRestore, shouldExit) {
|
|
1043
|
-
|
|
1030
|
+
logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
|
|
1044
1031
|
if (shouldExit) {
|
|
1045
1032
|
if (willCheckpointAndRestore) {
|
|
1046
|
-
|
|
1033
|
+
logger.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
|
|
1047
1034
|
}
|
|
1048
1035
|
await this.#exitGracefully();
|
|
1049
1036
|
return;
|
|
@@ -1082,7 +1069,7 @@ var ProdWorker = class {
|
|
|
1082
1069
|
// FIXME: If the the worker can't connect for a while, this runs MANY times - it should only run once
|
|
1083
1070
|
#createCoordinatorSocket(host) {
|
|
1084
1071
|
const extraHeaders = this.#returnValidatedExtraHeaders({
|
|
1085
|
-
"x-machine-name":
|
|
1072
|
+
"x-machine-name": MACHINE_NAME,
|
|
1086
1073
|
"x-pod-name": POD_NAME,
|
|
1087
1074
|
"x-trigger-content-hash": this.contentHash,
|
|
1088
1075
|
"x-trigger-project-ref": this.projectRef,
|
|
@@ -1094,9 +1081,9 @@ var ProdWorker = class {
|
|
|
1094
1081
|
if (this.attemptFriendlyId) {
|
|
1095
1082
|
extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
|
|
1096
1083
|
}
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
const coordinatorConnection = new
|
|
1084
|
+
logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
|
|
1085
|
+
logger.debug(`connecting with extra headers`, { extraHeaders });
|
|
1086
|
+
const coordinatorConnection = new ZodSocketConnection({
|
|
1100
1087
|
namespace: "prod-worker",
|
|
1101
1088
|
host,
|
|
1102
1089
|
port: COORDINATOR_PORT,
|
|
@@ -1106,21 +1093,21 @@ var ProdWorker = class {
|
|
|
1106
1093
|
handlers: {
|
|
1107
1094
|
RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
|
|
1108
1095
|
if (!this.paused) {
|
|
1109
|
-
|
|
1096
|
+
logger.error("Failed to resume after dependency: Worker not paused");
|
|
1110
1097
|
return;
|
|
1111
1098
|
}
|
|
1112
1099
|
if (completions.length === 0) {
|
|
1113
|
-
|
|
1100
|
+
logger.error("Failed to resume after dependency: No completions");
|
|
1114
1101
|
return;
|
|
1115
1102
|
}
|
|
1116
1103
|
if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
|
|
1117
|
-
|
|
1104
|
+
logger.error("Failed to resume after dependency: Invalid next resume", {
|
|
1118
1105
|
nextResumeAfter: this.nextResumeAfter
|
|
1119
1106
|
});
|
|
1120
1107
|
return;
|
|
1121
1108
|
}
|
|
1122
1109
|
if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
|
|
1123
|
-
|
|
1110
|
+
logger.error(
|
|
1124
1111
|
"Failed to resume after dependency: Waiting for single task but got multiple completions",
|
|
1125
1112
|
{
|
|
1126
1113
|
completions
|
|
@@ -1140,13 +1127,13 @@ var ProdWorker = class {
|
|
|
1140
1127
|
},
|
|
1141
1128
|
RESUME_AFTER_DURATION: async (message) => {
|
|
1142
1129
|
if (!this.paused) {
|
|
1143
|
-
|
|
1130
|
+
logger.error("worker not paused", {
|
|
1144
1131
|
attemptId: message.attemptId
|
|
1145
1132
|
});
|
|
1146
1133
|
return;
|
|
1147
1134
|
}
|
|
1148
1135
|
if (this.nextResumeAfter !== "WAIT_FOR_DURATION") {
|
|
1149
|
-
|
|
1136
|
+
logger.error("not waiting to resume after duration", {
|
|
1150
1137
|
nextResumeAfter: this.nextResumeAfter
|
|
1151
1138
|
});
|
|
1152
1139
|
return;
|
|
@@ -1155,42 +1142,42 @@ var ProdWorker = class {
|
|
|
1155
1142
|
},
|
|
1156
1143
|
EXECUTE_TASK_RUN: async ({ executionPayload }) => {
|
|
1157
1144
|
if (this.executing) {
|
|
1158
|
-
|
|
1145
|
+
logger.error("dropping execute request, already executing");
|
|
1159
1146
|
return;
|
|
1160
1147
|
}
|
|
1161
1148
|
if (this.completed.has(executionPayload.execution.attempt.id)) {
|
|
1162
|
-
|
|
1149
|
+
logger.error("dropping execute request, already completed");
|
|
1163
1150
|
return;
|
|
1164
1151
|
}
|
|
1165
1152
|
this.executing = true;
|
|
1166
1153
|
this.attemptFriendlyId = executionPayload.execution.attempt.id;
|
|
1167
1154
|
const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
|
|
1168
|
-
|
|
1155
|
+
logger.log("completed", completion);
|
|
1169
1156
|
this.completed.add(executionPayload.execution.attempt.id);
|
|
1170
1157
|
const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
|
1171
1158
|
version: "v1",
|
|
1172
1159
|
execution: executionPayload.execution,
|
|
1173
1160
|
completion
|
|
1174
1161
|
});
|
|
1175
|
-
|
|
1162
|
+
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1176
1163
|
this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
|
1177
1164
|
},
|
|
1178
1165
|
EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
|
|
1179
1166
|
if (this.executing) {
|
|
1180
|
-
|
|
1167
|
+
logger.error("dropping execute request, already executing");
|
|
1181
1168
|
return;
|
|
1182
1169
|
}
|
|
1183
1170
|
this.executing = true;
|
|
1184
1171
|
try {
|
|
1185
1172
|
const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
|
|
1186
|
-
|
|
1173
|
+
logger.log("completed", completion);
|
|
1187
1174
|
this.completed.add(execution.attempt.id);
|
|
1188
1175
|
const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
|
1189
1176
|
version: "v1",
|
|
1190
1177
|
execution,
|
|
1191
1178
|
completion
|
|
1192
1179
|
});
|
|
1193
|
-
|
|
1180
|
+
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1194
1181
|
this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
|
1195
1182
|
} catch (error) {
|
|
1196
1183
|
const completion = {
|
|
@@ -1217,15 +1204,15 @@ var ProdWorker = class {
|
|
|
1217
1204
|
},
|
|
1218
1205
|
REQUEST_ATTEMPT_CANCELLATION: async (message) => {
|
|
1219
1206
|
if (!this.executing) {
|
|
1220
|
-
|
|
1207
|
+
logger.log("dropping cancel request, not executing", { status: this.#status });
|
|
1221
1208
|
return;
|
|
1222
1209
|
}
|
|
1223
|
-
|
|
1210
|
+
logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
|
|
1224
1211
|
await this.#backgroundWorker.cancelAttempt(message.attemptId);
|
|
1225
1212
|
},
|
|
1226
1213
|
REQUEST_EXIT: async (message) => {
|
|
1227
1214
|
if (message.version === "v2" && message.delayInMs) {
|
|
1228
|
-
|
|
1215
|
+
logger.log("exit requested with delay", { delayInMs: message.delayInMs });
|
|
1229
1216
|
await setTimeout2(message.delayInMs);
|
|
1230
1217
|
}
|
|
1231
1218
|
this.#coordinatorSocket.close();
|
|
@@ -1242,15 +1229,15 @@ var ProdWorker = class {
|
|
|
1242
1229
|
});
|
|
1243
1230
|
}
|
|
1244
1231
|
},
|
|
1245
|
-
onConnection: async (socket, handler, sender,
|
|
1246
|
-
|
|
1232
|
+
onConnection: async (socket, handler, sender, logger2) => {
|
|
1233
|
+
logger2.log("connected to coordinator", { status: this.#status });
|
|
1247
1234
|
if (this.waitForPostStart) {
|
|
1248
|
-
|
|
1235
|
+
logger2.log("skip connection handler, waiting for post start hook");
|
|
1249
1236
|
return;
|
|
1250
1237
|
}
|
|
1251
1238
|
if (this.paused) {
|
|
1252
1239
|
if (!this.nextResumeAfter) {
|
|
1253
|
-
|
|
1240
|
+
logger2.error("Missing next resume reason", { status: this.#status });
|
|
1254
1241
|
this.#emitUnrecoverableError(
|
|
1255
1242
|
"NoNextResume",
|
|
1256
1243
|
"Next resume reason not set while resuming from paused state"
|
|
@@ -1258,7 +1245,7 @@ var ProdWorker = class {
|
|
|
1258
1245
|
return;
|
|
1259
1246
|
}
|
|
1260
1247
|
if (!this.attemptFriendlyId) {
|
|
1261
|
-
|
|
1248
|
+
logger2.error("Missing friendly ID", { status: this.#status });
|
|
1262
1249
|
this.#emitUnrecoverableError(
|
|
1263
1250
|
"NoAttemptId",
|
|
1264
1251
|
"Attempt ID not set while resuming from paused state"
|
|
@@ -1282,15 +1269,16 @@ var ProdWorker = class {
|
|
|
1282
1269
|
supportsLazyAttempts: true
|
|
1283
1270
|
});
|
|
1284
1271
|
if (success) {
|
|
1285
|
-
|
|
1272
|
+
logger2.info("indexing done, shutting down..");
|
|
1286
1273
|
process.exit(0);
|
|
1287
1274
|
} else {
|
|
1288
|
-
|
|
1275
|
+
logger2.info("indexing failure, shutting down..");
|
|
1289
1276
|
process.exit(1);
|
|
1290
1277
|
}
|
|
1291
1278
|
} catch (e) {
|
|
1279
|
+
const stderr = this.#backgroundWorker.stderr.join("\n");
|
|
1292
1280
|
if (e instanceof TaskMetadataParseError) {
|
|
1293
|
-
|
|
1281
|
+
logger2.error("tasks metadata parse error", {
|
|
1294
1282
|
zodIssues: e.zodIssues,
|
|
1295
1283
|
tasks: e.tasks
|
|
1296
1284
|
});
|
|
@@ -1300,16 +1288,18 @@ var ProdWorker = class {
|
|
|
1300
1288
|
error: {
|
|
1301
1289
|
name: "TaskMetadataParseError",
|
|
1302
1290
|
message: "There was an error parsing the task metadata",
|
|
1303
|
-
stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks })
|
|
1291
|
+
stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
|
|
1292
|
+
stderr
|
|
1304
1293
|
}
|
|
1305
1294
|
});
|
|
1306
1295
|
} else if (e instanceof UncaughtExceptionError) {
|
|
1307
1296
|
const error = {
|
|
1308
1297
|
name: e.originalError.name,
|
|
1309
1298
|
message: e.originalError.message,
|
|
1310
|
-
stack: e.originalError.stack
|
|
1299
|
+
stack: e.originalError.stack,
|
|
1300
|
+
stderr
|
|
1311
1301
|
};
|
|
1312
|
-
|
|
1302
|
+
logger2.error("uncaught exception", { originalError: error });
|
|
1313
1303
|
socket.emit("INDEXING_FAILED", {
|
|
1314
1304
|
version: "v1",
|
|
1315
1305
|
deploymentId: this.deploymentId,
|
|
@@ -1319,32 +1309,35 @@ var ProdWorker = class {
|
|
|
1319
1309
|
const error = {
|
|
1320
1310
|
name: e.name,
|
|
1321
1311
|
message: e.message,
|
|
1322
|
-
stack: e.stack
|
|
1312
|
+
stack: e.stack,
|
|
1313
|
+
stderr
|
|
1323
1314
|
};
|
|
1324
|
-
|
|
1315
|
+
logger2.error("error", { error });
|
|
1325
1316
|
socket.emit("INDEXING_FAILED", {
|
|
1326
1317
|
version: "v1",
|
|
1327
1318
|
deploymentId: this.deploymentId,
|
|
1328
1319
|
error
|
|
1329
1320
|
});
|
|
1330
1321
|
} else if (typeof e === "string") {
|
|
1331
|
-
|
|
1322
|
+
logger2.error("string error", { error: { message: e } });
|
|
1332
1323
|
socket.emit("INDEXING_FAILED", {
|
|
1333
1324
|
version: "v1",
|
|
1334
1325
|
deploymentId: this.deploymentId,
|
|
1335
1326
|
error: {
|
|
1336
1327
|
name: "Error",
|
|
1337
|
-
message: e
|
|
1328
|
+
message: e,
|
|
1329
|
+
stderr
|
|
1338
1330
|
}
|
|
1339
1331
|
});
|
|
1340
1332
|
} else {
|
|
1341
|
-
|
|
1333
|
+
logger2.error("unknown error", { error: e });
|
|
1342
1334
|
socket.emit("INDEXING_FAILED", {
|
|
1343
1335
|
version: "v1",
|
|
1344
1336
|
deploymentId: this.deploymentId,
|
|
1345
1337
|
error: {
|
|
1346
1338
|
name: "Error",
|
|
1347
|
-
message: "Unknown error"
|
|
1339
|
+
message: "Unknown error",
|
|
1340
|
+
stderr
|
|
1348
1341
|
}
|
|
1349
1342
|
});
|
|
1350
1343
|
}
|
|
@@ -1361,8 +1354,8 @@ var ProdWorker = class {
|
|
|
1361
1354
|
totalCompletions: this.completed.size
|
|
1362
1355
|
});
|
|
1363
1356
|
},
|
|
1364
|
-
onError: async (socket, err,
|
|
1365
|
-
|
|
1357
|
+
onError: async (socket, err, logger2) => {
|
|
1358
|
+
logger2.error("onError", {
|
|
1366
1359
|
error: {
|
|
1367
1360
|
name: err.name,
|
|
1368
1361
|
message: err.message
|
|
@@ -1370,14 +1363,14 @@ var ProdWorker = class {
|
|
|
1370
1363
|
});
|
|
1371
1364
|
await this.#reconnect();
|
|
1372
1365
|
},
|
|
1373
|
-
onDisconnect: async (socket, reason, description,
|
|
1366
|
+
onDisconnect: async (socket, reason, description, logger2) => {
|
|
1374
1367
|
}
|
|
1375
1368
|
});
|
|
1376
1369
|
return coordinatorConnection;
|
|
1377
1370
|
}
|
|
1378
1371
|
#createHttpServer() {
|
|
1379
1372
|
const httpServer = createServer(async (req, res) => {
|
|
1380
|
-
|
|
1373
|
+
logger.log(`[${req.method}]`, req.url);
|
|
1381
1374
|
const reply = new HttpReply(res);
|
|
1382
1375
|
try {
|
|
1383
1376
|
const url = new URL(req.url ?? "", `http://${req.headers.host}`);
|
|
@@ -1410,7 +1403,7 @@ var ProdWorker = class {
|
|
|
1410
1403
|
case "/preStop": {
|
|
1411
1404
|
const cause = PreStopCauses.safeParse(url.searchParams.get("cause"));
|
|
1412
1405
|
if (!cause.success) {
|
|
1413
|
-
|
|
1406
|
+
logger.error("Failed to parse cause", { cause });
|
|
1414
1407
|
return reply.text("Failed to parse cause", 400);
|
|
1415
1408
|
}
|
|
1416
1409
|
switch (cause.data) {
|
|
@@ -1418,7 +1411,7 @@ var ProdWorker = class {
|
|
|
1418
1411
|
break;
|
|
1419
1412
|
}
|
|
1420
1413
|
default: {
|
|
1421
|
-
|
|
1414
|
+
logger.error("Unhandled cause", { cause: cause.data });
|
|
1422
1415
|
break;
|
|
1423
1416
|
}
|
|
1424
1417
|
}
|
|
@@ -1427,7 +1420,7 @@ var ProdWorker = class {
|
|
|
1427
1420
|
case "/postStart": {
|
|
1428
1421
|
const cause = PostStartCauses.safeParse(url.searchParams.get("cause"));
|
|
1429
1422
|
if (!cause.success) {
|
|
1430
|
-
|
|
1423
|
+
logger.error("Failed to parse cause", { cause });
|
|
1431
1424
|
return reply.text("Failed to parse cause", 400);
|
|
1432
1425
|
}
|
|
1433
1426
|
switch (cause.data) {
|
|
@@ -1442,7 +1435,7 @@ var ProdWorker = class {
|
|
|
1442
1435
|
break;
|
|
1443
1436
|
}
|
|
1444
1437
|
default: {
|
|
1445
|
-
|
|
1438
|
+
logger.error("Unhandled cause", { cause: cause.data });
|
|
1446
1439
|
break;
|
|
1447
1440
|
}
|
|
1448
1441
|
}
|
|
@@ -1453,7 +1446,7 @@ var ProdWorker = class {
|
|
|
1453
1446
|
}
|
|
1454
1447
|
}
|
|
1455
1448
|
} catch (error) {
|
|
1456
|
-
|
|
1449
|
+
logger.error("HTTP server error", { error });
|
|
1457
1450
|
reply.empty(500);
|
|
1458
1451
|
}
|
|
1459
1452
|
});
|
|
@@ -1461,13 +1454,13 @@ var ProdWorker = class {
|
|
|
1461
1454
|
socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
1462
1455
|
});
|
|
1463
1456
|
httpServer.on("listening", () => {
|
|
1464
|
-
|
|
1457
|
+
logger.log("http server listening on port", this.#httpPort);
|
|
1465
1458
|
});
|
|
1466
1459
|
httpServer.on("error", async (error) => {
|
|
1467
1460
|
if (error.code != "EADDRINUSE") {
|
|
1468
1461
|
return;
|
|
1469
1462
|
}
|
|
1470
|
-
|
|
1463
|
+
logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
|
|
1471
1464
|
this.#httpPort = getRandomPortNumber();
|
|
1472
1465
|
await setTimeout2(100);
|
|
1473
1466
|
this.start();
|
|
@@ -1530,7 +1523,7 @@ var ProdWorker = class {
|
|
|
1530
1523
|
this.#httpServer.listen(this.#httpPort, this.host);
|
|
1531
1524
|
}
|
|
1532
1525
|
};
|
|
1533
|
-
var prodWorker = new ProdWorker(
|
|
1526
|
+
var prodWorker = new ProdWorker(HTTP_SERVER_PORT);
|
|
1534
1527
|
prodWorker.start();
|
|
1535
1528
|
function gatherProcessEnv() {
|
|
1536
1529
|
const env = {
|