trigger.dev 0.0.0-v3-prerelease-20240801102736 → 0.0.0-v3-prerelease-20240912191953
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/apiClient.d.ts +190 -0
- package/dist/esm/apiClient.js +239 -0
- package/dist/esm/apiClient.js.map +1 -0
- package/dist/esm/build/buildWorker.d.ts +68 -0
- package/dist/esm/build/buildWorker.js +159 -0
- package/dist/esm/build/buildWorker.js.map +1 -0
- package/dist/esm/build/bundle.d.ts +36 -0
- package/dist/esm/build/bundle.js +197 -0
- package/dist/esm/build/bundle.js.map +1 -0
- package/dist/esm/build/extensions.d.ts +15 -0
- package/dist/esm/build/extensions.js +158 -0
- package/dist/esm/build/extensions.js.map +1 -0
- package/dist/esm/build/externals.d.ts +15 -0
- package/dist/esm/build/externals.js +285 -0
- package/dist/esm/build/externals.js.map +1 -0
- package/dist/esm/build/manifests.d.ts +2 -0
- package/dist/esm/build/manifests.js +22 -0
- package/dist/esm/build/manifests.js.map +1 -0
- package/dist/esm/build/packageModules.d.ts +18 -0
- package/dist/esm/build/packageModules.js +84 -0
- package/dist/esm/build/packageModules.js.map +1 -0
- package/dist/esm/build/plugins.d.ts +6 -0
- package/dist/esm/build/plugins.js +67 -0
- package/dist/esm/build/plugins.js.map +1 -0
- package/dist/esm/build/resolveModule.d.ts +1 -0
- package/dist/esm/build/resolveModule.js +9 -0
- package/dist/esm/build/resolveModule.js.map +1 -0
- package/dist/esm/cli/common.d.ts +30 -0
- package/dist/esm/cli/common.js +85 -0
- package/dist/esm/cli/common.js.map +1 -0
- package/dist/esm/cli/index.d.ts +2 -0
- package/dist/esm/cli/index.js +27 -0
- package/dist/esm/cli/index.js.map +1 -0
- package/dist/esm/commands/deploy.d.ts +5 -0
- package/dist/esm/commands/deploy.js +402 -0
- package/dist/esm/commands/deploy.js.map +1 -0
- package/dist/esm/commands/dev.d.ts +37 -0
- package/dist/esm/commands/dev.js +128 -0
- package/dist/esm/commands/dev.js.map +1 -0
- package/dist/esm/commands/init.d.ts +3 -0
- package/dist/esm/commands/init.js +417 -0
- package/dist/esm/commands/init.js.map +1 -0
- package/dist/esm/commands/list-profiles.d.ts +23 -0
- package/dist/esm/commands/list-profiles.js +40 -0
- package/dist/esm/commands/list-profiles.js.map +1 -0
- package/dist/esm/commands/login.d.ts +28 -0
- package/dist/esm/commands/login.js +275 -0
- package/dist/esm/commands/login.js.map +1 -0
- package/dist/esm/commands/logout.d.ts +23 -0
- package/dist/esm/commands/logout.js +28 -0
- package/dist/esm/commands/logout.js.map +1 -0
- package/dist/esm/commands/update.d.ts +24 -0
- package/dist/esm/commands/update.js +233 -0
- package/dist/esm/commands/update.js.map +1 -0
- package/dist/esm/commands/whoami.d.ts +34 -0
- package/dist/esm/commands/whoami.js +70 -0
- package/dist/esm/commands/whoami.js.map +1 -0
- package/dist/esm/config.d.ts +22 -0
- package/dist/esm/config.js +204 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/consts.d.ts +4 -0
- package/dist/esm/consts.js +5 -0
- package/dist/esm/consts.js.map +1 -0
- package/dist/esm/deploy/buildImage.d.ts +69 -0
- package/dist/esm/deploy/buildImage.js +472 -0
- package/dist/esm/deploy/buildImage.js.map +1 -0
- package/dist/esm/deploy/logs.d.ts +19 -0
- package/dist/esm/deploy/logs.js +96 -0
- package/dist/esm/deploy/logs.js.map +1 -0
- package/dist/esm/dev/backgroundWorker.d.ts +114 -0
- package/dist/esm/dev/backgroundWorker.js +412 -0
- package/dist/esm/dev/backgroundWorker.js.map +1 -0
- package/dist/esm/dev/devOutput.d.ts +9 -0
- package/dist/esm/dev/devOutput.js +137 -0
- package/dist/esm/dev/devOutput.js.map +1 -0
- package/dist/esm/dev/devSession.d.ts +17 -0
- package/dist/esm/dev/devSession.js +156 -0
- package/dist/esm/dev/devSession.js.map +1 -0
- package/dist/esm/dev/workerRuntime.d.ts +16 -0
- package/dist/esm/dev/workerRuntime.js +258 -0
- package/dist/esm/dev/workerRuntime.js.map +1 -0
- package/dist/esm/entryPoints/deploy-index-controller.d.ts +1 -0
- package/dist/esm/entryPoints/deploy-index-controller.js +85 -0
- package/dist/esm/entryPoints/deploy-index-controller.js.map +1 -0
- package/dist/esm/entryPoints/deploy-index-worker.d.ts +1 -0
- package/dist/esm/entryPoints/deploy-index-worker.js +104 -0
- package/dist/esm/entryPoints/deploy-index-worker.js.map +1 -0
- package/dist/esm/entryPoints/deploy-run-controller.d.ts +1 -0
- package/dist/esm/entryPoints/deploy-run-controller.js +996 -0
- package/dist/esm/entryPoints/deploy-run-controller.js.map +1 -0
- package/dist/esm/entryPoints/deploy-run-worker.d.ts +1 -0
- package/dist/esm/entryPoints/deploy-run-worker.js +297 -0
- package/dist/esm/entryPoints/deploy-run-worker.js.map +1 -0
- package/dist/esm/entryPoints/dev-index-worker.d.ts +1 -0
- package/dist/esm/entryPoints/dev-index-worker.js +104 -0
- package/dist/esm/entryPoints/dev-index-worker.js.map +1 -0
- package/dist/esm/entryPoints/dev-run-worker.d.ts +1 -0
- package/dist/esm/entryPoints/dev-run-worker.js +254 -0
- package/dist/esm/entryPoints/dev-run-worker.js.map +1 -0
- package/dist/esm/entryPoints/loader.d.ts +1 -0
- package/dist/esm/entryPoints/loader.js +16 -0
- package/dist/esm/entryPoints/loader.js.map +1 -0
- package/dist/esm/executions/taskRunProcess.d.ts +115 -0
- package/dist/esm/executions/taskRunProcess.js +254 -0
- package/dist/esm/executions/taskRunProcess.js.map +1 -0
- package/dist/esm/imports/magicast.d.ts +2 -0
- package/dist/esm/imports/magicast.js +5 -0
- package/dist/esm/imports/magicast.js.map +1 -0
- package/dist/esm/imports/xdg-app-paths.d.ts +2 -0
- package/dist/esm/imports/xdg-app-paths.js +3 -0
- package/dist/esm/imports/xdg-app-paths.js.map +1 -0
- package/dist/{index.d.ts → esm/index.d.ts} +1 -0
- package/dist/esm/index.js +17 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/indexing/indexWorkerManifest.d.ts +85 -0
- package/dist/esm/indexing/indexWorkerManifest.js +76 -0
- package/dist/esm/indexing/indexWorkerManifest.js.map +1 -0
- package/dist/esm/indexing/registerTasks.d.ts +2 -0
- package/dist/esm/indexing/registerTasks.js +62 -0
- package/dist/esm/indexing/registerTasks.js.map +1 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/runtimes/bun.d.ts +2 -0
- package/dist/esm/runtimes/bun.js +11 -0
- package/dist/esm/runtimes/bun.js.map +1 -0
- package/dist/esm/shims/esm.d.ts +1 -0
- package/dist/esm/shims/esm.js +9 -0
- package/dist/esm/shims/esm.js.map +1 -0
- package/dist/esm/sourceDir.d.ts +1 -0
- package/dist/esm/sourceDir.js +4 -0
- package/dist/esm/sourceDir.js.map +1 -0
- package/dist/esm/telemetry/tracing.d.ts +3 -0
- package/dist/esm/telemetry/tracing.js +58 -0
- package/dist/esm/telemetry/tracing.js.map +1 -0
- package/dist/esm/types.d.ts +6 -0
- package/dist/esm/types.js +2 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/utilities/assertExhaustive.d.ts +1 -0
- package/dist/esm/utilities/assertExhaustive.js +4 -0
- package/dist/esm/utilities/assertExhaustive.js.map +1 -0
- package/dist/esm/utilities/buildManifest.d.ts +2 -0
- package/dist/esm/utilities/buildManifest.js +9 -0
- package/dist/esm/utilities/buildManifest.js.map +1 -0
- package/dist/esm/utilities/cliOutput.d.ts +20 -0
- package/dist/esm/utilities/cliOutput.js +88 -0
- package/dist/esm/utilities/cliOutput.js.map +1 -0
- package/dist/esm/utilities/configFiles.d.ts +29 -0
- package/dist/esm/utilities/configFiles.js +65 -0
- package/dist/esm/utilities/configFiles.js.map +1 -0
- package/dist/esm/utilities/createFileFromTemplate.d.ts +16 -0
- package/dist/esm/utilities/createFileFromTemplate.js +50 -0
- package/dist/esm/utilities/createFileFromTemplate.js.map +1 -0
- package/dist/esm/utilities/deployErrors.d.ts +19 -0
- package/dist/esm/utilities/deployErrors.js +117 -0
- package/dist/esm/utilities/deployErrors.js.map +1 -0
- package/dist/esm/utilities/dotEnv.d.ts +4 -0
- package/dist/esm/utilities/dotEnv.js +28 -0
- package/dist/esm/utilities/dotEnv.js.map +1 -0
- package/dist/esm/utilities/eventBus.d.ts +14 -0
- package/dist/esm/utilities/eventBus.js +3 -0
- package/dist/esm/utilities/eventBus.js.map +1 -0
- package/dist/esm/utilities/fileSystem.d.ts +12 -0
- package/dist/esm/utilities/fileSystem.js +81 -0
- package/dist/esm/utilities/fileSystem.js.map +1 -0
- package/dist/esm/utilities/getApiKeyType.d.ts +13 -0
- package/dist/esm/utilities/getApiKeyType.js +44 -0
- package/dist/esm/utilities/getApiKeyType.js.map +1 -0
- package/dist/esm/utilities/initialBanner.d.ts +4 -0
- package/dist/esm/utilities/initialBanner.js +82 -0
- package/dist/esm/utilities/initialBanner.js.map +1 -0
- package/dist/esm/utilities/keyValueBy.d.ts +8 -0
- package/dist/esm/utilities/keyValueBy.js +19 -0
- package/dist/esm/utilities/keyValueBy.js.map +1 -0
- package/dist/esm/utilities/links.d.ts +9 -0
- package/dist/esm/utilities/links.js +10 -0
- package/dist/esm/utilities/links.js.map +1 -0
- package/dist/esm/utilities/linux.d.ts +1 -0
- package/dist/esm/utilities/linux.js +24 -0
- package/dist/esm/utilities/linux.js.map +1 -0
- package/dist/esm/utilities/logger.d.ts +40 -0
- package/dist/esm/utilities/logger.js +109 -0
- package/dist/esm/utilities/logger.js.map +1 -0
- package/dist/esm/utilities/normalizeImportPath.d.ts +1 -0
- package/dist/esm/utilities/normalizeImportPath.js +5 -0
- package/dist/esm/utilities/normalizeImportPath.js.map +1 -0
- package/dist/esm/utilities/obfuscateApiKey.d.ts +1 -0
- package/dist/esm/utilities/obfuscateApiKey.js +5 -0
- package/dist/esm/utilities/obfuscateApiKey.js.map +1 -0
- package/dist/esm/utilities/parseNameAndPath.d.ts +2 -0
- package/dist/esm/utilities/parseNameAndPath.js +10 -0
- package/dist/esm/utilities/parseNameAndPath.js.map +1 -0
- package/dist/esm/utilities/resolveInternalFilePath.d.ts +1 -0
- package/dist/esm/utilities/resolveInternalFilePath.js +8 -0
- package/dist/esm/utilities/resolveInternalFilePath.js.map +1 -0
- package/dist/esm/utilities/runtimeCheck.d.ts +8 -0
- package/dist/esm/utilities/runtimeCheck.js +52 -0
- package/dist/esm/utilities/runtimeCheck.js.map +1 -0
- package/dist/esm/utilities/safeJsonParse.d.ts +1 -0
- package/dist/esm/utilities/safeJsonParse.js +12 -0
- package/dist/esm/utilities/safeJsonParse.js.map +1 -0
- package/dist/esm/utilities/sanitizeEnvVars.d.ts +3 -0
- package/dist/esm/utilities/sanitizeEnvVars.js +4 -0
- package/dist/esm/utilities/sanitizeEnvVars.js.map +1 -0
- package/dist/esm/utilities/session.d.ts +33 -0
- package/dist/esm/utilities/session.js +82 -0
- package/dist/esm/utilities/session.js.map +1 -0
- package/dist/esm/utilities/sourceFiles.d.ts +12 -0
- package/dist/esm/utilities/sourceFiles.js +76 -0
- package/dist/esm/utilities/sourceFiles.js.map +1 -0
- package/dist/esm/utilities/taskFiles.d.ts +6 -0
- package/dist/esm/utilities/taskFiles.js +69 -0
- package/dist/esm/utilities/taskFiles.js.map +1 -0
- package/dist/esm/utilities/tempDirectories.d.ts +16 -0
- package/dist/esm/utilities/tempDirectories.js +27 -0
- package/dist/esm/utilities/tempDirectories.js.map +1 -0
- package/dist/esm/utilities/windows.d.ts +7 -0
- package/dist/esm/utilities/windows.js +21 -0
- package/dist/esm/utilities/windows.js.map +1 -0
- package/dist/esm/version.d.ts +1 -0
- package/dist/esm/version.js +2 -0
- package/dist/esm/version.js.map +1 -0
- package/package.json +82 -78
- package/dist/Containerfile.prod +0 -58
- package/dist/index.js +0 -7780
- package/dist/index.js.map +0 -1
- package/dist/templates/examples/simple.ts.template +0 -14
- package/dist/templates/trigger.config.ts.template +0 -16
- package/dist/workers/dev/worker-facade.js +0 -192
- package/dist/workers/dev/worker-setup.js +0 -35
- package/dist/workers/prod/entry-point.js +0 -2163
- package/dist/workers/prod/worker-facade.js +0 -223
- package/dist/workers/prod/worker-setup.js +0 -24
|
@@ -1,2163 +0,0 @@
|
|
|
1
|
-
// src/workers/prod/entry-point.ts
|
|
2
|
-
import {
|
|
3
|
-
CoordinatorToProdWorkerMessages,
|
|
4
|
-
PostStartCauses,
|
|
5
|
-
PreStopCauses,
|
|
6
|
-
ProdWorkerToCoordinatorMessages,
|
|
7
|
-
TaskRunErrorCodes as TaskRunErrorCodes2
|
|
8
|
-
} from "@trigger.dev/core/v3";
|
|
9
|
-
import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
|
|
10
|
-
|
|
11
|
-
// ../core-apps/src/http.ts
|
|
12
|
-
var HttpReply = class {
|
|
13
|
-
constructor(response) {
|
|
14
|
-
this.response = response;
|
|
15
|
-
}
|
|
16
|
-
empty(status) {
|
|
17
|
-
return this.response.writeHead(status ?? 200).end();
|
|
18
|
-
}
|
|
19
|
-
text(text, status, contentType) {
|
|
20
|
-
return this.response.writeHead(status ?? 200, { "Content-Type": contentType || "text/plain" }).end(text.endsWith("\n") ? text : `${text}
|
|
21
|
-
`);
|
|
22
|
-
}
|
|
23
|
-
json(value, pretty) {
|
|
24
|
-
return this.text(
|
|
25
|
-
JSON.stringify(value, void 0, pretty ? 2 : void 0),
|
|
26
|
-
200,
|
|
27
|
-
"application/json"
|
|
28
|
-
);
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
function getRandomInteger(min, max) {
|
|
32
|
-
const intMin = Math.ceil(min);
|
|
33
|
-
const intMax = Math.floor(max);
|
|
34
|
-
return Math.floor(Math.random() * (intMax - intMin + 1)) + intMin;
|
|
35
|
-
}
|
|
36
|
-
function getRandomPortNumber() {
|
|
37
|
-
return getRandomInteger(8e3, 9999);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// ../core-apps/src/logger.ts
|
|
41
|
-
var SimpleLogger = class {
|
|
42
|
-
constructor(prefix) {
|
|
43
|
-
this.prefix = prefix;
|
|
44
|
-
}
|
|
45
|
-
#debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? "");
|
|
46
|
-
log(arg0, ...argN) {
|
|
47
|
-
console.log(...this.#getPrefixedArgs(arg0, ...argN));
|
|
48
|
-
return arg0;
|
|
49
|
-
}
|
|
50
|
-
debug(arg0, ...argN) {
|
|
51
|
-
if (!this.#debugEnabled) {
|
|
52
|
-
return arg0;
|
|
53
|
-
}
|
|
54
|
-
console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN));
|
|
55
|
-
return arg0;
|
|
56
|
-
}
|
|
57
|
-
error(arg0, ...argN) {
|
|
58
|
-
console.error(...this.#getPrefixedArgs(arg0, ...argN));
|
|
59
|
-
return arg0;
|
|
60
|
-
}
|
|
61
|
-
#getPrefixedArgs(...args) {
|
|
62
|
-
if (!this.prefix) {
|
|
63
|
-
return args;
|
|
64
|
-
}
|
|
65
|
-
return [this.prefix, ...args];
|
|
66
|
-
}
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
// ../core-apps/src/process.ts
|
|
70
|
-
var EXIT_CODE_ALREADY_HANDLED = 111;
|
|
71
|
-
var EXIT_CODE_CHILD_NONZERO = 112;
|
|
72
|
-
|
|
73
|
-
// ../core-apps/src/backoff.ts
|
|
74
|
-
import { setTimeout as timeout } from "node:timers/promises";
|
|
75
|
-
var StopRetrying = class extends Error {
|
|
76
|
-
constructor(message) {
|
|
77
|
-
super(message);
|
|
78
|
-
this.name = "StopRetrying";
|
|
79
|
-
}
|
|
80
|
-
};
|
|
81
|
-
var AttemptTimeout = class extends Error {
|
|
82
|
-
constructor(message) {
|
|
83
|
-
super(message);
|
|
84
|
-
this.name = "AttemptTimeout";
|
|
85
|
-
}
|
|
86
|
-
};
|
|
87
|
-
var RetryLimitExceeded = class extends Error {
|
|
88
|
-
constructor(message) {
|
|
89
|
-
super(message);
|
|
90
|
-
this.name = "RetryLimitExceeded";
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
var ExponentialBackoff = class _ExponentialBackoff {
|
|
94
|
-
#retries = 0;
|
|
95
|
-
#type;
|
|
96
|
-
#base;
|
|
97
|
-
#factor;
|
|
98
|
-
#min;
|
|
99
|
-
#max;
|
|
100
|
-
#maxRetries;
|
|
101
|
-
#maxElapsed;
|
|
102
|
-
constructor(type, opts = {}) {
|
|
103
|
-
this.#type = type ?? "NoJitter";
|
|
104
|
-
this.#base = opts.base ?? 2;
|
|
105
|
-
this.#factor = opts.factor ?? 1;
|
|
106
|
-
this.#min = opts.min ?? -Infinity;
|
|
107
|
-
this.#max = opts.max ?? Infinity;
|
|
108
|
-
this.#maxRetries = opts.maxRetries ?? Infinity;
|
|
109
|
-
this.#maxElapsed = opts.maxElapsed ?? Infinity;
|
|
110
|
-
}
|
|
111
|
-
#clone(type, opts = {}) {
|
|
112
|
-
return new _ExponentialBackoff(type ?? this.#type, {
|
|
113
|
-
base: opts.base ?? this.#base,
|
|
114
|
-
factor: opts.factor ?? this.#factor,
|
|
115
|
-
min: opts.min ?? this.#min,
|
|
116
|
-
max: opts.max ?? this.#max,
|
|
117
|
-
maxRetries: opts.maxRetries ?? this.#maxRetries,
|
|
118
|
-
maxElapsed: opts.maxElapsed ?? this.#maxElapsed
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
type(type) {
|
|
122
|
-
return this.#clone(type);
|
|
123
|
-
}
|
|
124
|
-
base(base) {
|
|
125
|
-
return this.#clone(void 0, { base });
|
|
126
|
-
}
|
|
127
|
-
factor(factor) {
|
|
128
|
-
return this.#clone(void 0, { factor });
|
|
129
|
-
}
|
|
130
|
-
min(min) {
|
|
131
|
-
return this.#clone(void 0, { min });
|
|
132
|
-
}
|
|
133
|
-
max(max) {
|
|
134
|
-
return this.#clone(void 0, { max });
|
|
135
|
-
}
|
|
136
|
-
maxRetries(maxRetries) {
|
|
137
|
-
return this.#clone(void 0, { maxRetries });
|
|
138
|
-
}
|
|
139
|
-
// TODO: With .execute(), should this also include the time it takes to execute the callback?
|
|
140
|
-
maxElapsed(maxElapsed) {
|
|
141
|
-
return this.#clone(void 0, { maxElapsed });
|
|
142
|
-
}
|
|
143
|
-
retries(retries) {
|
|
144
|
-
if (typeof retries !== "undefined") {
|
|
145
|
-
if (retries > this.#maxRetries) {
|
|
146
|
-
console.error(
|
|
147
|
-
`Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
|
|
148
|
-
);
|
|
149
|
-
this.#retries = this.#maxRetries;
|
|
150
|
-
} else {
|
|
151
|
-
this.#retries = retries;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
return this.#clone();
|
|
155
|
-
}
|
|
156
|
-
async *retryAsync(maxRetries = this.#maxRetries ?? Infinity) {
|
|
157
|
-
let elapsed = 0;
|
|
158
|
-
let retry = 0;
|
|
159
|
-
while (retry <= maxRetries) {
|
|
160
|
-
const delay = this.delay(retry);
|
|
161
|
-
elapsed += delay;
|
|
162
|
-
if (elapsed > this.#maxElapsed) {
|
|
163
|
-
break;
|
|
164
|
-
}
|
|
165
|
-
yield {
|
|
166
|
-
delay: {
|
|
167
|
-
seconds: delay,
|
|
168
|
-
milliseconds: delay * 1e3
|
|
169
|
-
},
|
|
170
|
-
retry
|
|
171
|
-
};
|
|
172
|
-
retry++;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
async *[Symbol.asyncIterator]() {
|
|
176
|
-
yield* this.retryAsync();
|
|
177
|
-
}
|
|
178
|
-
/** Returns the delay for the current retry in seconds. */
|
|
179
|
-
delay(retries = this.#retries, jitter = true) {
|
|
180
|
-
if (retries > this.#maxRetries) {
|
|
181
|
-
console.error(
|
|
182
|
-
`Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
|
|
183
|
-
);
|
|
184
|
-
retries = this.#maxRetries;
|
|
185
|
-
}
|
|
186
|
-
let delay = this.#factor * this.#base ** retries;
|
|
187
|
-
switch (this.#type) {
|
|
188
|
-
case "NoJitter": {
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
|
-
case "FullJitter": {
|
|
192
|
-
if (!jitter) {
|
|
193
|
-
delay = 0;
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
196
|
-
delay *= Math.random();
|
|
197
|
-
break;
|
|
198
|
-
}
|
|
199
|
-
case "EqualJitter": {
|
|
200
|
-
if (!jitter) {
|
|
201
|
-
delay *= 0.5;
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
delay *= 0.5 * (1 + Math.random());
|
|
205
|
-
break;
|
|
206
|
-
}
|
|
207
|
-
default: {
|
|
208
|
-
throw new Error(`Unknown backoff type: ${this.#type}`);
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
if (delay < this.#min) {
|
|
212
|
-
delay = this.#min + Math.random() * (this.#min * 0.2);
|
|
213
|
-
}
|
|
214
|
-
if (delay > this.#max) {
|
|
215
|
-
delay = this.#max - Math.random() * (this.#max * 0.2);
|
|
216
|
-
}
|
|
217
|
-
delay = Math.round(delay);
|
|
218
|
-
return delay;
|
|
219
|
-
}
|
|
220
|
-
/** Waits with the appropriate delay for the current retry. */
|
|
221
|
-
async wait(retries = this.#retries, jitter = true) {
|
|
222
|
-
if (retries > this.#maxRetries) {
|
|
223
|
-
console.error(`Retry limit exceeded: ${retries} > ${this.#maxRetries}`);
|
|
224
|
-
throw new RetryLimitExceeded();
|
|
225
|
-
}
|
|
226
|
-
const delay = this.delay(retries, jitter);
|
|
227
|
-
return await timeout(delay * 1e3);
|
|
228
|
-
}
|
|
229
|
-
elapsed(retries = this.#retries, jitter = true) {
|
|
230
|
-
let elapsed = 0;
|
|
231
|
-
for (let i = 0; i <= retries; i++) {
|
|
232
|
-
elapsed += this.delay(i, jitter);
|
|
233
|
-
}
|
|
234
|
-
const total = elapsed;
|
|
235
|
-
let days = 0;
|
|
236
|
-
if (elapsed > 3600 * 24) {
|
|
237
|
-
days = Math.floor(elapsed / 3600 / 24);
|
|
238
|
-
elapsed -= days * 3600 * 24;
|
|
239
|
-
}
|
|
240
|
-
let hours = 0;
|
|
241
|
-
if (elapsed > 3600) {
|
|
242
|
-
hours = Math.floor(elapsed / 3600);
|
|
243
|
-
elapsed -= hours * 3600;
|
|
244
|
-
}
|
|
245
|
-
let minutes = 0;
|
|
246
|
-
if (elapsed > 60) {
|
|
247
|
-
minutes = Math.floor(elapsed / 60);
|
|
248
|
-
elapsed -= minutes * 60;
|
|
249
|
-
}
|
|
250
|
-
const seconds = elapsed;
|
|
251
|
-
return {
|
|
252
|
-
seconds,
|
|
253
|
-
minutes,
|
|
254
|
-
hours,
|
|
255
|
-
days,
|
|
256
|
-
total
|
|
257
|
-
};
|
|
258
|
-
}
|
|
259
|
-
reset() {
|
|
260
|
-
this.#retries = 0;
|
|
261
|
-
return this;
|
|
262
|
-
}
|
|
263
|
-
next() {
|
|
264
|
-
this.#retries++;
|
|
265
|
-
return this.delay();
|
|
266
|
-
}
|
|
267
|
-
stop() {
|
|
268
|
-
throw new StopRetrying();
|
|
269
|
-
}
|
|
270
|
-
get state() {
|
|
271
|
-
return {
|
|
272
|
-
retries: this.#retries,
|
|
273
|
-
type: this.#type,
|
|
274
|
-
base: this.#base,
|
|
275
|
-
factor: this.#factor,
|
|
276
|
-
min: this.#min,
|
|
277
|
-
max: this.#max,
|
|
278
|
-
maxRetries: this.#maxRetries,
|
|
279
|
-
maxElapsed: this.#maxElapsed
|
|
280
|
-
};
|
|
281
|
-
}
|
|
282
|
-
async execute(callback, { attemptTimeoutMs = 0 } = {}) {
|
|
283
|
-
let elapsedMs = 0;
|
|
284
|
-
let finalError = void 0;
|
|
285
|
-
for await (const { delay, retry } of this) {
|
|
286
|
-
const start = Date.now();
|
|
287
|
-
if (retry > 0) {
|
|
288
|
-
console.log(`Retrying in ${delay.milliseconds}ms`);
|
|
289
|
-
await timeout(delay.milliseconds);
|
|
290
|
-
}
|
|
291
|
-
let attemptTimeout = void 0;
|
|
292
|
-
try {
|
|
293
|
-
const result = await new Promise(async (resolve, reject) => {
|
|
294
|
-
if (attemptTimeoutMs > 0) {
|
|
295
|
-
attemptTimeout = setTimeout(() => {
|
|
296
|
-
reject(new AttemptTimeout());
|
|
297
|
-
}, attemptTimeoutMs);
|
|
298
|
-
}
|
|
299
|
-
try {
|
|
300
|
-
const callbackResult = await callback({ delay, retry, elapsedMs });
|
|
301
|
-
resolve(callbackResult);
|
|
302
|
-
} catch (error) {
|
|
303
|
-
reject(error);
|
|
304
|
-
}
|
|
305
|
-
});
|
|
306
|
-
return {
|
|
307
|
-
success: true,
|
|
308
|
-
result
|
|
309
|
-
};
|
|
310
|
-
} catch (error) {
|
|
311
|
-
finalError = error;
|
|
312
|
-
if (error instanceof StopRetrying) {
|
|
313
|
-
return {
|
|
314
|
-
success: false,
|
|
315
|
-
cause: "StopRetrying",
|
|
316
|
-
error: error.message
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
if (error instanceof AttemptTimeout) {
|
|
320
|
-
continue;
|
|
321
|
-
}
|
|
322
|
-
} finally {
|
|
323
|
-
elapsedMs += Date.now() - start;
|
|
324
|
-
clearTimeout(attemptTimeout);
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
if (finalError instanceof AttemptTimeout) {
|
|
328
|
-
return {
|
|
329
|
-
success: false,
|
|
330
|
-
cause: "Timeout"
|
|
331
|
-
};
|
|
332
|
-
} else {
|
|
333
|
-
return {
|
|
334
|
-
success: false,
|
|
335
|
-
cause: "MaxRetries",
|
|
336
|
-
error: finalError
|
|
337
|
-
};
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
static RetryLimitExceeded = RetryLimitExceeded;
|
|
341
|
-
static StopRetrying = StopRetrying;
|
|
342
|
-
};
|
|
343
|
-
|
|
344
|
-
// src/workers/prod/backgroundWorker.ts
|
|
345
|
-
import {
|
|
346
|
-
ProdChildToWorkerMessages,
|
|
347
|
-
ProdWorkerToChildMessages,
|
|
348
|
-
SemanticInternalAttributes,
|
|
349
|
-
TaskRunErrorCodes,
|
|
350
|
-
correctErrorStackTrace
|
|
351
|
-
} from "@trigger.dev/core/v3";
|
|
352
|
-
import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
|
|
353
|
-
import { Evt } from "evt";
|
|
354
|
-
import { fork } from "node:child_process";
|
|
355
|
-
|
|
356
|
-
// src/workers/common/errors.ts
|
|
357
|
-
var UncaughtExceptionError = class extends Error {
|
|
358
|
-
constructor(originalError, origin) {
|
|
359
|
-
super(`Uncaught exception: ${originalError.message}`);
|
|
360
|
-
this.originalError = originalError;
|
|
361
|
-
this.origin = origin;
|
|
362
|
-
this.name = "UncaughtExceptionError";
|
|
363
|
-
}
|
|
364
|
-
};
|
|
365
|
-
var TaskMetadataParseError = class extends Error {
|
|
366
|
-
constructor(zodIssues, tasks) {
|
|
367
|
-
super(`Failed to parse task metadata`);
|
|
368
|
-
this.zodIssues = zodIssues;
|
|
369
|
-
this.tasks = tasks;
|
|
370
|
-
this.name = "TaskMetadataParseError";
|
|
371
|
-
}
|
|
372
|
-
};
|
|
373
|
-
var UnexpectedExitError = class extends Error {
|
|
374
|
-
constructor(code, signal, stderr) {
|
|
375
|
-
super(`Unexpected exit with code ${code}`);
|
|
376
|
-
this.code = code;
|
|
377
|
-
this.signal = signal;
|
|
378
|
-
this.stderr = stderr;
|
|
379
|
-
this.name = "UnexpectedExitError";
|
|
380
|
-
}
|
|
381
|
-
};
|
|
382
|
-
var CleanupProcessError = class extends Error {
|
|
383
|
-
constructor() {
|
|
384
|
-
super("Cancelled");
|
|
385
|
-
this.name = "CleanupProcessError";
|
|
386
|
-
}
|
|
387
|
-
};
|
|
388
|
-
var CancelledProcessError = class extends Error {
|
|
389
|
-
constructor() {
|
|
390
|
-
super("Cancelled");
|
|
391
|
-
this.name = "CancelledProcessError";
|
|
392
|
-
}
|
|
393
|
-
};
|
|
394
|
-
var SigKillTimeoutProcessError = class extends Error {
|
|
395
|
-
constructor() {
|
|
396
|
-
super("Process kill timeout");
|
|
397
|
-
this.name = "SigKillTimeoutProcessError";
|
|
398
|
-
}
|
|
399
|
-
};
|
|
400
|
-
var GracefulExitTimeoutError = class extends Error {
|
|
401
|
-
constructor() {
|
|
402
|
-
super("Graceful exit timeout");
|
|
403
|
-
this.name = "GracefulExitTimeoutError";
|
|
404
|
-
}
|
|
405
|
-
};
|
|
406
|
-
function getFriendlyErrorMessage(code, signal, stderr, dockerMode = true) {
|
|
407
|
-
const message = (text) => {
|
|
408
|
-
if (signal) {
|
|
409
|
-
return `[${signal}] ${text}`;
|
|
410
|
-
} else {
|
|
411
|
-
return text;
|
|
412
|
-
}
|
|
413
|
-
};
|
|
414
|
-
if (code === 137) {
|
|
415
|
-
if (dockerMode) {
|
|
416
|
-
return message(
|
|
417
|
-
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
|
|
418
|
-
);
|
|
419
|
-
} else {
|
|
420
|
-
return message(
|
|
421
|
-
"Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
|
|
422
|
-
);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
if (stderr?.includes("OOMErrorHandler")) {
|
|
426
|
-
return message(
|
|
427
|
-
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
|
|
428
|
-
);
|
|
429
|
-
}
|
|
430
|
-
return message(`Process exited with code ${code}.`);
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// src/workers/prod/backgroundWorker.ts
|
|
434
|
-
var ProdBackgroundWorker = class {
|
|
435
|
-
constructor(path, params) {
|
|
436
|
-
this.path = path;
|
|
437
|
-
this.params = params;
|
|
438
|
-
}
|
|
439
|
-
_initialized = false;
|
|
440
|
-
/**
|
|
441
|
-
* @deprecated use onTaskRunHeartbeat instead
|
|
442
|
-
*/
|
|
443
|
-
onTaskHeartbeat = new Evt();
|
|
444
|
-
onTaskRunHeartbeat = new Evt();
|
|
445
|
-
onWaitForDuration = new Evt();
|
|
446
|
-
onWaitForTask = new Evt();
|
|
447
|
-
onWaitForBatch = new Evt();
|
|
448
|
-
onCreateTaskRunAttempt = Evt.create();
|
|
449
|
-
attemptCreatedNotification = Evt.create();
|
|
450
|
-
_onClose = new Evt();
|
|
451
|
-
tasks = [];
|
|
452
|
-
stderr = [];
|
|
453
|
-
_taskRunProcess;
|
|
454
|
-
_taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
|
|
455
|
-
_closed = false;
|
|
456
|
-
async close(gracefulExitTimeoutElapsed = false) {
|
|
457
|
-
console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
|
|
458
|
-
if (this._closed) {
|
|
459
|
-
return;
|
|
460
|
-
}
|
|
461
|
-
this._closed = true;
|
|
462
|
-
this.onTaskHeartbeat.detach();
|
|
463
|
-
this.onTaskRunHeartbeat.detach();
|
|
464
|
-
await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
|
|
465
|
-
}
|
|
466
|
-
async #killTaskRunProcess(flush = true, initialSignal = "SIGTERM") {
|
|
467
|
-
console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
|
|
468
|
-
if (this._closed || !this._taskRunProcess) {
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
if (flush) {
|
|
472
|
-
await this.flushTelemetry();
|
|
473
|
-
}
|
|
474
|
-
const currentTaskRunProcess = this._taskRunProcess;
|
|
475
|
-
this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
|
|
476
|
-
console.error("Error while trying graceful exit", error);
|
|
477
|
-
});
|
|
478
|
-
console.log("Killed task run process, setting closed to true", {
|
|
479
|
-
closed: this._closed,
|
|
480
|
-
pid: currentTaskRunProcess.pid
|
|
481
|
-
});
|
|
482
|
-
this._closed = true;
|
|
483
|
-
}
|
|
484
|
-
async flushTelemetry() {
|
|
485
|
-
console.log("Flushing telemetry");
|
|
486
|
-
const start = performance.now();
|
|
487
|
-
await this._taskRunProcess?.cleanup(false);
|
|
488
|
-
console.log("Flushed telemetry", { duration: performance.now() - start });
|
|
489
|
-
}
|
|
490
|
-
async initialize(options) {
|
|
491
|
-
if (this._initialized) {
|
|
492
|
-
throw new Error("Worker already initialized");
|
|
493
|
-
}
|
|
494
|
-
let resolved = false;
|
|
495
|
-
this.tasks = await new Promise((resolve, reject) => {
|
|
496
|
-
const child = fork(this.path, {
|
|
497
|
-
stdio: [
|
|
498
|
-
/*stdin*/
|
|
499
|
-
"ignore",
|
|
500
|
-
/*stdout*/
|
|
501
|
-
"pipe",
|
|
502
|
-
/*stderr*/
|
|
503
|
-
"pipe",
|
|
504
|
-
"ipc"
|
|
505
|
-
],
|
|
506
|
-
env: {
|
|
507
|
-
...this.params.env,
|
|
508
|
-
...options?.env
|
|
509
|
-
}
|
|
510
|
-
});
|
|
511
|
-
const timeout3 = setTimeout(() => {
|
|
512
|
-
if (resolved) {
|
|
513
|
-
return;
|
|
514
|
-
}
|
|
515
|
-
resolved = true;
|
|
516
|
-
child.kill();
|
|
517
|
-
reject(new Error("Worker timed out"));
|
|
518
|
-
}, 1e4);
|
|
519
|
-
child.stdout?.on("data", (data) => {
|
|
520
|
-
console.log(data.toString());
|
|
521
|
-
});
|
|
522
|
-
child.stderr?.on("data", (data) => {
|
|
523
|
-
console.error(data.toString());
|
|
524
|
-
this.stderr.push(data.toString());
|
|
525
|
-
});
|
|
526
|
-
child.on("exit", (code) => {
|
|
527
|
-
if (!resolved) {
|
|
528
|
-
clearTimeout(timeout3);
|
|
529
|
-
resolved = true;
|
|
530
|
-
reject(new Error(`Worker exited with code ${code}`));
|
|
531
|
-
}
|
|
532
|
-
});
|
|
533
|
-
new ZodIpcConnection({
|
|
534
|
-
listenSchema: ProdChildToWorkerMessages,
|
|
535
|
-
emitSchema: ProdWorkerToChildMessages,
|
|
536
|
-
process: child,
|
|
537
|
-
handlers: {
|
|
538
|
-
TASKS_READY: async (message) => {
|
|
539
|
-
if (!resolved) {
|
|
540
|
-
clearTimeout(timeout3);
|
|
541
|
-
resolved = true;
|
|
542
|
-
resolve(message.tasks);
|
|
543
|
-
child.kill();
|
|
544
|
-
}
|
|
545
|
-
},
|
|
546
|
-
UNCAUGHT_EXCEPTION: async (message) => {
|
|
547
|
-
if (!resolved) {
|
|
548
|
-
clearTimeout(timeout3);
|
|
549
|
-
resolved = true;
|
|
550
|
-
reject(new UncaughtExceptionError(message.error, message.origin));
|
|
551
|
-
child.kill();
|
|
552
|
-
}
|
|
553
|
-
},
|
|
554
|
-
TASKS_FAILED_TO_PARSE: async (message) => {
|
|
555
|
-
if (!resolved) {
|
|
556
|
-
clearTimeout(timeout3);
|
|
557
|
-
resolved = true;
|
|
558
|
-
reject(new TaskMetadataParseError(message.zodIssues, message.tasks));
|
|
559
|
-
child.kill();
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
});
|
|
564
|
-
});
|
|
565
|
-
this._initialized = true;
|
|
566
|
-
}
|
|
567
|
-
getMetadata(workerId, version) {
|
|
568
|
-
return {
|
|
569
|
-
contentHash: this.params.contentHash,
|
|
570
|
-
id: workerId,
|
|
571
|
-
version
|
|
572
|
-
};
|
|
573
|
-
}
|
|
574
|
-
// We need to notify all the task run processes that a task run has completed,
|
|
575
|
-
// in case they are waiting for it through triggerAndWait
|
|
576
|
-
async taskRunCompletedNotification(completion) {
|
|
577
|
-
this._taskRunProcess?.taskRunCompletedNotification(completion);
|
|
578
|
-
}
|
|
579
|
-
async waitCompletedNotification() {
|
|
580
|
-
this._taskRunProcess?.waitCompletedNotification();
|
|
581
|
-
}
|
|
582
|
-
async #getFreshTaskRunProcess(payload, messageId) {
|
|
583
|
-
const metadata = this.getMetadata(
|
|
584
|
-
payload.execution.worker.id,
|
|
585
|
-
payload.execution.worker.version
|
|
586
|
-
);
|
|
587
|
-
console.log("Getting fresh task run process, setting closed to false", {
|
|
588
|
-
closed: this._closed
|
|
589
|
-
});
|
|
590
|
-
this._closed = false;
|
|
591
|
-
await this.#killCurrentTaskRunProcessBeforeAttempt();
|
|
592
|
-
const taskRunProcess = new TaskRunProcess(
|
|
593
|
-
payload.execution.run.id,
|
|
594
|
-
payload.execution.run.isTest,
|
|
595
|
-
this.path,
|
|
596
|
-
{
|
|
597
|
-
...this.params.env,
|
|
598
|
-
...payload.environment ?? {}
|
|
599
|
-
},
|
|
600
|
-
metadata,
|
|
601
|
-
this.params,
|
|
602
|
-
messageId
|
|
603
|
-
);
|
|
604
|
-
taskRunProcess.onExit.attach(({ pid }) => {
|
|
605
|
-
console.log("Task run process exited", { pid });
|
|
606
|
-
if (this._taskRunProcess?.pid === pid) {
|
|
607
|
-
this._taskRunProcess = void 0;
|
|
608
|
-
}
|
|
609
|
-
if (pid) {
|
|
610
|
-
this._taskRunProcessesBeingKilled.delete(pid);
|
|
611
|
-
}
|
|
612
|
-
});
|
|
613
|
-
taskRunProcess.onIsBeingKilled.attach((taskRunProcess2) => {
|
|
614
|
-
if (taskRunProcess2?.pid) {
|
|
615
|
-
this._taskRunProcessesBeingKilled.set(taskRunProcess2.pid, taskRunProcess2);
|
|
616
|
-
}
|
|
617
|
-
});
|
|
618
|
-
taskRunProcess.onTaskHeartbeat.attach((id) => {
|
|
619
|
-
this.onTaskHeartbeat.post(id);
|
|
620
|
-
});
|
|
621
|
-
taskRunProcess.onTaskRunHeartbeat.attach((id) => {
|
|
622
|
-
this.onTaskRunHeartbeat.post(id);
|
|
623
|
-
});
|
|
624
|
-
taskRunProcess.onWaitForBatch.attach((message) => {
|
|
625
|
-
this.onWaitForBatch.post(message);
|
|
626
|
-
});
|
|
627
|
-
taskRunProcess.onWaitForDuration.attach((message) => {
|
|
628
|
-
this.onWaitForDuration.post(message);
|
|
629
|
-
});
|
|
630
|
-
taskRunProcess.onWaitForTask.attach((message) => {
|
|
631
|
-
this.onWaitForTask.post(message);
|
|
632
|
-
});
|
|
633
|
-
await taskRunProcess.initialize();
|
|
634
|
-
this._taskRunProcess = taskRunProcess;
|
|
635
|
-
return this._taskRunProcess;
|
|
636
|
-
}
|
|
637
|
-
async forceKillOldTaskRunProcesses() {
|
|
638
|
-
for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
|
|
639
|
-
try {
|
|
640
|
-
await taskRunProcess.kill("SIGKILL");
|
|
641
|
-
} catch (error) {
|
|
642
|
-
console.error("Error while force killing old task run processes", error);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
async #killCurrentTaskRunProcessBeforeAttempt() {
|
|
647
|
-
console.log("killCurrentTaskRunProcessBeforeAttempt()", {
|
|
648
|
-
hasTaskRunProcess: !!this._taskRunProcess
|
|
649
|
-
});
|
|
650
|
-
if (!this._taskRunProcess) {
|
|
651
|
-
return;
|
|
652
|
-
}
|
|
653
|
-
const currentTaskRunProcess = this._taskRunProcess;
|
|
654
|
-
console.log("Killing current task run process", {
|
|
655
|
-
isBeingKilled: currentTaskRunProcess?.isBeingKilled,
|
|
656
|
-
totalBeingKilled: this._taskRunProcessesBeingKilled.size
|
|
657
|
-
});
|
|
658
|
-
if (currentTaskRunProcess.isBeingKilled) {
|
|
659
|
-
if (this._taskRunProcessesBeingKilled.size > 1) {
|
|
660
|
-
await this.#tryGracefulExit(currentTaskRunProcess);
|
|
661
|
-
} else {
|
|
662
|
-
}
|
|
663
|
-
} else {
|
|
664
|
-
if (this._taskRunProcessesBeingKilled.size > 0) {
|
|
665
|
-
await this.#tryGracefulExit(currentTaskRunProcess);
|
|
666
|
-
} else {
|
|
667
|
-
currentTaskRunProcess.kill("SIGTERM", 5e3).catch(() => {
|
|
668
|
-
});
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
|
|
673
|
-
console.log("Trying graceful exit", { kill, initialSignal });
|
|
674
|
-
try {
|
|
675
|
-
const initialExit = taskRunProcess.onExit.waitFor(5e3);
|
|
676
|
-
if (kill) {
|
|
677
|
-
taskRunProcess.kill(initialSignal);
|
|
678
|
-
}
|
|
679
|
-
await initialExit;
|
|
680
|
-
} catch (error) {
|
|
681
|
-
console.error("TaskRunProcess graceful kill timeout exceeded", error);
|
|
682
|
-
this.#tryForcefulExit(taskRunProcess);
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
async #tryForcefulExit(taskRunProcess) {
|
|
686
|
-
console.log("Trying forceful exit");
|
|
687
|
-
try {
|
|
688
|
-
const forcedKill = taskRunProcess.onExit.waitFor(5e3);
|
|
689
|
-
taskRunProcess.kill("SIGKILL");
|
|
690
|
-
await forcedKill;
|
|
691
|
-
} catch (error) {
|
|
692
|
-
console.error("TaskRunProcess forced kill timeout exceeded", error);
|
|
693
|
-
throw new SigKillTimeoutProcessError();
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
// We need to fork the process before we can execute any tasks, use a fresh process for each execution
|
|
697
|
-
async executeTaskRun(payload, messageId) {
|
|
698
|
-
try {
|
|
699
|
-
const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
|
|
700
|
-
console.log("executing task run", {
|
|
701
|
-
attempt: payload.execution.attempt.id,
|
|
702
|
-
taskRunPid: taskRunProcess.pid
|
|
703
|
-
});
|
|
704
|
-
const result = await taskRunProcess.executeTaskRun(payload);
|
|
705
|
-
if (result.ok) {
|
|
706
|
-
return result;
|
|
707
|
-
}
|
|
708
|
-
const error = result.error;
|
|
709
|
-
if (error.type === "BUILT_IN_ERROR") {
|
|
710
|
-
const mappedError = await this.#correctError(error, payload.execution);
|
|
711
|
-
return {
|
|
712
|
-
...result,
|
|
713
|
-
error: mappedError
|
|
714
|
-
};
|
|
715
|
-
}
|
|
716
|
-
return result;
|
|
717
|
-
} catch (e) {
|
|
718
|
-
if (e instanceof CancelledProcessError) {
|
|
719
|
-
return {
|
|
720
|
-
id: payload.execution.attempt.id,
|
|
721
|
-
ok: false,
|
|
722
|
-
retry: void 0,
|
|
723
|
-
error: {
|
|
724
|
-
type: "INTERNAL_ERROR",
|
|
725
|
-
code: TaskRunErrorCodes.TASK_RUN_CANCELLED
|
|
726
|
-
}
|
|
727
|
-
};
|
|
728
|
-
}
|
|
729
|
-
if (e instanceof CleanupProcessError) {
|
|
730
|
-
return {
|
|
731
|
-
id: payload.execution.attempt.id,
|
|
732
|
-
ok: false,
|
|
733
|
-
retry: void 0,
|
|
734
|
-
error: {
|
|
735
|
-
type: "INTERNAL_ERROR",
|
|
736
|
-
code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED
|
|
737
|
-
}
|
|
738
|
-
};
|
|
739
|
-
}
|
|
740
|
-
if (e instanceof UnexpectedExitError) {
|
|
741
|
-
return {
|
|
742
|
-
id: payload.execution.attempt.id,
|
|
743
|
-
ok: false,
|
|
744
|
-
retry: void 0,
|
|
745
|
-
error: {
|
|
746
|
-
type: "INTERNAL_ERROR",
|
|
747
|
-
code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
|
|
748
|
-
message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
|
|
749
|
-
stackTrace: e.stderr
|
|
750
|
-
}
|
|
751
|
-
};
|
|
752
|
-
}
|
|
753
|
-
if (e instanceof SigKillTimeoutProcessError) {
|
|
754
|
-
return {
|
|
755
|
-
id: payload.execution.attempt.id,
|
|
756
|
-
ok: false,
|
|
757
|
-
retry: void 0,
|
|
758
|
-
error: {
|
|
759
|
-
type: "INTERNAL_ERROR",
|
|
760
|
-
code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT
|
|
761
|
-
}
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
if (e instanceof GracefulExitTimeoutError) {
|
|
765
|
-
return {
|
|
766
|
-
id: payload.execution.attempt.id,
|
|
767
|
-
ok: false,
|
|
768
|
-
retry: void 0,
|
|
769
|
-
error: {
|
|
770
|
-
type: "INTERNAL_ERROR",
|
|
771
|
-
code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
|
|
772
|
-
message: "Worker process killed while attempt in progress."
|
|
773
|
-
}
|
|
774
|
-
};
|
|
775
|
-
}
|
|
776
|
-
return {
|
|
777
|
-
id: payload.execution.attempt.id,
|
|
778
|
-
ok: false,
|
|
779
|
-
retry: void 0,
|
|
780
|
-
error: {
|
|
781
|
-
type: "INTERNAL_ERROR",
|
|
782
|
-
code: TaskRunErrorCodes.TASK_EXECUTION_FAILED
|
|
783
|
-
}
|
|
784
|
-
};
|
|
785
|
-
} finally {
|
|
786
|
-
await this.#killTaskRunProcess();
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
async cancelAttempt(attemptId) {
|
|
790
|
-
if (!this._taskRunProcess) {
|
|
791
|
-
console.error("No task run process to cancel attempt", { attemptId });
|
|
792
|
-
return;
|
|
793
|
-
}
|
|
794
|
-
await this._taskRunProcess.cancel();
|
|
795
|
-
}
|
|
796
|
-
async executeTaskRunLazyAttempt(payload) {
|
|
797
|
-
this.onCreateTaskRunAttempt.post({ runId: payload.runId });
|
|
798
|
-
let execution;
|
|
799
|
-
try {
|
|
800
|
-
const start = performance.now();
|
|
801
|
-
const attemptCreated = await this.attemptCreatedNotification.waitFor(12e4);
|
|
802
|
-
if (!attemptCreated.success) {
|
|
803
|
-
throw new Error(`${attemptCreated.reason ?? "Unknown error"}`);
|
|
804
|
-
}
|
|
805
|
-
console.log("Attempt created", {
|
|
806
|
-
number: attemptCreated.execution.attempt.number,
|
|
807
|
-
duration: performance.now() - start
|
|
808
|
-
});
|
|
809
|
-
execution = attemptCreated.execution;
|
|
810
|
-
} catch (error) {
|
|
811
|
-
console.error("Error while creating attempt", error);
|
|
812
|
-
throw new Error(`Failed to create attempt: ${error}`);
|
|
813
|
-
}
|
|
814
|
-
const completion = await this.executeTaskRun(
|
|
815
|
-
{
|
|
816
|
-
execution,
|
|
817
|
-
traceContext: payload.traceContext,
|
|
818
|
-
environment: payload.environment
|
|
819
|
-
},
|
|
820
|
-
payload.messageId
|
|
821
|
-
);
|
|
822
|
-
return { execution, completion };
|
|
823
|
-
}
|
|
824
|
-
async #correctError(error, execution) {
|
|
825
|
-
return {
|
|
826
|
-
...error,
|
|
827
|
-
stackTrace: correctErrorStackTrace(error.stackTrace, this.params.projectConfig.projectDir)
|
|
828
|
-
};
|
|
829
|
-
}
|
|
830
|
-
};
|
|
831
|
-
var TaskRunProcess = class {
|
|
832
|
-
constructor(runId, isTest, path, env, metadata, worker, messageId) {
|
|
833
|
-
this.runId = runId;
|
|
834
|
-
this.isTest = isTest;
|
|
835
|
-
this.path = path;
|
|
836
|
-
this.env = env;
|
|
837
|
-
this.metadata = metadata;
|
|
838
|
-
this.worker = worker;
|
|
839
|
-
this.messageId = messageId;
|
|
840
|
-
}
|
|
841
|
-
_ipc;
|
|
842
|
-
_child;
|
|
843
|
-
_childPid;
|
|
844
|
-
_attemptPromises = /* @__PURE__ */ new Map();
|
|
845
|
-
_attemptStatuses = /* @__PURE__ */ new Map();
|
|
846
|
-
_currentExecution;
|
|
847
|
-
_isBeingKilled = false;
|
|
848
|
-
_isBeingCancelled = false;
|
|
849
|
-
_gracefulExitTimeoutElapsed = false;
|
|
850
|
-
_stderr = [];
|
|
851
|
-
/**
|
|
852
|
-
* @deprecated use onTaskRunHeartbeat instead
|
|
853
|
-
*/
|
|
854
|
-
onTaskHeartbeat = new Evt();
|
|
855
|
-
onTaskRunHeartbeat = new Evt();
|
|
856
|
-
onExit = new Evt();
|
|
857
|
-
onIsBeingKilled = new Evt();
|
|
858
|
-
onWaitForDuration = new Evt();
|
|
859
|
-
onWaitForTask = new Evt();
|
|
860
|
-
onWaitForBatch = new Evt();
|
|
861
|
-
preCheckpointNotification = Evt.create();
|
|
862
|
-
async initialize() {
|
|
863
|
-
this._child = fork(this.path, {
|
|
864
|
-
stdio: [
|
|
865
|
-
/*stdin*/
|
|
866
|
-
"ignore",
|
|
867
|
-
/*stdout*/
|
|
868
|
-
"pipe",
|
|
869
|
-
/*stderr*/
|
|
870
|
-
"pipe",
|
|
871
|
-
"ipc"
|
|
872
|
-
],
|
|
873
|
-
env: {
|
|
874
|
-
...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
|
|
875
|
-
...this.env,
|
|
876
|
-
OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
|
|
877
|
-
[SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
|
|
878
|
-
}),
|
|
879
|
-
...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
|
|
880
|
-
}
|
|
881
|
-
});
|
|
882
|
-
this._childPid = this._child?.pid;
|
|
883
|
-
this._ipc = new ZodIpcConnection({
|
|
884
|
-
listenSchema: ProdChildToWorkerMessages,
|
|
885
|
-
emitSchema: ProdWorkerToChildMessages,
|
|
886
|
-
process: this._child,
|
|
887
|
-
handlers: {
|
|
888
|
-
TASK_RUN_COMPLETED: async (message) => {
|
|
889
|
-
const { result, execution } = message;
|
|
890
|
-
const promiseStatus = this._attemptStatuses.get(execution.attempt.id);
|
|
891
|
-
if (promiseStatus !== "PENDING") {
|
|
892
|
-
return;
|
|
893
|
-
}
|
|
894
|
-
this._attemptStatuses.set(execution.attempt.id, "RESOLVED");
|
|
895
|
-
const attemptPromise = this._attemptPromises.get(execution.attempt.id);
|
|
896
|
-
if (!attemptPromise) {
|
|
897
|
-
return;
|
|
898
|
-
}
|
|
899
|
-
const { resolver } = attemptPromise;
|
|
900
|
-
resolver(result);
|
|
901
|
-
},
|
|
902
|
-
READY_TO_DISPOSE: async (message) => {
|
|
903
|
-
process.exit(0);
|
|
904
|
-
},
|
|
905
|
-
TASK_HEARTBEAT: async (message) => {
|
|
906
|
-
if (this.messageId) {
|
|
907
|
-
this.onTaskRunHeartbeat.post(this.messageId);
|
|
908
|
-
} else {
|
|
909
|
-
console.error(
|
|
910
|
-
"No message id for task heartbeat, falling back to (deprecated) attempt heartbeat",
|
|
911
|
-
{ id: message.id }
|
|
912
|
-
);
|
|
913
|
-
this.onTaskHeartbeat.post(message.id);
|
|
914
|
-
}
|
|
915
|
-
},
|
|
916
|
-
TASKS_READY: async (message) => {
|
|
917
|
-
},
|
|
918
|
-
WAIT_FOR_TASK: async (message) => {
|
|
919
|
-
this.onWaitForTask.post(message);
|
|
920
|
-
},
|
|
921
|
-
WAIT_FOR_BATCH: async (message) => {
|
|
922
|
-
this.onWaitForBatch.post(message);
|
|
923
|
-
},
|
|
924
|
-
WAIT_FOR_DURATION: async (message) => {
|
|
925
|
-
this.onWaitForDuration.post(message);
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
});
|
|
929
|
-
this._child.on("exit", this.#handleExit.bind(this));
|
|
930
|
-
this._child.stdout?.on("data", this.#handleLog.bind(this));
|
|
931
|
-
this._child.stderr?.on("data", this.#handleStdErr.bind(this));
|
|
932
|
-
}
|
|
933
|
-
async cancel() {
|
|
934
|
-
this._isBeingCancelled = true;
|
|
935
|
-
await this.cleanup(true);
|
|
936
|
-
}
|
|
937
|
-
async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
|
|
938
|
-
console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
|
|
939
|
-
if (kill && this._isBeingKilled) {
|
|
940
|
-
return;
|
|
941
|
-
}
|
|
942
|
-
if (kill) {
|
|
943
|
-
this._isBeingKilled = true;
|
|
944
|
-
this.onIsBeingKilled.post(this);
|
|
945
|
-
}
|
|
946
|
-
const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
|
|
947
|
-
const killParentProcess = kill && !killChildProcess;
|
|
948
|
-
console.log("Cleaning up task run process", {
|
|
949
|
-
killChildProcess,
|
|
950
|
-
killParentProcess,
|
|
951
|
-
ipc: this._ipc,
|
|
952
|
-
childPid: this._childPid,
|
|
953
|
-
realChildPid: this._child?.pid
|
|
954
|
-
});
|
|
955
|
-
try {
|
|
956
|
-
await this._ipc?.sendWithAck(
|
|
957
|
-
"CLEANUP",
|
|
958
|
-
{
|
|
959
|
-
flush: true,
|
|
960
|
-
kill: killParentProcess
|
|
961
|
-
},
|
|
962
|
-
3e4
|
|
963
|
-
);
|
|
964
|
-
} catch (error) {
|
|
965
|
-
console.error("Error while cleaning up task run process", error);
|
|
966
|
-
if (killParentProcess) {
|
|
967
|
-
process.exit(0);
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
|
-
if (killChildProcess) {
|
|
971
|
-
this._gracefulExitTimeoutElapsed = true;
|
|
972
|
-
await this.kill("SIGKILL");
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
async executeTaskRun(payload) {
|
|
976
|
-
let resolver;
|
|
977
|
-
let rejecter;
|
|
978
|
-
const promise = new Promise((resolve, reject) => {
|
|
979
|
-
resolver = resolve;
|
|
980
|
-
rejecter = reject;
|
|
981
|
-
});
|
|
982
|
-
this._attemptStatuses.set(payload.execution.attempt.id, "PENDING");
|
|
983
|
-
this._attemptPromises.set(payload.execution.attempt.id, { resolver, rejecter });
|
|
984
|
-
const { execution, traceContext } = payload;
|
|
985
|
-
this._currentExecution = execution;
|
|
986
|
-
if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
|
|
987
|
-
await this._ipc?.send("EXECUTE_TASK_RUN", {
|
|
988
|
-
execution,
|
|
989
|
-
traceContext,
|
|
990
|
-
metadata: this.metadata
|
|
991
|
-
});
|
|
992
|
-
}
|
|
993
|
-
const result = await promise;
|
|
994
|
-
this._currentExecution = void 0;
|
|
995
|
-
return result;
|
|
996
|
-
}
|
|
997
|
-
taskRunCompletedNotification(completion) {
|
|
998
|
-
if (!completion.ok && typeof completion.retry !== "undefined") {
|
|
999
|
-
console.error(
|
|
1000
|
-
"Task run completed with error and wants to retry, won't send task run completed notification"
|
|
1001
|
-
);
|
|
1002
|
-
return;
|
|
1003
|
-
}
|
|
1004
|
-
if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
|
|
1005
|
-
console.error(
|
|
1006
|
-
"Child process not connected or being killed, can't send task run completed notification"
|
|
1007
|
-
);
|
|
1008
|
-
return;
|
|
1009
|
-
}
|
|
1010
|
-
this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
|
|
1011
|
-
version: "v2",
|
|
1012
|
-
completion
|
|
1013
|
-
});
|
|
1014
|
-
}
|
|
1015
|
-
waitCompletedNotification() {
|
|
1016
|
-
if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
|
|
1017
|
-
console.error(
|
|
1018
|
-
"Child process not connected or being killed, can't send wait completed notification"
|
|
1019
|
-
);
|
|
1020
|
-
return;
|
|
1021
|
-
}
|
|
1022
|
-
this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
|
|
1023
|
-
}
|
|
1024
|
-
async #handleExit(code, signal) {
|
|
1025
|
-
console.log("handling child exit", { code, signal });
|
|
1026
|
-
for (const [id, status] of this._attemptStatuses.entries()) {
|
|
1027
|
-
if (status === "PENDING") {
|
|
1028
|
-
console.log("found pending attempt", { id });
|
|
1029
|
-
this._attemptStatuses.set(id, "REJECTED");
|
|
1030
|
-
const attemptPromise = this._attemptPromises.get(id);
|
|
1031
|
-
if (!attemptPromise) {
|
|
1032
|
-
continue;
|
|
1033
|
-
}
|
|
1034
|
-
const { rejecter } = attemptPromise;
|
|
1035
|
-
if (this._isBeingCancelled) {
|
|
1036
|
-
rejecter(new CancelledProcessError());
|
|
1037
|
-
} else if (this._gracefulExitTimeoutElapsed) {
|
|
1038
|
-
rejecter(new GracefulExitTimeoutError());
|
|
1039
|
-
} else if (this._isBeingKilled) {
|
|
1040
|
-
rejecter(new CleanupProcessError());
|
|
1041
|
-
} else {
|
|
1042
|
-
rejecter(
|
|
1043
|
-
new UnexpectedExitError(
|
|
1044
|
-
code ?? -1,
|
|
1045
|
-
signal,
|
|
1046
|
-
this._stderr.length ? this._stderr.join("\n") : void 0
|
|
1047
|
-
)
|
|
1048
|
-
);
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
}
|
|
1052
|
-
this.onExit.post({ code, signal, pid: this.pid });
|
|
1053
|
-
}
|
|
1054
|
-
#handleLog(data) {
|
|
1055
|
-
console.log(data.toString());
|
|
1056
|
-
}
|
|
1057
|
-
#handleStdErr(data) {
|
|
1058
|
-
const text = data.toString();
|
|
1059
|
-
console.error(text);
|
|
1060
|
-
if (this._stderr.length > 100) {
|
|
1061
|
-
this._stderr.shift();
|
|
1062
|
-
}
|
|
1063
|
-
this._stderr.push(text);
|
|
1064
|
-
}
|
|
1065
|
-
async kill(signal, timeoutInMs) {
|
|
1066
|
-
this._isBeingKilled = true;
|
|
1067
|
-
const killTimeout = this.onExit.waitFor(timeoutInMs);
|
|
1068
|
-
this.onIsBeingKilled.post(this);
|
|
1069
|
-
this._child?.kill(signal);
|
|
1070
|
-
if (timeoutInMs) {
|
|
1071
|
-
await killTimeout;
|
|
1072
|
-
}
|
|
1073
|
-
}
|
|
1074
|
-
get isBeingKilled() {
|
|
1075
|
-
return this._isBeingKilled || this._child?.killed;
|
|
1076
|
-
}
|
|
1077
|
-
get pid() {
|
|
1078
|
-
return this._childPid;
|
|
1079
|
-
}
|
|
1080
|
-
};
|
|
1081
|
-
|
|
1082
|
-
// src/workers/prod/entry-point.ts
|
|
1083
|
-
import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers";
|
|
1084
|
-
import { randomUUID } from "node:crypto";
|
|
1085
|
-
import { readFile } from "node:fs/promises";
|
|
1086
|
-
import { createServer } from "node:http";
|
|
1087
|
-
import { setTimeout as timeout2 } from "node:timers/promises";
|
|
1088
|
-
var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
|
1089
|
-
var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
|
1090
|
-
var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
|
|
1091
|
-
var MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
|
1092
|
-
var POD_NAME = process.env.POD_NAME || "some-pod";
|
|
1093
|
-
var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
|
|
1094
|
-
var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
|
|
1095
|
-
var defaultBackoff = new ExponentialBackoff("FullJitter", {
|
|
1096
|
-
maxRetries: 5
|
|
1097
|
-
});
|
|
1098
|
-
var ProdWorker = class {
|
|
1099
|
-
constructor(port, host = "0.0.0.0") {
|
|
1100
|
-
this.host = host;
|
|
1101
|
-
process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
|
|
1102
|
-
this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
|
|
1103
|
-
this.#backgroundWorker = this.#createBackgroundWorker();
|
|
1104
|
-
this.#httpPort = port;
|
|
1105
|
-
this.#httpServer = this.#createHttpServer();
|
|
1106
|
-
}
|
|
1107
|
-
apiUrl = process.env.TRIGGER_API_URL;
|
|
1108
|
-
apiKey = process.env.TRIGGER_SECRET_KEY;
|
|
1109
|
-
contentHash = process.env.TRIGGER_CONTENT_HASH;
|
|
1110
|
-
projectRef = process.env.TRIGGER_PROJECT_REF;
|
|
1111
|
-
envId = process.env.TRIGGER_ENV_ID;
|
|
1112
|
-
runId = process.env.TRIGGER_RUN_ID || "index-only";
|
|
1113
|
-
deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
|
|
1114
|
-
deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
|
|
1115
|
-
runningInKubernetes = !!process.env.KUBERNETES_PORT;
|
|
1116
|
-
executing = false;
|
|
1117
|
-
completed = /* @__PURE__ */ new Set();
|
|
1118
|
-
paused = false;
|
|
1119
|
-
attemptFriendlyId;
|
|
1120
|
-
nextResumeAfter;
|
|
1121
|
-
waitForPostStart = false;
|
|
1122
|
-
connectionCount = 0;
|
|
1123
|
-
waitForTaskReplay;
|
|
1124
|
-
waitForBatchReplay;
|
|
1125
|
-
readyForLazyAttemptReplay;
|
|
1126
|
-
submitAttemptCompletionReplay;
|
|
1127
|
-
durationResumeFallback;
|
|
1128
|
-
#httpPort;
|
|
1129
|
-
#backgroundWorker;
|
|
1130
|
-
#httpServer;
|
|
1131
|
-
#coordinatorSocket;
|
|
1132
|
-
async #handleSignal(signal) {
|
|
1133
|
-
logger.log("Received signal", { signal });
|
|
1134
|
-
if (signal === "SIGTERM") {
|
|
1135
|
-
let gracefulExitTimeoutElapsed = false;
|
|
1136
|
-
if (this.executing) {
|
|
1137
|
-
const terminationGracePeriodSeconds = 60 * 60;
|
|
1138
|
-
logger.log("Waiting for attempt to complete before exiting", {
|
|
1139
|
-
terminationGracePeriodSeconds
|
|
1140
|
-
});
|
|
1141
|
-
await timeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
|
|
1142
|
-
gracefulExitTimeoutElapsed = true;
|
|
1143
|
-
logger.log("Termination timeout reached, exiting gracefully.");
|
|
1144
|
-
} else {
|
|
1145
|
-
logger.log("Not executing, exiting immediately.");
|
|
1146
|
-
}
|
|
1147
|
-
await this.#exitGracefully(gracefulExitTimeoutElapsed);
|
|
1148
|
-
return;
|
|
1149
|
-
}
|
|
1150
|
-
logger.log("Unhandled signal", { signal });
|
|
1151
|
-
}
|
|
1152
|
-
async #exitGracefully(gracefulExitTimeoutElapsed = false, exitCode = 0) {
|
|
1153
|
-
await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
|
|
1154
|
-
if (!gracefulExitTimeoutElapsed) {
|
|
1155
|
-
process.exit(exitCode);
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
async #reconnectAfterPostStart() {
|
|
1159
|
-
this.waitForPostStart = false;
|
|
1160
|
-
this.#coordinatorSocket.close();
|
|
1161
|
-
this.connectionCount = 0;
|
|
1162
|
-
let coordinatorHost = COORDINATOR_HOST;
|
|
1163
|
-
try {
|
|
1164
|
-
if (this.runningInKubernetes) {
|
|
1165
|
-
coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
|
|
1166
|
-
"\n",
|
|
1167
|
-
""
|
|
1168
|
-
);
|
|
1169
|
-
logger.log("reconnecting", {
|
|
1170
|
-
coordinatorHost: {
|
|
1171
|
-
fromEnv: COORDINATOR_HOST,
|
|
1172
|
-
fromVolume: coordinatorHost,
|
|
1173
|
-
current: this.#coordinatorSocket.socket.io.opts.hostname
|
|
1174
|
-
}
|
|
1175
|
-
});
|
|
1176
|
-
}
|
|
1177
|
-
} catch (error) {
|
|
1178
|
-
logger.error("taskinfo read error during reconnect", {
|
|
1179
|
-
error: error instanceof Error ? error.message : error
|
|
1180
|
-
});
|
|
1181
|
-
} finally {
|
|
1182
|
-
this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
|
|
1183
|
-
}
|
|
1184
|
-
}
|
|
1185
|
-
// MARK: TASK WAIT
|
|
1186
|
-
async #waitForTaskHandler(message, replayIdempotencyKey) {
|
|
1187
|
-
const waitForTask = await defaultBackoff.execute(async ({ retry }) => {
|
|
1188
|
-
logger.log("Wait for task with backoff", { retry });
|
|
1189
|
-
if (!this.attemptFriendlyId) {
|
|
1190
|
-
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1191
|
-
throw new ExponentialBackoff.StopRetrying("No attempt ID");
|
|
1192
|
-
}
|
|
1193
|
-
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_TASK", {
|
|
1194
|
-
version: "v2",
|
|
1195
|
-
friendlyId: message.friendlyId,
|
|
1196
|
-
attemptFriendlyId: this.attemptFriendlyId
|
|
1197
|
-
});
|
|
1198
|
-
});
|
|
1199
|
-
if (!waitForTask.success) {
|
|
1200
|
-
logger.error("Failed to wait for task with backoff", {
|
|
1201
|
-
cause: waitForTask.cause,
|
|
1202
|
-
error: waitForTask.error
|
|
1203
|
-
});
|
|
1204
|
-
this.#emitUnrecoverableError(
|
|
1205
|
-
"WaitForTaskFailed",
|
|
1206
|
-
`${waitForTask.cause}: ${waitForTask.error}`
|
|
1207
|
-
);
|
|
1208
|
-
return;
|
|
1209
|
-
}
|
|
1210
|
-
const { willCheckpointAndRestore } = waitForTask.result;
|
|
1211
|
-
await this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
|
|
1212
|
-
if (willCheckpointAndRestore) {
|
|
1213
|
-
if (!this.waitForTaskReplay) {
|
|
1214
|
-
this.waitForTaskReplay = {
|
|
1215
|
-
message,
|
|
1216
|
-
attempt: 1,
|
|
1217
|
-
idempotencyKey: randomUUID()
|
|
1218
|
-
};
|
|
1219
|
-
} else {
|
|
1220
|
-
if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
|
|
1221
|
-
logger.error(
|
|
1222
|
-
"wait for task handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1223
|
-
);
|
|
1224
|
-
return;
|
|
1225
|
-
}
|
|
1226
|
-
this.waitForTaskReplay.attempt++;
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
// MARK: BATCH WAIT
|
|
1231
|
-
async #waitForBatchHandler(message, replayIdempotencyKey) {
|
|
1232
|
-
const waitForBatch = await defaultBackoff.execute(async ({ retry }) => {
|
|
1233
|
-
logger.log("Wait for batch with backoff", { retry });
|
|
1234
|
-
if (!this.attemptFriendlyId) {
|
|
1235
|
-
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1236
|
-
throw new ExponentialBackoff.StopRetrying("No attempt ID");
|
|
1237
|
-
}
|
|
1238
|
-
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_BATCH", {
|
|
1239
|
-
version: "v2",
|
|
1240
|
-
batchFriendlyId: message.batchFriendlyId,
|
|
1241
|
-
runFriendlyIds: message.runFriendlyIds,
|
|
1242
|
-
attemptFriendlyId: this.attemptFriendlyId
|
|
1243
|
-
});
|
|
1244
|
-
});
|
|
1245
|
-
if (!waitForBatch.success) {
|
|
1246
|
-
logger.error("Failed to wait for batch with backoff", {
|
|
1247
|
-
cause: waitForBatch.cause,
|
|
1248
|
-
error: waitForBatch.error
|
|
1249
|
-
});
|
|
1250
|
-
this.#emitUnrecoverableError(
|
|
1251
|
-
"WaitForBatchFailed",
|
|
1252
|
-
`${waitForBatch.cause}: ${waitForBatch.error}`
|
|
1253
|
-
);
|
|
1254
|
-
return;
|
|
1255
|
-
}
|
|
1256
|
-
const { willCheckpointAndRestore } = waitForBatch.result;
|
|
1257
|
-
await this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
|
|
1258
|
-
if (willCheckpointAndRestore) {
|
|
1259
|
-
if (!this.waitForBatchReplay) {
|
|
1260
|
-
this.waitForBatchReplay = {
|
|
1261
|
-
message,
|
|
1262
|
-
attempt: 1,
|
|
1263
|
-
idempotencyKey: randomUUID()
|
|
1264
|
-
};
|
|
1265
|
-
} else {
|
|
1266
|
-
if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
|
|
1267
|
-
logger.error(
|
|
1268
|
-
"wait for task handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1269
|
-
);
|
|
1270
|
-
return;
|
|
1271
|
-
}
|
|
1272
|
-
this.waitForBatchReplay.attempt++;
|
|
1273
|
-
}
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
// MARK: WORKER CREATION
|
|
1277
|
-
#createBackgroundWorker() {
|
|
1278
|
-
const backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
|
1279
|
-
projectConfig: __PROJECT_CONFIG__,
|
|
1280
|
-
env: {
|
|
1281
|
-
...gatherProcessEnv(),
|
|
1282
|
-
TRIGGER_API_URL: this.apiUrl,
|
|
1283
|
-
TRIGGER_SECRET_KEY: this.apiKey,
|
|
1284
|
-
OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"
|
|
1285
|
-
},
|
|
1286
|
-
contentHash: this.contentHash
|
|
1287
|
-
});
|
|
1288
|
-
backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
|
|
1289
|
-
logger.log("onTaskHeartbeat", { attemptFriendlyId });
|
|
1290
|
-
this.#coordinatorSocket.socket.volatile.emit("TASK_HEARTBEAT", {
|
|
1291
|
-
version: "v1",
|
|
1292
|
-
attemptFriendlyId
|
|
1293
|
-
});
|
|
1294
|
-
});
|
|
1295
|
-
backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
|
|
1296
|
-
logger.log("onTaskRunHeartbeat", { runId });
|
|
1297
|
-
this.#coordinatorSocket.socket.volatile.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
|
|
1298
|
-
});
|
|
1299
|
-
backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
|
|
1300
|
-
logger.log("onCreateTaskRunAttempt()", { message });
|
|
1301
|
-
const createAttempt = await defaultBackoff.execute(async ({ retry }) => {
|
|
1302
|
-
logger.log("Create task run attempt with backoff", { retry });
|
|
1303
|
-
return await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CREATE_TASK_RUN_ATTEMPT", {
|
|
1304
|
-
version: "v1",
|
|
1305
|
-
runId: message.runId
|
|
1306
|
-
});
|
|
1307
|
-
});
|
|
1308
|
-
if (!createAttempt.success) {
|
|
1309
|
-
backgroundWorker.attemptCreatedNotification.post({
|
|
1310
|
-
success: false,
|
|
1311
|
-
reason: `Failed to create attempt with backoff due to ${createAttempt.cause}. ${createAttempt.error}`
|
|
1312
|
-
});
|
|
1313
|
-
return;
|
|
1314
|
-
}
|
|
1315
|
-
if (!createAttempt.result.success) {
|
|
1316
|
-
backgroundWorker.attemptCreatedNotification.post({
|
|
1317
|
-
success: false,
|
|
1318
|
-
reason: createAttempt.result.reason
|
|
1319
|
-
});
|
|
1320
|
-
return;
|
|
1321
|
-
}
|
|
1322
|
-
backgroundWorker.attemptCreatedNotification.post({
|
|
1323
|
-
success: true,
|
|
1324
|
-
execution: createAttempt.result.executionPayload.execution
|
|
1325
|
-
});
|
|
1326
|
-
});
|
|
1327
|
-
backgroundWorker.attemptCreatedNotification.attach((message) => {
|
|
1328
|
-
logger.log("attemptCreatedNotification", {
|
|
1329
|
-
success: message.success,
|
|
1330
|
-
...message.success ? {
|
|
1331
|
-
attempt: message.execution.attempt,
|
|
1332
|
-
queue: message.execution.queue,
|
|
1333
|
-
worker: message.execution.worker,
|
|
1334
|
-
machine: message.execution.machine
|
|
1335
|
-
} : {
|
|
1336
|
-
reason: message.reason
|
|
1337
|
-
}
|
|
1338
|
-
});
|
|
1339
|
-
if (!message.success) {
|
|
1340
|
-
return;
|
|
1341
|
-
}
|
|
1342
|
-
this.attemptFriendlyId = message.execution.attempt.id;
|
|
1343
|
-
});
|
|
1344
|
-
backgroundWorker.onWaitForDuration.attach(async (message) => {
|
|
1345
|
-
logger.log("onWaitForDuration", { ...message, drift: Date.now() - message.now });
|
|
1346
|
-
noResume: {
|
|
1347
|
-
const { ms, waitThresholdInMs } = message;
|
|
1348
|
-
const internalTimeout = unboundedTimeout(ms, "internal");
|
|
1349
|
-
const checkpointSafeInternalTimeout = checkpointSafeTimeout(ms);
|
|
1350
|
-
if (ms < waitThresholdInMs) {
|
|
1351
|
-
await internalTimeout;
|
|
1352
|
-
break noResume;
|
|
1353
|
-
}
|
|
1354
|
-
const waitForDuration = await defaultBackoff.execute(async ({ retry }) => {
|
|
1355
|
-
logger.log("Wait for duration with backoff", { retry });
|
|
1356
|
-
if (!this.attemptFriendlyId) {
|
|
1357
|
-
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1358
|
-
throw new ExponentialBackoff.StopRetrying("No attempt ID");
|
|
1359
|
-
}
|
|
1360
|
-
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_DURATION", {
|
|
1361
|
-
...message,
|
|
1362
|
-
attemptFriendlyId: this.attemptFriendlyId
|
|
1363
|
-
});
|
|
1364
|
-
});
|
|
1365
|
-
if (!waitForDuration.success) {
|
|
1366
|
-
logger.error("Failed to wait for duration with backoff", {
|
|
1367
|
-
cause: waitForDuration.cause,
|
|
1368
|
-
error: waitForDuration.error
|
|
1369
|
-
});
|
|
1370
|
-
this.#emitUnrecoverableError(
|
|
1371
|
-
"WaitForDurationFailed",
|
|
1372
|
-
`${waitForDuration.cause}: ${waitForDuration.error}`
|
|
1373
|
-
);
|
|
1374
|
-
return;
|
|
1375
|
-
}
|
|
1376
|
-
const { willCheckpointAndRestore } = waitForDuration.result;
|
|
1377
|
-
if (!willCheckpointAndRestore) {
|
|
1378
|
-
await internalTimeout;
|
|
1379
|
-
break noResume;
|
|
1380
|
-
}
|
|
1381
|
-
await this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
|
|
1382
|
-
await Promise.race([internalTimeout, checkpointSafeInternalTimeout]);
|
|
1383
|
-
try {
|
|
1384
|
-
const { checkpointCanceled } = await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CANCEL_CHECKPOINT", {
|
|
1385
|
-
version: "v2",
|
|
1386
|
-
reason: "WAIT_FOR_DURATION"
|
|
1387
|
-
});
|
|
1388
|
-
logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
|
|
1389
|
-
if (checkpointCanceled) {
|
|
1390
|
-
break noResume;
|
|
1391
|
-
}
|
|
1392
|
-
logger.log("Waiting for external duration resume as we may have been restored");
|
|
1393
|
-
const idempotencyKey = randomUUID();
|
|
1394
|
-
this.durationResumeFallback = { idempotencyKey };
|
|
1395
|
-
setTimeout(() => {
|
|
1396
|
-
if (!this.durationResumeFallback) {
|
|
1397
|
-
logger.error("Already resumed after duration, skipping fallback");
|
|
1398
|
-
return;
|
|
1399
|
-
}
|
|
1400
|
-
if (this.durationResumeFallback.idempotencyKey !== idempotencyKey) {
|
|
1401
|
-
logger.error("Duration resume idempotency key mismatch, skipping fallback");
|
|
1402
|
-
return;
|
|
1403
|
-
}
|
|
1404
|
-
logger.log("Resuming after duration with fallback");
|
|
1405
|
-
this.#resumeAfterDuration();
|
|
1406
|
-
}, 15e3);
|
|
1407
|
-
} catch (error) {
|
|
1408
|
-
logger.debug("Checkpoint cancellation timed out", { error });
|
|
1409
|
-
break noResume;
|
|
1410
|
-
}
|
|
1411
|
-
return;
|
|
1412
|
-
}
|
|
1413
|
-
this.#resumeAfterDuration();
|
|
1414
|
-
});
|
|
1415
|
-
backgroundWorker.onWaitForTask.attach(this.#waitForTaskHandler.bind(this));
|
|
1416
|
-
backgroundWorker.onWaitForBatch.attach(this.#waitForBatchHandler.bind(this));
|
|
1417
|
-
return backgroundWorker;
|
|
1418
|
-
}
|
|
1419
|
-
async #prepareForWait(reason, willCheckpointAndRestore) {
|
|
1420
|
-
logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
|
|
1421
|
-
if (!willCheckpointAndRestore) {
|
|
1422
|
-
return;
|
|
1423
|
-
}
|
|
1424
|
-
this.paused = true;
|
|
1425
|
-
this.nextResumeAfter = reason;
|
|
1426
|
-
this.waitForPostStart = true;
|
|
1427
|
-
await this.#prepareForCheckpoint();
|
|
1428
|
-
}
|
|
1429
|
-
// MARK: RETRY PREP
|
|
1430
|
-
async #prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode) {
|
|
1431
|
-
logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit, exitCode });
|
|
1432
|
-
if (shouldExit) {
|
|
1433
|
-
if (willCheckpointAndRestore) {
|
|
1434
|
-
logger.error("WARNING: Will checkpoint but also requested exit. This won't end well.");
|
|
1435
|
-
}
|
|
1436
|
-
await this.#exitGracefully(false, exitCode);
|
|
1437
|
-
return;
|
|
1438
|
-
}
|
|
1439
|
-
this.paused = false;
|
|
1440
|
-
this.waitForPostStart = false;
|
|
1441
|
-
this.executing = false;
|
|
1442
|
-
this.attemptFriendlyId = void 0;
|
|
1443
|
-
if (!willCheckpointAndRestore) {
|
|
1444
|
-
return;
|
|
1445
|
-
}
|
|
1446
|
-
this.waitForPostStart = true;
|
|
1447
|
-
await this.#prepareForCheckpoint(false);
|
|
1448
|
-
}
|
|
1449
|
-
// MARK: CHECKPOINT PREP
|
|
1450
|
-
async #prepareForCheckpoint(flush = true) {
|
|
1451
|
-
if (flush) {
|
|
1452
|
-
try {
|
|
1453
|
-
await this.#backgroundWorker.flushTelemetry();
|
|
1454
|
-
} catch (error) {
|
|
1455
|
-
logger.error(
|
|
1456
|
-
"Failed to flush telemetry while preparing for checkpoint, will proceed anyway",
|
|
1457
|
-
{ error }
|
|
1458
|
-
);
|
|
1459
|
-
}
|
|
1460
|
-
}
|
|
1461
|
-
try {
|
|
1462
|
-
await this.#backgroundWorker.forceKillOldTaskRunProcesses();
|
|
1463
|
-
} catch (error) {
|
|
1464
|
-
logger.error(
|
|
1465
|
-
"Failed to kill previous worker while preparing for checkpoint, will proceed anyway",
|
|
1466
|
-
{ error }
|
|
1467
|
-
);
|
|
1468
|
-
}
|
|
1469
|
-
this.#readyForCheckpoint();
|
|
1470
|
-
}
|
|
1471
|
-
#resumeAfterDuration() {
|
|
1472
|
-
this.paused = false;
|
|
1473
|
-
this.nextResumeAfter = void 0;
|
|
1474
|
-
this.waitForPostStart = false;
|
|
1475
|
-
this.#backgroundWorker.waitCompletedNotification();
|
|
1476
|
-
}
|
|
1477
|
-
async #readyForLazyAttempt() {
|
|
1478
|
-
const idempotencyKey = randomUUID();
|
|
1479
|
-
this.readyForLazyAttemptReplay = {
|
|
1480
|
-
idempotencyKey
|
|
1481
|
-
};
|
|
1482
|
-
for await (const { delay, retry } of defaultBackoff.min(10).maxRetries(3)) {
|
|
1483
|
-
if (retry > 0) {
|
|
1484
|
-
logger.log("retrying ready for lazy attempt", { retry });
|
|
1485
|
-
}
|
|
1486
|
-
this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
|
|
1487
|
-
version: "v1",
|
|
1488
|
-
runId: this.runId,
|
|
1489
|
-
totalCompletions: this.completed.size
|
|
1490
|
-
});
|
|
1491
|
-
await timeout2(delay.milliseconds);
|
|
1492
|
-
if (!this.readyForLazyAttemptReplay) {
|
|
1493
|
-
logger.error("replay ready for lazy attempt cancelled, discarding", {
|
|
1494
|
-
idempotencyKey
|
|
1495
|
-
});
|
|
1496
|
-
return;
|
|
1497
|
-
}
|
|
1498
|
-
if (idempotencyKey !== this.readyForLazyAttemptReplay.idempotencyKey) {
|
|
1499
|
-
logger.error("replay ready for lazy attempt idempotency key mismatch, discarding", {
|
|
1500
|
-
idempotencyKey,
|
|
1501
|
-
newIdempotencyKey: this.readyForLazyAttemptReplay.idempotencyKey
|
|
1502
|
-
});
|
|
1503
|
-
return;
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
this.#failRun(this.runId, "Failed to receive execute request in a reasonable time");
|
|
1507
|
-
}
|
|
1508
|
-
#readyForCheckpoint() {
|
|
1509
|
-
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
|
1510
|
-
}
|
|
1511
|
-
#failRun(anyRunId, error) {
|
|
1512
|
-
logger.error("Failing run", { anyRunId, error });
|
|
1513
|
-
const completion = {
|
|
1514
|
-
ok: false,
|
|
1515
|
-
id: anyRunId,
|
|
1516
|
-
retry: void 0,
|
|
1517
|
-
error: error instanceof Error ? {
|
|
1518
|
-
type: "BUILT_IN_ERROR",
|
|
1519
|
-
name: error.name,
|
|
1520
|
-
message: error.message,
|
|
1521
|
-
stackTrace: error.stack ?? ""
|
|
1522
|
-
} : {
|
|
1523
|
-
type: "BUILT_IN_ERROR",
|
|
1524
|
-
name: "UnknownError",
|
|
1525
|
-
message: String(error),
|
|
1526
|
-
stackTrace: ""
|
|
1527
|
-
}
|
|
1528
|
-
};
|
|
1529
|
-
this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
|
|
1530
|
-
version: "v1",
|
|
1531
|
-
completion
|
|
1532
|
-
});
|
|
1533
|
-
}
|
|
1534
|
-
// MARK: ATTEMPT COMPLETION
|
|
1535
|
-
async #submitAttemptCompletion(execution, completion, replayIdempotencyKey) {
|
|
1536
|
-
const taskRunCompleted = await defaultBackoff.execute(async ({ retry }) => {
|
|
1537
|
-
logger.log("Submit attempt completion with backoff", { retry });
|
|
1538
|
-
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("TASK_RUN_COMPLETED", {
|
|
1539
|
-
version: "v1",
|
|
1540
|
-
execution,
|
|
1541
|
-
completion
|
|
1542
|
-
});
|
|
1543
|
-
});
|
|
1544
|
-
if (!taskRunCompleted.success) {
|
|
1545
|
-
logger.error("Failed to complete lazy attempt with backoff", {
|
|
1546
|
-
cause: taskRunCompleted.cause,
|
|
1547
|
-
error: taskRunCompleted.error
|
|
1548
|
-
});
|
|
1549
|
-
this.#failRun(execution.run.id, taskRunCompleted.error);
|
|
1550
|
-
return;
|
|
1551
|
-
}
|
|
1552
|
-
const { willCheckpointAndRestore, shouldExit } = taskRunCompleted.result;
|
|
1553
|
-
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1554
|
-
const exitCode = !completion.ok && completion.error.type === "INTERNAL_ERROR" && completion.error.code === TaskRunErrorCodes2.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE ? EXIT_CODE_CHILD_NONZERO : 0;
|
|
1555
|
-
await this.#prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode);
|
|
1556
|
-
if (willCheckpointAndRestore) {
|
|
1557
|
-
if (!this.submitAttemptCompletionReplay) {
|
|
1558
|
-
this.submitAttemptCompletionReplay = {
|
|
1559
|
-
message: {
|
|
1560
|
-
execution,
|
|
1561
|
-
completion
|
|
1562
|
-
},
|
|
1563
|
-
attempt: 1,
|
|
1564
|
-
idempotencyKey: randomUUID()
|
|
1565
|
-
};
|
|
1566
|
-
} else {
|
|
1567
|
-
if (replayIdempotencyKey && replayIdempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
|
|
1568
|
-
logger.error(
|
|
1569
|
-
"attempt completion handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1570
|
-
);
|
|
1571
|
-
return;
|
|
1572
|
-
}
|
|
1573
|
-
this.submitAttemptCompletionReplay.attempt++;
|
|
1574
|
-
}
|
|
1575
|
-
}
|
|
1576
|
-
}
|
|
1577
|
-
#returnValidatedExtraHeaders(headers) {
|
|
1578
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
1579
|
-
if (value === void 0) {
|
|
1580
|
-
throw new Error(`Extra header is undefined: ${key}`);
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1583
|
-
return headers;
|
|
1584
|
-
}
|
|
1585
|
-
// MARK: COORDINATOR SOCKET
|
|
1586
|
-
#createCoordinatorSocket(host) {
|
|
1587
|
-
const extraHeaders = this.#returnValidatedExtraHeaders({
|
|
1588
|
-
"x-machine-name": MACHINE_NAME,
|
|
1589
|
-
"x-pod-name": POD_NAME,
|
|
1590
|
-
"x-trigger-content-hash": this.contentHash,
|
|
1591
|
-
"x-trigger-project-ref": this.projectRef,
|
|
1592
|
-
"x-trigger-env-id": this.envId,
|
|
1593
|
-
"x-trigger-deployment-id": this.deploymentId,
|
|
1594
|
-
"x-trigger-run-id": this.runId,
|
|
1595
|
-
"x-trigger-deployment-version": this.deploymentVersion
|
|
1596
|
-
});
|
|
1597
|
-
if (this.attemptFriendlyId) {
|
|
1598
|
-
extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
|
|
1599
|
-
}
|
|
1600
|
-
logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
|
|
1601
|
-
logger.debug(`connecting with extra headers`, { extraHeaders });
|
|
1602
|
-
const coordinatorConnection = new ZodSocketConnection({
|
|
1603
|
-
namespace: "prod-worker",
|
|
1604
|
-
host,
|
|
1605
|
-
port: COORDINATOR_PORT,
|
|
1606
|
-
clientMessages: ProdWorkerToCoordinatorMessages,
|
|
1607
|
-
serverMessages: CoordinatorToProdWorkerMessages,
|
|
1608
|
-
extraHeaders,
|
|
1609
|
-
ioOptions: {
|
|
1610
|
-
reconnectionDelay: 1e3,
|
|
1611
|
-
reconnectionDelayMax: 3e3
|
|
1612
|
-
},
|
|
1613
|
-
handlers: {
|
|
1614
|
-
RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
|
|
1615
|
-
if (!this.paused) {
|
|
1616
|
-
logger.error("Failed to resume after dependency: Worker not paused");
|
|
1617
|
-
return;
|
|
1618
|
-
}
|
|
1619
|
-
if (completions.length === 0) {
|
|
1620
|
-
logger.error("Failed to resume after dependency: No completions");
|
|
1621
|
-
return;
|
|
1622
|
-
}
|
|
1623
|
-
if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
|
|
1624
|
-
logger.error("Failed to resume after dependency: Invalid next resume", {
|
|
1625
|
-
nextResumeAfter: this.nextResumeAfter
|
|
1626
|
-
});
|
|
1627
|
-
return;
|
|
1628
|
-
}
|
|
1629
|
-
if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
|
|
1630
|
-
logger.error(
|
|
1631
|
-
"Failed to resume after dependency: Waiting for single task but got multiple completions",
|
|
1632
|
-
{
|
|
1633
|
-
completions
|
|
1634
|
-
}
|
|
1635
|
-
);
|
|
1636
|
-
return;
|
|
1637
|
-
}
|
|
1638
|
-
switch (this.nextResumeAfter) {
|
|
1639
|
-
case "WAIT_FOR_TASK": {
|
|
1640
|
-
this.waitForTaskReplay = void 0;
|
|
1641
|
-
break;
|
|
1642
|
-
}
|
|
1643
|
-
case "WAIT_FOR_BATCH": {
|
|
1644
|
-
this.waitForBatchReplay = void 0;
|
|
1645
|
-
break;
|
|
1646
|
-
}
|
|
1647
|
-
}
|
|
1648
|
-
this.paused = false;
|
|
1649
|
-
this.nextResumeAfter = void 0;
|
|
1650
|
-
this.waitForPostStart = false;
|
|
1651
|
-
for (let i = 0; i < completions.length; i++) {
|
|
1652
|
-
const completion = completions[i];
|
|
1653
|
-
if (!completion)
|
|
1654
|
-
continue;
|
|
1655
|
-
this.#backgroundWorker.taskRunCompletedNotification(completion);
|
|
1656
|
-
}
|
|
1657
|
-
},
|
|
1658
|
-
RESUME_AFTER_DURATION: async (message) => {
|
|
1659
|
-
if (!this.paused) {
|
|
1660
|
-
logger.error("worker not paused", {
|
|
1661
|
-
attemptId: message.attemptId
|
|
1662
|
-
});
|
|
1663
|
-
return;
|
|
1664
|
-
}
|
|
1665
|
-
if (this.nextResumeAfter !== "WAIT_FOR_DURATION") {
|
|
1666
|
-
logger.error("not waiting to resume after duration", {
|
|
1667
|
-
nextResumeAfter: this.nextResumeAfter
|
|
1668
|
-
});
|
|
1669
|
-
return;
|
|
1670
|
-
}
|
|
1671
|
-
this.durationResumeFallback = void 0;
|
|
1672
|
-
this.#resumeAfterDuration();
|
|
1673
|
-
},
|
|
1674
|
-
// Deprecated: This will never get called as this worker supports lazy attempts. It's only here for a quick view of the flow old workers use.
|
|
1675
|
-
EXECUTE_TASK_RUN: async ({ executionPayload }) => {
|
|
1676
|
-
if (this.executing) {
|
|
1677
|
-
logger.error("dropping execute request, already executing");
|
|
1678
|
-
return;
|
|
1679
|
-
}
|
|
1680
|
-
if (this.completed.has(executionPayload.execution.attempt.id)) {
|
|
1681
|
-
logger.error("dropping execute request, already completed");
|
|
1682
|
-
return;
|
|
1683
|
-
}
|
|
1684
|
-
this.executing = true;
|
|
1685
|
-
this.attemptFriendlyId = executionPayload.execution.attempt.id;
|
|
1686
|
-
const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
|
|
1687
|
-
logger.log("completed", completion);
|
|
1688
|
-
this.completed.add(executionPayload.execution.attempt.id);
|
|
1689
|
-
const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
|
1690
|
-
version: "v1",
|
|
1691
|
-
execution: executionPayload.execution,
|
|
1692
|
-
completion
|
|
1693
|
-
});
|
|
1694
|
-
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1695
|
-
await this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
|
1696
|
-
},
|
|
1697
|
-
EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
|
|
1698
|
-
this.readyForLazyAttemptReplay = void 0;
|
|
1699
|
-
if (this.executing) {
|
|
1700
|
-
logger.error("dropping execute request, already executing");
|
|
1701
|
-
return;
|
|
1702
|
-
}
|
|
1703
|
-
const attemptCount = message.lazyPayload.attemptCount ?? 0;
|
|
1704
|
-
logger.log("execute attempt counts", { attemptCount, completed: this.completed.size });
|
|
1705
|
-
if (this.completed.size > 0 && this.completed.size >= attemptCount + 1) {
|
|
1706
|
-
logger.error("dropping execute request, already completed");
|
|
1707
|
-
return;
|
|
1708
|
-
}
|
|
1709
|
-
this.executing = true;
|
|
1710
|
-
try {
|
|
1711
|
-
const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
|
|
1712
|
-
logger.log("completed", completion);
|
|
1713
|
-
this.completed.add(execution.attempt.id);
|
|
1714
|
-
await this.#submitAttemptCompletion(execution, completion);
|
|
1715
|
-
} catch (error) {
|
|
1716
|
-
logger.error("Failed to complete lazy attempt", {
|
|
1717
|
-
error
|
|
1718
|
-
});
|
|
1719
|
-
this.#failRun(message.lazyPayload.runId, error);
|
|
1720
|
-
}
|
|
1721
|
-
},
|
|
1722
|
-
REQUEST_ATTEMPT_CANCELLATION: async (message) => {
|
|
1723
|
-
if (!this.executing) {
|
|
1724
|
-
logger.log("dropping cancel request, not executing", { status: this.#status });
|
|
1725
|
-
return;
|
|
1726
|
-
}
|
|
1727
|
-
logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
|
|
1728
|
-
await this.#backgroundWorker.cancelAttempt(message.attemptId);
|
|
1729
|
-
},
|
|
1730
|
-
REQUEST_EXIT: async (message) => {
|
|
1731
|
-
if (message.version === "v2" && message.delayInMs) {
|
|
1732
|
-
logger.log("exit requested with delay", { delayInMs: message.delayInMs });
|
|
1733
|
-
await timeout2(message.delayInMs);
|
|
1734
|
-
}
|
|
1735
|
-
this.#coordinatorSocket.close();
|
|
1736
|
-
process.exit(0);
|
|
1737
|
-
},
|
|
1738
|
-
READY_FOR_RETRY: async (message) => {
|
|
1739
|
-
if (this.completed.size < 1) {
|
|
1740
|
-
logger.error("Received READY_FOR_RETRY but no completions yet. This is a bug.");
|
|
1741
|
-
return;
|
|
1742
|
-
}
|
|
1743
|
-
this.submitAttemptCompletionReplay = void 0;
|
|
1744
|
-
await this.#readyForLazyAttempt();
|
|
1745
|
-
}
|
|
1746
|
-
},
|
|
1747
|
-
// MARK: ON CONNECTION
|
|
1748
|
-
onConnection: async (socket, handler, sender, logger2) => {
|
|
1749
|
-
logger2.log("connected to coordinator", {
|
|
1750
|
-
status: this.#status,
|
|
1751
|
-
connectionCount: ++this.connectionCount
|
|
1752
|
-
});
|
|
1753
|
-
socket.emit("SET_STATE", { version: "v1", attemptFriendlyId: this.attemptFriendlyId });
|
|
1754
|
-
try {
|
|
1755
|
-
if (this.waitForPostStart) {
|
|
1756
|
-
logger2.log("skip connection handler, waiting for post start hook");
|
|
1757
|
-
return;
|
|
1758
|
-
}
|
|
1759
|
-
if (this.paused) {
|
|
1760
|
-
if (!this.nextResumeAfter) {
|
|
1761
|
-
logger2.error("Missing next resume reason", { status: this.#status });
|
|
1762
|
-
this.#emitUnrecoverableError(
|
|
1763
|
-
"NoNextResume",
|
|
1764
|
-
"Next resume reason not set while resuming from paused state"
|
|
1765
|
-
);
|
|
1766
|
-
return;
|
|
1767
|
-
}
|
|
1768
|
-
if (!this.attemptFriendlyId) {
|
|
1769
|
-
logger2.error("Missing friendly ID", { status: this.#status });
|
|
1770
|
-
this.#emitUnrecoverableError(
|
|
1771
|
-
"NoAttemptId",
|
|
1772
|
-
"Attempt ID not set while resuming from paused state"
|
|
1773
|
-
);
|
|
1774
|
-
return;
|
|
1775
|
-
}
|
|
1776
|
-
socket.emit("READY_FOR_RESUME", {
|
|
1777
|
-
version: "v1",
|
|
1778
|
-
attemptFriendlyId: this.attemptFriendlyId,
|
|
1779
|
-
type: this.nextResumeAfter
|
|
1780
|
-
});
|
|
1781
|
-
return;
|
|
1782
|
-
}
|
|
1783
|
-
if (process.env.INDEX_TASKS === "true") {
|
|
1784
|
-
const failIndex = (error) => {
|
|
1785
|
-
socket.emit("INDEXING_FAILED", {
|
|
1786
|
-
version: "v1",
|
|
1787
|
-
deploymentId: this.deploymentId,
|
|
1788
|
-
error
|
|
1789
|
-
});
|
|
1790
|
-
};
|
|
1791
|
-
process.removeAllListeners("uncaughtException");
|
|
1792
|
-
process.on("uncaughtException", (error) => {
|
|
1793
|
-
console.error("Uncaught exception while indexing", error);
|
|
1794
|
-
failIndex(error);
|
|
1795
|
-
});
|
|
1796
|
-
try {
|
|
1797
|
-
const taskResources = await this.#initializeWorker();
|
|
1798
|
-
const indexTasks = await defaultBackoff.maxRetries(3).execute(async () => {
|
|
1799
|
-
return await socket.timeout(2e4).emitWithAck("INDEX_TASKS", {
|
|
1800
|
-
version: "v2",
|
|
1801
|
-
deploymentId: this.deploymentId,
|
|
1802
|
-
...taskResources,
|
|
1803
|
-
supportsLazyAttempts: true
|
|
1804
|
-
});
|
|
1805
|
-
});
|
|
1806
|
-
if (!indexTasks.success || !indexTasks.result.success) {
|
|
1807
|
-
logger2.error("indexing failure, shutting down..", { indexTasks });
|
|
1808
|
-
process.exit(1);
|
|
1809
|
-
} else {
|
|
1810
|
-
logger2.info("indexing done, shutting down..");
|
|
1811
|
-
process.exit(0);
|
|
1812
|
-
}
|
|
1813
|
-
} catch (e) {
|
|
1814
|
-
const stderr = this.#backgroundWorker.stderr.join("\n");
|
|
1815
|
-
if (e instanceof TaskMetadataParseError) {
|
|
1816
|
-
logger2.error("tasks metadata parse error", {
|
|
1817
|
-
zodIssues: e.zodIssues,
|
|
1818
|
-
tasks: e.tasks
|
|
1819
|
-
});
|
|
1820
|
-
failIndex({
|
|
1821
|
-
name: "TaskMetadataParseError",
|
|
1822
|
-
message: "There was an error parsing the task metadata",
|
|
1823
|
-
stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
|
|
1824
|
-
stderr
|
|
1825
|
-
});
|
|
1826
|
-
} else if (e instanceof UncaughtExceptionError) {
|
|
1827
|
-
const error = {
|
|
1828
|
-
name: e.originalError.name,
|
|
1829
|
-
message: e.originalError.message,
|
|
1830
|
-
stack: e.originalError.stack,
|
|
1831
|
-
stderr
|
|
1832
|
-
};
|
|
1833
|
-
logger2.error("uncaught exception", { originalError: error });
|
|
1834
|
-
failIndex(error);
|
|
1835
|
-
} else if (e instanceof Error) {
|
|
1836
|
-
const error = {
|
|
1837
|
-
name: e.name,
|
|
1838
|
-
message: e.message,
|
|
1839
|
-
stack: e.stack,
|
|
1840
|
-
stderr
|
|
1841
|
-
};
|
|
1842
|
-
logger2.error("error", { error });
|
|
1843
|
-
failIndex(error);
|
|
1844
|
-
} else if (typeof e === "string") {
|
|
1845
|
-
logger2.error("string error", { error: { message: e } });
|
|
1846
|
-
failIndex({
|
|
1847
|
-
name: "Error",
|
|
1848
|
-
message: e,
|
|
1849
|
-
stderr
|
|
1850
|
-
});
|
|
1851
|
-
} else {
|
|
1852
|
-
logger2.error("unknown error", { error: e });
|
|
1853
|
-
failIndex({
|
|
1854
|
-
name: "Error",
|
|
1855
|
-
message: "Unknown error",
|
|
1856
|
-
stderr
|
|
1857
|
-
});
|
|
1858
|
-
}
|
|
1859
|
-
await timeout2(1e3);
|
|
1860
|
-
process.exit(EXIT_CODE_ALREADY_HANDLED);
|
|
1861
|
-
}
|
|
1862
|
-
}
|
|
1863
|
-
if (this.executing) {
|
|
1864
|
-
return;
|
|
1865
|
-
}
|
|
1866
|
-
process.removeAllListeners("uncaughtException");
|
|
1867
|
-
process.on("uncaughtException", (error) => {
|
|
1868
|
-
console.error("Uncaught exception during run", error);
|
|
1869
|
-
this.#failRun(this.runId, error);
|
|
1870
|
-
});
|
|
1871
|
-
await this.#readyForLazyAttempt();
|
|
1872
|
-
} catch (error) {
|
|
1873
|
-
logger2.error("connection handler error", { error });
|
|
1874
|
-
} finally {
|
|
1875
|
-
if (this.connectionCount === 1) {
|
|
1876
|
-
return;
|
|
1877
|
-
}
|
|
1878
|
-
this.#handleReplays();
|
|
1879
|
-
}
|
|
1880
|
-
},
|
|
1881
|
-
onError: async (socket, err, logger2) => {
|
|
1882
|
-
logger2.error("onError", {
|
|
1883
|
-
error: {
|
|
1884
|
-
name: err.name,
|
|
1885
|
-
message: err.message
|
|
1886
|
-
}
|
|
1887
|
-
});
|
|
1888
|
-
}
|
|
1889
|
-
});
|
|
1890
|
-
return coordinatorConnection;
|
|
1891
|
-
}
|
|
1892
|
-
// MARK: REPLAYS
|
|
1893
|
-
async #handleReplays() {
|
|
1894
|
-
const backoff = new ExponentialBackoff().type("FullJitter").maxRetries(3);
|
|
1895
|
-
const replayCancellationDelay = 2e4;
|
|
1896
|
-
if (this.waitForTaskReplay) {
|
|
1897
|
-
logger.log("replaying wait for task", { ...this.waitForTaskReplay });
|
|
1898
|
-
const { idempotencyKey, message, attempt } = this.waitForTaskReplay;
|
|
1899
|
-
await timeout2(replayCancellationDelay);
|
|
1900
|
-
if (!this.waitForTaskReplay) {
|
|
1901
|
-
logger.error("wait for task replay cancelled, discarding", {
|
|
1902
|
-
originalMessage: { idempotencyKey, message, attempt }
|
|
1903
|
-
});
|
|
1904
|
-
return;
|
|
1905
|
-
}
|
|
1906
|
-
if (idempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
|
|
1907
|
-
logger.error("wait for task replay idempotency key mismatch, discarding", {
|
|
1908
|
-
originalMessage: { idempotencyKey, message, attempt },
|
|
1909
|
-
newMessage: this.waitForTaskReplay
|
|
1910
|
-
});
|
|
1911
|
-
return;
|
|
1912
|
-
}
|
|
1913
|
-
try {
|
|
1914
|
-
await backoff.wait(attempt + 1);
|
|
1915
|
-
await this.#waitForTaskHandler(message);
|
|
1916
|
-
} catch (error) {
|
|
1917
|
-
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1918
|
-
logger.error("wait for task replay retry limit exceeded", { error });
|
|
1919
|
-
} else {
|
|
1920
|
-
logger.error("wait for task replay error", { error });
|
|
1921
|
-
}
|
|
1922
|
-
}
|
|
1923
|
-
return;
|
|
1924
|
-
}
|
|
1925
|
-
if (this.waitForBatchReplay) {
|
|
1926
|
-
logger.log("replaying wait for batch", {
|
|
1927
|
-
...this.waitForBatchReplay,
|
|
1928
|
-
cancellationDelay: replayCancellationDelay
|
|
1929
|
-
});
|
|
1930
|
-
const { idempotencyKey, message, attempt } = this.waitForBatchReplay;
|
|
1931
|
-
await timeout2(replayCancellationDelay);
|
|
1932
|
-
if (!this.waitForBatchReplay) {
|
|
1933
|
-
logger.error("wait for batch replay cancelled, discarding", {
|
|
1934
|
-
originalMessage: { idempotencyKey, message, attempt }
|
|
1935
|
-
});
|
|
1936
|
-
return;
|
|
1937
|
-
}
|
|
1938
|
-
if (idempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
|
|
1939
|
-
logger.error("wait for batch replay idempotency key mismatch, discarding", {
|
|
1940
|
-
originalMessage: { idempotencyKey, message, attempt },
|
|
1941
|
-
newMessage: this.waitForBatchReplay
|
|
1942
|
-
});
|
|
1943
|
-
return;
|
|
1944
|
-
}
|
|
1945
|
-
try {
|
|
1946
|
-
await backoff.wait(attempt + 1);
|
|
1947
|
-
await this.#waitForBatchHandler(message);
|
|
1948
|
-
} catch (error) {
|
|
1949
|
-
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1950
|
-
logger.error("wait for batch replay retry limit exceeded", { error });
|
|
1951
|
-
} else {
|
|
1952
|
-
logger.error("wait for batch replay error", { error });
|
|
1953
|
-
}
|
|
1954
|
-
}
|
|
1955
|
-
return;
|
|
1956
|
-
}
|
|
1957
|
-
if (this.submitAttemptCompletionReplay) {
|
|
1958
|
-
logger.log("replaying attempt completion", {
|
|
1959
|
-
...this.submitAttemptCompletionReplay,
|
|
1960
|
-
cancellationDelay: replayCancellationDelay
|
|
1961
|
-
});
|
|
1962
|
-
const { idempotencyKey, message, attempt } = this.submitAttemptCompletionReplay;
|
|
1963
|
-
await timeout2(replayCancellationDelay);
|
|
1964
|
-
if (!this.submitAttemptCompletionReplay) {
|
|
1965
|
-
logger.error("attempt completion replay cancelled, discarding", {
|
|
1966
|
-
originalMessage: { idempotencyKey, message, attempt }
|
|
1967
|
-
});
|
|
1968
|
-
return;
|
|
1969
|
-
}
|
|
1970
|
-
if (idempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
|
|
1971
|
-
logger.error("attempt completion replay idempotency key mismatch, discarding", {
|
|
1972
|
-
originalMessage: { idempotencyKey, message, attempt },
|
|
1973
|
-
newMessage: this.submitAttemptCompletionReplay
|
|
1974
|
-
});
|
|
1975
|
-
return;
|
|
1976
|
-
}
|
|
1977
|
-
try {
|
|
1978
|
-
await backoff.wait(attempt + 1);
|
|
1979
|
-
await this.#submitAttemptCompletion(message.execution, message.completion, idempotencyKey);
|
|
1980
|
-
} catch (error) {
|
|
1981
|
-
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1982
|
-
logger.error("attempt completion replay retry limit exceeded", { error });
|
|
1983
|
-
} else {
|
|
1984
|
-
logger.error("attempt completion replay error", { error });
|
|
1985
|
-
}
|
|
1986
|
-
}
|
|
1987
|
-
return;
|
|
1988
|
-
}
|
|
1989
|
-
}
|
|
1990
|
-
// MARK: HTTP SERVER
|
|
1991
|
-
#createHttpServer() {
|
|
1992
|
-
const httpServer = createServer(async (req, res) => {
|
|
1993
|
-
logger.log(`[${req.method}]`, req.url);
|
|
1994
|
-
const reply = new HttpReply(res);
|
|
1995
|
-
try {
|
|
1996
|
-
const url = new URL(req.url ?? "", `http://${req.headers.host}`);
|
|
1997
|
-
switch (url.pathname) {
|
|
1998
|
-
case "/health": {
|
|
1999
|
-
return reply.text("ok");
|
|
2000
|
-
}
|
|
2001
|
-
case "/status": {
|
|
2002
|
-
return reply.json(this.#status);
|
|
2003
|
-
}
|
|
2004
|
-
case "/connect": {
|
|
2005
|
-
this.#coordinatorSocket.connect();
|
|
2006
|
-
return reply.text("Connected to coordinator");
|
|
2007
|
-
}
|
|
2008
|
-
case "/close": {
|
|
2009
|
-
this.#coordinatorSocket.close();
|
|
2010
|
-
this.connectionCount = 0;
|
|
2011
|
-
return reply.text("Disconnected from coordinator");
|
|
2012
|
-
}
|
|
2013
|
-
case "/test": {
|
|
2014
|
-
await this.#coordinatorSocket.socket.timeout(1e4).emitWithAck("TEST", {
|
|
2015
|
-
version: "v1"
|
|
2016
|
-
});
|
|
2017
|
-
return reply.text("Received ACK from coordinator");
|
|
2018
|
-
}
|
|
2019
|
-
case "/preStop": {
|
|
2020
|
-
const cause = PreStopCauses.safeParse(url.searchParams.get("cause"));
|
|
2021
|
-
if (!cause.success) {
|
|
2022
|
-
logger.error("Failed to parse cause", { cause });
|
|
2023
|
-
return reply.text("Failed to parse cause", 400);
|
|
2024
|
-
}
|
|
2025
|
-
switch (cause.data) {
|
|
2026
|
-
case "terminate": {
|
|
2027
|
-
break;
|
|
2028
|
-
}
|
|
2029
|
-
default: {
|
|
2030
|
-
logger.error("Unhandled cause", { cause: cause.data });
|
|
2031
|
-
break;
|
|
2032
|
-
}
|
|
2033
|
-
}
|
|
2034
|
-
return reply.text("preStop ok");
|
|
2035
|
-
}
|
|
2036
|
-
case "/postStart": {
|
|
2037
|
-
const cause = PostStartCauses.safeParse(url.searchParams.get("cause"));
|
|
2038
|
-
if (!cause.success) {
|
|
2039
|
-
logger.error("Failed to parse cause", { cause });
|
|
2040
|
-
return reply.text("Failed to parse cause", 400);
|
|
2041
|
-
}
|
|
2042
|
-
switch (cause.data) {
|
|
2043
|
-
case "index": {
|
|
2044
|
-
break;
|
|
2045
|
-
}
|
|
2046
|
-
case "create": {
|
|
2047
|
-
break;
|
|
2048
|
-
}
|
|
2049
|
-
case "restore": {
|
|
2050
|
-
await this.#reconnectAfterPostStart();
|
|
2051
|
-
break;
|
|
2052
|
-
}
|
|
2053
|
-
default: {
|
|
2054
|
-
logger.error("Unhandled cause", { cause: cause.data });
|
|
2055
|
-
break;
|
|
2056
|
-
}
|
|
2057
|
-
}
|
|
2058
|
-
return reply.text("postStart ok");
|
|
2059
|
-
}
|
|
2060
|
-
default: {
|
|
2061
|
-
return reply.empty(404);
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
} catch (error) {
|
|
2065
|
-
logger.error("HTTP server error", { error });
|
|
2066
|
-
reply.empty(500);
|
|
2067
|
-
}
|
|
2068
|
-
});
|
|
2069
|
-
httpServer.on("clientError", (err, socket) => {
|
|
2070
|
-
socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
2071
|
-
});
|
|
2072
|
-
httpServer.on("listening", () => {
|
|
2073
|
-
logger.log("http server listening on port", this.#httpPort);
|
|
2074
|
-
});
|
|
2075
|
-
httpServer.on("error", async (error) => {
|
|
2076
|
-
if (error.code != "EADDRINUSE") {
|
|
2077
|
-
return;
|
|
2078
|
-
}
|
|
2079
|
-
logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
|
|
2080
|
-
this.#httpPort = getRandomPortNumber();
|
|
2081
|
-
await timeout2(100);
|
|
2082
|
-
this.start();
|
|
2083
|
-
});
|
|
2084
|
-
return httpServer;
|
|
2085
|
-
}
|
|
2086
|
-
async #initializeWorker() {
|
|
2087
|
-
const envVars = await this.#fetchEnvironmentVariables();
|
|
2088
|
-
await this.#backgroundWorker.initialize({ env: envVars });
|
|
2089
|
-
let packageVersion;
|
|
2090
|
-
const taskResources = [];
|
|
2091
|
-
if (!this.#backgroundWorker.tasks || this.#backgroundWorker.tasks.length === 0) {
|
|
2092
|
-
throw new Error(
|
|
2093
|
-
`Background Worker started without tasks. Searched in: ${__PROJECT_CONFIG__.triggerDirectories?.join(
|
|
2094
|
-
", "
|
|
2095
|
-
)}`
|
|
2096
|
-
);
|
|
2097
|
-
}
|
|
2098
|
-
for (const task of this.#backgroundWorker.tasks) {
|
|
2099
|
-
taskResources.push(task);
|
|
2100
|
-
packageVersion = task.packageVersion;
|
|
2101
|
-
}
|
|
2102
|
-
if (!packageVersion) {
|
|
2103
|
-
throw new Error(`Background Worker started without package version`);
|
|
2104
|
-
}
|
|
2105
|
-
return {
|
|
2106
|
-
packageVersion,
|
|
2107
|
-
tasks: taskResources
|
|
2108
|
-
};
|
|
2109
|
-
}
|
|
2110
|
-
async #fetchEnvironmentVariables() {
|
|
2111
|
-
const response = await fetch(`${this.apiUrl}/api/v1/projects/${this.projectRef}/envvars`, {
|
|
2112
|
-
method: "GET",
|
|
2113
|
-
headers: {
|
|
2114
|
-
Authorization: `Bearer ${this.apiKey}`
|
|
2115
|
-
}
|
|
2116
|
-
});
|
|
2117
|
-
if (!response.ok) {
|
|
2118
|
-
return {};
|
|
2119
|
-
}
|
|
2120
|
-
const data = await response.json();
|
|
2121
|
-
return data?.variables ?? {};
|
|
2122
|
-
}
|
|
2123
|
-
get #status() {
|
|
2124
|
-
return {
|
|
2125
|
-
executing: this.executing,
|
|
2126
|
-
paused: this.paused,
|
|
2127
|
-
completed: this.completed.size,
|
|
2128
|
-
nextResumeAfter: this.nextResumeAfter,
|
|
2129
|
-
waitForPostStart: this.waitForPostStart,
|
|
2130
|
-
attemptFriendlyId: this.attemptFriendlyId,
|
|
2131
|
-
waitForTaskReplay: this.waitForTaskReplay,
|
|
2132
|
-
waitForBatchReplay: this.waitForBatchReplay
|
|
2133
|
-
};
|
|
2134
|
-
}
|
|
2135
|
-
#emitUnrecoverableError(name, message) {
|
|
2136
|
-
this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
|
|
2137
|
-
version: "v1",
|
|
2138
|
-
error: {
|
|
2139
|
-
name,
|
|
2140
|
-
message
|
|
2141
|
-
}
|
|
2142
|
-
});
|
|
2143
|
-
}
|
|
2144
|
-
start() {
|
|
2145
|
-
this.#httpServer.listen(this.#httpPort, this.host);
|
|
2146
|
-
}
|
|
2147
|
-
};
|
|
2148
|
-
var prodWorker = new ProdWorker(HTTP_SERVER_PORT);
|
|
2149
|
-
prodWorker.start();
|
|
2150
|
-
function gatherProcessEnv() {
|
|
2151
|
-
const env = {
|
|
2152
|
-
NODE_ENV: process.env.NODE_ENV ?? "production",
|
|
2153
|
-
PATH: process.env.PATH,
|
|
2154
|
-
USER: process.env.USER,
|
|
2155
|
-
SHELL: process.env.SHELL,
|
|
2156
|
-
LANG: process.env.LANG,
|
|
2157
|
-
TERM: process.env.TERM,
|
|
2158
|
-
NODE_PATH: process.env.NODE_PATH,
|
|
2159
|
-
HOME: process.env.HOME,
|
|
2160
|
-
NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS
|
|
2161
|
-
};
|
|
2162
|
-
return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== void 0));
|
|
2163
|
-
}
|