trigger.dev 0.0.0-v3-prerelease-20240801102736 → 0.0.0-v3-prerelease-20240912191953

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/dist/esm/apiClient.d.ts +190 -0
  2. package/dist/esm/apiClient.js +239 -0
  3. package/dist/esm/apiClient.js.map +1 -0
  4. package/dist/esm/build/buildWorker.d.ts +68 -0
  5. package/dist/esm/build/buildWorker.js +159 -0
  6. package/dist/esm/build/buildWorker.js.map +1 -0
  7. package/dist/esm/build/bundle.d.ts +36 -0
  8. package/dist/esm/build/bundle.js +197 -0
  9. package/dist/esm/build/bundle.js.map +1 -0
  10. package/dist/esm/build/extensions.d.ts +15 -0
  11. package/dist/esm/build/extensions.js +158 -0
  12. package/dist/esm/build/extensions.js.map +1 -0
  13. package/dist/esm/build/externals.d.ts +15 -0
  14. package/dist/esm/build/externals.js +285 -0
  15. package/dist/esm/build/externals.js.map +1 -0
  16. package/dist/esm/build/manifests.d.ts +2 -0
  17. package/dist/esm/build/manifests.js +22 -0
  18. package/dist/esm/build/manifests.js.map +1 -0
  19. package/dist/esm/build/packageModules.d.ts +18 -0
  20. package/dist/esm/build/packageModules.js +84 -0
  21. package/dist/esm/build/packageModules.js.map +1 -0
  22. package/dist/esm/build/plugins.d.ts +6 -0
  23. package/dist/esm/build/plugins.js +67 -0
  24. package/dist/esm/build/plugins.js.map +1 -0
  25. package/dist/esm/build/resolveModule.d.ts +1 -0
  26. package/dist/esm/build/resolveModule.js +9 -0
  27. package/dist/esm/build/resolveModule.js.map +1 -0
  28. package/dist/esm/cli/common.d.ts +30 -0
  29. package/dist/esm/cli/common.js +85 -0
  30. package/dist/esm/cli/common.js.map +1 -0
  31. package/dist/esm/cli/index.d.ts +2 -0
  32. package/dist/esm/cli/index.js +27 -0
  33. package/dist/esm/cli/index.js.map +1 -0
  34. package/dist/esm/commands/deploy.d.ts +5 -0
  35. package/dist/esm/commands/deploy.js +402 -0
  36. package/dist/esm/commands/deploy.js.map +1 -0
  37. package/dist/esm/commands/dev.d.ts +37 -0
  38. package/dist/esm/commands/dev.js +128 -0
  39. package/dist/esm/commands/dev.js.map +1 -0
  40. package/dist/esm/commands/init.d.ts +3 -0
  41. package/dist/esm/commands/init.js +417 -0
  42. package/dist/esm/commands/init.js.map +1 -0
  43. package/dist/esm/commands/list-profiles.d.ts +23 -0
  44. package/dist/esm/commands/list-profiles.js +40 -0
  45. package/dist/esm/commands/list-profiles.js.map +1 -0
  46. package/dist/esm/commands/login.d.ts +28 -0
  47. package/dist/esm/commands/login.js +275 -0
  48. package/dist/esm/commands/login.js.map +1 -0
  49. package/dist/esm/commands/logout.d.ts +23 -0
  50. package/dist/esm/commands/logout.js +28 -0
  51. package/dist/esm/commands/logout.js.map +1 -0
  52. package/dist/esm/commands/update.d.ts +24 -0
  53. package/dist/esm/commands/update.js +233 -0
  54. package/dist/esm/commands/update.js.map +1 -0
  55. package/dist/esm/commands/whoami.d.ts +34 -0
  56. package/dist/esm/commands/whoami.js +70 -0
  57. package/dist/esm/commands/whoami.js.map +1 -0
  58. package/dist/esm/config.d.ts +22 -0
  59. package/dist/esm/config.js +204 -0
  60. package/dist/esm/config.js.map +1 -0
  61. package/dist/esm/consts.d.ts +4 -0
  62. package/dist/esm/consts.js +5 -0
  63. package/dist/esm/consts.js.map +1 -0
  64. package/dist/esm/deploy/buildImage.d.ts +69 -0
  65. package/dist/esm/deploy/buildImage.js +472 -0
  66. package/dist/esm/deploy/buildImage.js.map +1 -0
  67. package/dist/esm/deploy/logs.d.ts +19 -0
  68. package/dist/esm/deploy/logs.js +96 -0
  69. package/dist/esm/deploy/logs.js.map +1 -0
  70. package/dist/esm/dev/backgroundWorker.d.ts +114 -0
  71. package/dist/esm/dev/backgroundWorker.js +412 -0
  72. package/dist/esm/dev/backgroundWorker.js.map +1 -0
  73. package/dist/esm/dev/devOutput.d.ts +9 -0
  74. package/dist/esm/dev/devOutput.js +137 -0
  75. package/dist/esm/dev/devOutput.js.map +1 -0
  76. package/dist/esm/dev/devSession.d.ts +17 -0
  77. package/dist/esm/dev/devSession.js +156 -0
  78. package/dist/esm/dev/devSession.js.map +1 -0
  79. package/dist/esm/dev/workerRuntime.d.ts +16 -0
  80. package/dist/esm/dev/workerRuntime.js +258 -0
  81. package/dist/esm/dev/workerRuntime.js.map +1 -0
  82. package/dist/esm/entryPoints/deploy-index-controller.d.ts +1 -0
  83. package/dist/esm/entryPoints/deploy-index-controller.js +85 -0
  84. package/dist/esm/entryPoints/deploy-index-controller.js.map +1 -0
  85. package/dist/esm/entryPoints/deploy-index-worker.d.ts +1 -0
  86. package/dist/esm/entryPoints/deploy-index-worker.js +104 -0
  87. package/dist/esm/entryPoints/deploy-index-worker.js.map +1 -0
  88. package/dist/esm/entryPoints/deploy-run-controller.d.ts +1 -0
  89. package/dist/esm/entryPoints/deploy-run-controller.js +996 -0
  90. package/dist/esm/entryPoints/deploy-run-controller.js.map +1 -0
  91. package/dist/esm/entryPoints/deploy-run-worker.d.ts +1 -0
  92. package/dist/esm/entryPoints/deploy-run-worker.js +297 -0
  93. package/dist/esm/entryPoints/deploy-run-worker.js.map +1 -0
  94. package/dist/esm/entryPoints/dev-index-worker.d.ts +1 -0
  95. package/dist/esm/entryPoints/dev-index-worker.js +104 -0
  96. package/dist/esm/entryPoints/dev-index-worker.js.map +1 -0
  97. package/dist/esm/entryPoints/dev-run-worker.d.ts +1 -0
  98. package/dist/esm/entryPoints/dev-run-worker.js +254 -0
  99. package/dist/esm/entryPoints/dev-run-worker.js.map +1 -0
  100. package/dist/esm/entryPoints/loader.d.ts +1 -0
  101. package/dist/esm/entryPoints/loader.js +16 -0
  102. package/dist/esm/entryPoints/loader.js.map +1 -0
  103. package/dist/esm/executions/taskRunProcess.d.ts +115 -0
  104. package/dist/esm/executions/taskRunProcess.js +254 -0
  105. package/dist/esm/executions/taskRunProcess.js.map +1 -0
  106. package/dist/esm/imports/magicast.d.ts +2 -0
  107. package/dist/esm/imports/magicast.js +5 -0
  108. package/dist/esm/imports/magicast.js.map +1 -0
  109. package/dist/esm/imports/xdg-app-paths.d.ts +2 -0
  110. package/dist/esm/imports/xdg-app-paths.js +3 -0
  111. package/dist/esm/imports/xdg-app-paths.js.map +1 -0
  112. package/dist/{index.d.ts → esm/index.d.ts} +1 -0
  113. package/dist/esm/index.js +17 -0
  114. package/dist/esm/index.js.map +1 -0
  115. package/dist/esm/indexing/indexWorkerManifest.d.ts +85 -0
  116. package/dist/esm/indexing/indexWorkerManifest.js +76 -0
  117. package/dist/esm/indexing/indexWorkerManifest.js.map +1 -0
  118. package/dist/esm/indexing/registerTasks.d.ts +2 -0
  119. package/dist/esm/indexing/registerTasks.js +62 -0
  120. package/dist/esm/indexing/registerTasks.js.map +1 -0
  121. package/dist/esm/package.json +3 -0
  122. package/dist/esm/runtimes/bun.d.ts +2 -0
  123. package/dist/esm/runtimes/bun.js +11 -0
  124. package/dist/esm/runtimes/bun.js.map +1 -0
  125. package/dist/esm/shims/esm.d.ts +1 -0
  126. package/dist/esm/shims/esm.js +9 -0
  127. package/dist/esm/shims/esm.js.map +1 -0
  128. package/dist/esm/sourceDir.d.ts +1 -0
  129. package/dist/esm/sourceDir.js +4 -0
  130. package/dist/esm/sourceDir.js.map +1 -0
  131. package/dist/esm/telemetry/tracing.d.ts +3 -0
  132. package/dist/esm/telemetry/tracing.js +58 -0
  133. package/dist/esm/telemetry/tracing.js.map +1 -0
  134. package/dist/esm/types.d.ts +6 -0
  135. package/dist/esm/types.js +2 -0
  136. package/dist/esm/types.js.map +1 -0
  137. package/dist/esm/utilities/assertExhaustive.d.ts +1 -0
  138. package/dist/esm/utilities/assertExhaustive.js +4 -0
  139. package/dist/esm/utilities/assertExhaustive.js.map +1 -0
  140. package/dist/esm/utilities/buildManifest.d.ts +2 -0
  141. package/dist/esm/utilities/buildManifest.js +9 -0
  142. package/dist/esm/utilities/buildManifest.js.map +1 -0
  143. package/dist/esm/utilities/cliOutput.d.ts +20 -0
  144. package/dist/esm/utilities/cliOutput.js +88 -0
  145. package/dist/esm/utilities/cliOutput.js.map +1 -0
  146. package/dist/esm/utilities/configFiles.d.ts +29 -0
  147. package/dist/esm/utilities/configFiles.js +65 -0
  148. package/dist/esm/utilities/configFiles.js.map +1 -0
  149. package/dist/esm/utilities/createFileFromTemplate.d.ts +16 -0
  150. package/dist/esm/utilities/createFileFromTemplate.js +50 -0
  151. package/dist/esm/utilities/createFileFromTemplate.js.map +1 -0
  152. package/dist/esm/utilities/deployErrors.d.ts +19 -0
  153. package/dist/esm/utilities/deployErrors.js +117 -0
  154. package/dist/esm/utilities/deployErrors.js.map +1 -0
  155. package/dist/esm/utilities/dotEnv.d.ts +4 -0
  156. package/dist/esm/utilities/dotEnv.js +28 -0
  157. package/dist/esm/utilities/dotEnv.js.map +1 -0
  158. package/dist/esm/utilities/eventBus.d.ts +14 -0
  159. package/dist/esm/utilities/eventBus.js +3 -0
  160. package/dist/esm/utilities/eventBus.js.map +1 -0
  161. package/dist/esm/utilities/fileSystem.d.ts +12 -0
  162. package/dist/esm/utilities/fileSystem.js +81 -0
  163. package/dist/esm/utilities/fileSystem.js.map +1 -0
  164. package/dist/esm/utilities/getApiKeyType.d.ts +13 -0
  165. package/dist/esm/utilities/getApiKeyType.js +44 -0
  166. package/dist/esm/utilities/getApiKeyType.js.map +1 -0
  167. package/dist/esm/utilities/initialBanner.d.ts +4 -0
  168. package/dist/esm/utilities/initialBanner.js +82 -0
  169. package/dist/esm/utilities/initialBanner.js.map +1 -0
  170. package/dist/esm/utilities/keyValueBy.d.ts +8 -0
  171. package/dist/esm/utilities/keyValueBy.js +19 -0
  172. package/dist/esm/utilities/keyValueBy.js.map +1 -0
  173. package/dist/esm/utilities/links.d.ts +9 -0
  174. package/dist/esm/utilities/links.js +10 -0
  175. package/dist/esm/utilities/links.js.map +1 -0
  176. package/dist/esm/utilities/linux.d.ts +1 -0
  177. package/dist/esm/utilities/linux.js +24 -0
  178. package/dist/esm/utilities/linux.js.map +1 -0
  179. package/dist/esm/utilities/logger.d.ts +40 -0
  180. package/dist/esm/utilities/logger.js +109 -0
  181. package/dist/esm/utilities/logger.js.map +1 -0
  182. package/dist/esm/utilities/normalizeImportPath.d.ts +1 -0
  183. package/dist/esm/utilities/normalizeImportPath.js +5 -0
  184. package/dist/esm/utilities/normalizeImportPath.js.map +1 -0
  185. package/dist/esm/utilities/obfuscateApiKey.d.ts +1 -0
  186. package/dist/esm/utilities/obfuscateApiKey.js +5 -0
  187. package/dist/esm/utilities/obfuscateApiKey.js.map +1 -0
  188. package/dist/esm/utilities/parseNameAndPath.d.ts +2 -0
  189. package/dist/esm/utilities/parseNameAndPath.js +10 -0
  190. package/dist/esm/utilities/parseNameAndPath.js.map +1 -0
  191. package/dist/esm/utilities/resolveInternalFilePath.d.ts +1 -0
  192. package/dist/esm/utilities/resolveInternalFilePath.js +8 -0
  193. package/dist/esm/utilities/resolveInternalFilePath.js.map +1 -0
  194. package/dist/esm/utilities/runtimeCheck.d.ts +8 -0
  195. package/dist/esm/utilities/runtimeCheck.js +52 -0
  196. package/dist/esm/utilities/runtimeCheck.js.map +1 -0
  197. package/dist/esm/utilities/safeJsonParse.d.ts +1 -0
  198. package/dist/esm/utilities/safeJsonParse.js +12 -0
  199. package/dist/esm/utilities/safeJsonParse.js.map +1 -0
  200. package/dist/esm/utilities/sanitizeEnvVars.d.ts +3 -0
  201. package/dist/esm/utilities/sanitizeEnvVars.js +4 -0
  202. package/dist/esm/utilities/sanitizeEnvVars.js.map +1 -0
  203. package/dist/esm/utilities/session.d.ts +33 -0
  204. package/dist/esm/utilities/session.js +82 -0
  205. package/dist/esm/utilities/session.js.map +1 -0
  206. package/dist/esm/utilities/sourceFiles.d.ts +12 -0
  207. package/dist/esm/utilities/sourceFiles.js +76 -0
  208. package/dist/esm/utilities/sourceFiles.js.map +1 -0
  209. package/dist/esm/utilities/taskFiles.d.ts +6 -0
  210. package/dist/esm/utilities/taskFiles.js +69 -0
  211. package/dist/esm/utilities/taskFiles.js.map +1 -0
  212. package/dist/esm/utilities/tempDirectories.d.ts +16 -0
  213. package/dist/esm/utilities/tempDirectories.js +27 -0
  214. package/dist/esm/utilities/tempDirectories.js.map +1 -0
  215. package/dist/esm/utilities/windows.d.ts +7 -0
  216. package/dist/esm/utilities/windows.js +21 -0
  217. package/dist/esm/utilities/windows.js.map +1 -0
  218. package/dist/esm/version.d.ts +1 -0
  219. package/dist/esm/version.js +2 -0
  220. package/dist/esm/version.js.map +1 -0
  221. package/package.json +82 -78
  222. package/dist/Containerfile.prod +0 -58
  223. package/dist/index.js +0 -7780
  224. package/dist/index.js.map +0 -1
  225. package/dist/templates/examples/simple.ts.template +0 -14
  226. package/dist/templates/trigger.config.ts.template +0 -16
  227. package/dist/workers/dev/worker-facade.js +0 -192
  228. package/dist/workers/dev/worker-setup.js +0 -35
  229. package/dist/workers/prod/entry-point.js +0 -2163
  230. package/dist/workers/prod/worker-facade.js +0 -223
  231. package/dist/workers/prod/worker-setup.js +0 -24
@@ -1,2163 +0,0 @@
1
- // src/workers/prod/entry-point.ts
2
- import {
3
- CoordinatorToProdWorkerMessages,
4
- PostStartCauses,
5
- PreStopCauses,
6
- ProdWorkerToCoordinatorMessages,
7
- TaskRunErrorCodes as TaskRunErrorCodes2
8
- } from "@trigger.dev/core/v3";
9
- import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
10
-
11
- // ../core-apps/src/http.ts
12
- var HttpReply = class {
13
- constructor(response) {
14
- this.response = response;
15
- }
16
- empty(status) {
17
- return this.response.writeHead(status ?? 200).end();
18
- }
19
- text(text, status, contentType) {
20
- return this.response.writeHead(status ?? 200, { "Content-Type": contentType || "text/plain" }).end(text.endsWith("\n") ? text : `${text}
21
- `);
22
- }
23
- json(value, pretty) {
24
- return this.text(
25
- JSON.stringify(value, void 0, pretty ? 2 : void 0),
26
- 200,
27
- "application/json"
28
- );
29
- }
30
- };
31
- function getRandomInteger(min, max) {
32
- const intMin = Math.ceil(min);
33
- const intMax = Math.floor(max);
34
- return Math.floor(Math.random() * (intMax - intMin + 1)) + intMin;
35
- }
36
- function getRandomPortNumber() {
37
- return getRandomInteger(8e3, 9999);
38
- }
39
-
40
- // ../core-apps/src/logger.ts
41
- var SimpleLogger = class {
42
- constructor(prefix) {
43
- this.prefix = prefix;
44
- }
45
- #debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? "");
46
- log(arg0, ...argN) {
47
- console.log(...this.#getPrefixedArgs(arg0, ...argN));
48
- return arg0;
49
- }
50
- debug(arg0, ...argN) {
51
- if (!this.#debugEnabled) {
52
- return arg0;
53
- }
54
- console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN));
55
- return arg0;
56
- }
57
- error(arg0, ...argN) {
58
- console.error(...this.#getPrefixedArgs(arg0, ...argN));
59
- return arg0;
60
- }
61
- #getPrefixedArgs(...args) {
62
- if (!this.prefix) {
63
- return args;
64
- }
65
- return [this.prefix, ...args];
66
- }
67
- };
68
-
69
- // ../core-apps/src/process.ts
70
- var EXIT_CODE_ALREADY_HANDLED = 111;
71
- var EXIT_CODE_CHILD_NONZERO = 112;
72
-
73
- // ../core-apps/src/backoff.ts
74
- import { setTimeout as timeout } from "node:timers/promises";
75
- var StopRetrying = class extends Error {
76
- constructor(message) {
77
- super(message);
78
- this.name = "StopRetrying";
79
- }
80
- };
81
- var AttemptTimeout = class extends Error {
82
- constructor(message) {
83
- super(message);
84
- this.name = "AttemptTimeout";
85
- }
86
- };
87
- var RetryLimitExceeded = class extends Error {
88
- constructor(message) {
89
- super(message);
90
- this.name = "RetryLimitExceeded";
91
- }
92
- };
93
- var ExponentialBackoff = class _ExponentialBackoff {
94
- #retries = 0;
95
- #type;
96
- #base;
97
- #factor;
98
- #min;
99
- #max;
100
- #maxRetries;
101
- #maxElapsed;
102
- constructor(type, opts = {}) {
103
- this.#type = type ?? "NoJitter";
104
- this.#base = opts.base ?? 2;
105
- this.#factor = opts.factor ?? 1;
106
- this.#min = opts.min ?? -Infinity;
107
- this.#max = opts.max ?? Infinity;
108
- this.#maxRetries = opts.maxRetries ?? Infinity;
109
- this.#maxElapsed = opts.maxElapsed ?? Infinity;
110
- }
111
- #clone(type, opts = {}) {
112
- return new _ExponentialBackoff(type ?? this.#type, {
113
- base: opts.base ?? this.#base,
114
- factor: opts.factor ?? this.#factor,
115
- min: opts.min ?? this.#min,
116
- max: opts.max ?? this.#max,
117
- maxRetries: opts.maxRetries ?? this.#maxRetries,
118
- maxElapsed: opts.maxElapsed ?? this.#maxElapsed
119
- });
120
- }
121
- type(type) {
122
- return this.#clone(type);
123
- }
124
- base(base) {
125
- return this.#clone(void 0, { base });
126
- }
127
- factor(factor) {
128
- return this.#clone(void 0, { factor });
129
- }
130
- min(min) {
131
- return this.#clone(void 0, { min });
132
- }
133
- max(max) {
134
- return this.#clone(void 0, { max });
135
- }
136
- maxRetries(maxRetries) {
137
- return this.#clone(void 0, { maxRetries });
138
- }
139
- // TODO: With .execute(), should this also include the time it takes to execute the callback?
140
- maxElapsed(maxElapsed) {
141
- return this.#clone(void 0, { maxElapsed });
142
- }
143
- retries(retries) {
144
- if (typeof retries !== "undefined") {
145
- if (retries > this.#maxRetries) {
146
- console.error(
147
- `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
148
- );
149
- this.#retries = this.#maxRetries;
150
- } else {
151
- this.#retries = retries;
152
- }
153
- }
154
- return this.#clone();
155
- }
156
- async *retryAsync(maxRetries = this.#maxRetries ?? Infinity) {
157
- let elapsed = 0;
158
- let retry = 0;
159
- while (retry <= maxRetries) {
160
- const delay = this.delay(retry);
161
- elapsed += delay;
162
- if (elapsed > this.#maxElapsed) {
163
- break;
164
- }
165
- yield {
166
- delay: {
167
- seconds: delay,
168
- milliseconds: delay * 1e3
169
- },
170
- retry
171
- };
172
- retry++;
173
- }
174
- }
175
- async *[Symbol.asyncIterator]() {
176
- yield* this.retryAsync();
177
- }
178
- /** Returns the delay for the current retry in seconds. */
179
- delay(retries = this.#retries, jitter = true) {
180
- if (retries > this.#maxRetries) {
181
- console.error(
182
- `Can't set retries ${retries} higher than maxRetries (${this.#maxRetries}), setting to maxRetries instead.`
183
- );
184
- retries = this.#maxRetries;
185
- }
186
- let delay = this.#factor * this.#base ** retries;
187
- switch (this.#type) {
188
- case "NoJitter": {
189
- break;
190
- }
191
- case "FullJitter": {
192
- if (!jitter) {
193
- delay = 0;
194
- break;
195
- }
196
- delay *= Math.random();
197
- break;
198
- }
199
- case "EqualJitter": {
200
- if (!jitter) {
201
- delay *= 0.5;
202
- break;
203
- }
204
- delay *= 0.5 * (1 + Math.random());
205
- break;
206
- }
207
- default: {
208
- throw new Error(`Unknown backoff type: ${this.#type}`);
209
- }
210
- }
211
- if (delay < this.#min) {
212
- delay = this.#min + Math.random() * (this.#min * 0.2);
213
- }
214
- if (delay > this.#max) {
215
- delay = this.#max - Math.random() * (this.#max * 0.2);
216
- }
217
- delay = Math.round(delay);
218
- return delay;
219
- }
220
- /** Waits with the appropriate delay for the current retry. */
221
- async wait(retries = this.#retries, jitter = true) {
222
- if (retries > this.#maxRetries) {
223
- console.error(`Retry limit exceeded: ${retries} > ${this.#maxRetries}`);
224
- throw new RetryLimitExceeded();
225
- }
226
- const delay = this.delay(retries, jitter);
227
- return await timeout(delay * 1e3);
228
- }
229
- elapsed(retries = this.#retries, jitter = true) {
230
- let elapsed = 0;
231
- for (let i = 0; i <= retries; i++) {
232
- elapsed += this.delay(i, jitter);
233
- }
234
- const total = elapsed;
235
- let days = 0;
236
- if (elapsed > 3600 * 24) {
237
- days = Math.floor(elapsed / 3600 / 24);
238
- elapsed -= days * 3600 * 24;
239
- }
240
- let hours = 0;
241
- if (elapsed > 3600) {
242
- hours = Math.floor(elapsed / 3600);
243
- elapsed -= hours * 3600;
244
- }
245
- let minutes = 0;
246
- if (elapsed > 60) {
247
- minutes = Math.floor(elapsed / 60);
248
- elapsed -= minutes * 60;
249
- }
250
- const seconds = elapsed;
251
- return {
252
- seconds,
253
- minutes,
254
- hours,
255
- days,
256
- total
257
- };
258
- }
259
- reset() {
260
- this.#retries = 0;
261
- return this;
262
- }
263
- next() {
264
- this.#retries++;
265
- return this.delay();
266
- }
267
- stop() {
268
- throw new StopRetrying();
269
- }
270
- get state() {
271
- return {
272
- retries: this.#retries,
273
- type: this.#type,
274
- base: this.#base,
275
- factor: this.#factor,
276
- min: this.#min,
277
- max: this.#max,
278
- maxRetries: this.#maxRetries,
279
- maxElapsed: this.#maxElapsed
280
- };
281
- }
282
- async execute(callback, { attemptTimeoutMs = 0 } = {}) {
283
- let elapsedMs = 0;
284
- let finalError = void 0;
285
- for await (const { delay, retry } of this) {
286
- const start = Date.now();
287
- if (retry > 0) {
288
- console.log(`Retrying in ${delay.milliseconds}ms`);
289
- await timeout(delay.milliseconds);
290
- }
291
- let attemptTimeout = void 0;
292
- try {
293
- const result = await new Promise(async (resolve, reject) => {
294
- if (attemptTimeoutMs > 0) {
295
- attemptTimeout = setTimeout(() => {
296
- reject(new AttemptTimeout());
297
- }, attemptTimeoutMs);
298
- }
299
- try {
300
- const callbackResult = await callback({ delay, retry, elapsedMs });
301
- resolve(callbackResult);
302
- } catch (error) {
303
- reject(error);
304
- }
305
- });
306
- return {
307
- success: true,
308
- result
309
- };
310
- } catch (error) {
311
- finalError = error;
312
- if (error instanceof StopRetrying) {
313
- return {
314
- success: false,
315
- cause: "StopRetrying",
316
- error: error.message
317
- };
318
- }
319
- if (error instanceof AttemptTimeout) {
320
- continue;
321
- }
322
- } finally {
323
- elapsedMs += Date.now() - start;
324
- clearTimeout(attemptTimeout);
325
- }
326
- }
327
- if (finalError instanceof AttemptTimeout) {
328
- return {
329
- success: false,
330
- cause: "Timeout"
331
- };
332
- } else {
333
- return {
334
- success: false,
335
- cause: "MaxRetries",
336
- error: finalError
337
- };
338
- }
339
- }
340
- static RetryLimitExceeded = RetryLimitExceeded;
341
- static StopRetrying = StopRetrying;
342
- };
343
-
344
- // src/workers/prod/backgroundWorker.ts
345
- import {
346
- ProdChildToWorkerMessages,
347
- ProdWorkerToChildMessages,
348
- SemanticInternalAttributes,
349
- TaskRunErrorCodes,
350
- correctErrorStackTrace
351
- } from "@trigger.dev/core/v3";
352
- import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
353
- import { Evt } from "evt";
354
- import { fork } from "node:child_process";
355
-
356
- // src/workers/common/errors.ts
357
- var UncaughtExceptionError = class extends Error {
358
- constructor(originalError, origin) {
359
- super(`Uncaught exception: ${originalError.message}`);
360
- this.originalError = originalError;
361
- this.origin = origin;
362
- this.name = "UncaughtExceptionError";
363
- }
364
- };
365
- var TaskMetadataParseError = class extends Error {
366
- constructor(zodIssues, tasks) {
367
- super(`Failed to parse task metadata`);
368
- this.zodIssues = zodIssues;
369
- this.tasks = tasks;
370
- this.name = "TaskMetadataParseError";
371
- }
372
- };
373
- var UnexpectedExitError = class extends Error {
374
- constructor(code, signal, stderr) {
375
- super(`Unexpected exit with code ${code}`);
376
- this.code = code;
377
- this.signal = signal;
378
- this.stderr = stderr;
379
- this.name = "UnexpectedExitError";
380
- }
381
- };
382
- var CleanupProcessError = class extends Error {
383
- constructor() {
384
- super("Cancelled");
385
- this.name = "CleanupProcessError";
386
- }
387
- };
388
- var CancelledProcessError = class extends Error {
389
- constructor() {
390
- super("Cancelled");
391
- this.name = "CancelledProcessError";
392
- }
393
- };
394
- var SigKillTimeoutProcessError = class extends Error {
395
- constructor() {
396
- super("Process kill timeout");
397
- this.name = "SigKillTimeoutProcessError";
398
- }
399
- };
400
- var GracefulExitTimeoutError = class extends Error {
401
- constructor() {
402
- super("Graceful exit timeout");
403
- this.name = "GracefulExitTimeoutError";
404
- }
405
- };
406
- function getFriendlyErrorMessage(code, signal, stderr, dockerMode = true) {
407
- const message = (text) => {
408
- if (signal) {
409
- return `[${signal}] ${text}`;
410
- } else {
411
- return text;
412
- }
413
- };
414
- if (code === 137) {
415
- if (dockerMode) {
416
- return message(
417
- "Process ran out of memory! Try choosing a machine preset with more memory for this task."
418
- );
419
- } else {
420
- return message(
421
- "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
422
- );
423
- }
424
- }
425
- if (stderr?.includes("OOMErrorHandler")) {
426
- return message(
427
- "Process ran out of memory! Try choosing a machine preset with more memory for this task."
428
- );
429
- }
430
- return message(`Process exited with code ${code}.`);
431
- }
432
-
433
- // src/workers/prod/backgroundWorker.ts
434
- var ProdBackgroundWorker = class {
435
- constructor(path, params) {
436
- this.path = path;
437
- this.params = params;
438
- }
439
- _initialized = false;
440
- /**
441
- * @deprecated use onTaskRunHeartbeat instead
442
- */
443
- onTaskHeartbeat = new Evt();
444
- onTaskRunHeartbeat = new Evt();
445
- onWaitForDuration = new Evt();
446
- onWaitForTask = new Evt();
447
- onWaitForBatch = new Evt();
448
- onCreateTaskRunAttempt = Evt.create();
449
- attemptCreatedNotification = Evt.create();
450
- _onClose = new Evt();
451
- tasks = [];
452
- stderr = [];
453
- _taskRunProcess;
454
- _taskRunProcessesBeingKilled = /* @__PURE__ */ new Map();
455
- _closed = false;
456
- async close(gracefulExitTimeoutElapsed = false) {
457
- console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
458
- if (this._closed) {
459
- return;
460
- }
461
- this._closed = true;
462
- this.onTaskHeartbeat.detach();
463
- this.onTaskRunHeartbeat.detach();
464
- await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
465
- }
466
- async #killTaskRunProcess(flush = true, initialSignal = "SIGTERM") {
467
- console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
468
- if (this._closed || !this._taskRunProcess) {
469
- return;
470
- }
471
- if (flush) {
472
- await this.flushTelemetry();
473
- }
474
- const currentTaskRunProcess = this._taskRunProcess;
475
- this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
476
- console.error("Error while trying graceful exit", error);
477
- });
478
- console.log("Killed task run process, setting closed to true", {
479
- closed: this._closed,
480
- pid: currentTaskRunProcess.pid
481
- });
482
- this._closed = true;
483
- }
484
- async flushTelemetry() {
485
- console.log("Flushing telemetry");
486
- const start = performance.now();
487
- await this._taskRunProcess?.cleanup(false);
488
- console.log("Flushed telemetry", { duration: performance.now() - start });
489
- }
490
- async initialize(options) {
491
- if (this._initialized) {
492
- throw new Error("Worker already initialized");
493
- }
494
- let resolved = false;
495
- this.tasks = await new Promise((resolve, reject) => {
496
- const child = fork(this.path, {
497
- stdio: [
498
- /*stdin*/
499
- "ignore",
500
- /*stdout*/
501
- "pipe",
502
- /*stderr*/
503
- "pipe",
504
- "ipc"
505
- ],
506
- env: {
507
- ...this.params.env,
508
- ...options?.env
509
- }
510
- });
511
- const timeout3 = setTimeout(() => {
512
- if (resolved) {
513
- return;
514
- }
515
- resolved = true;
516
- child.kill();
517
- reject(new Error("Worker timed out"));
518
- }, 1e4);
519
- child.stdout?.on("data", (data) => {
520
- console.log(data.toString());
521
- });
522
- child.stderr?.on("data", (data) => {
523
- console.error(data.toString());
524
- this.stderr.push(data.toString());
525
- });
526
- child.on("exit", (code) => {
527
- if (!resolved) {
528
- clearTimeout(timeout3);
529
- resolved = true;
530
- reject(new Error(`Worker exited with code ${code}`));
531
- }
532
- });
533
- new ZodIpcConnection({
534
- listenSchema: ProdChildToWorkerMessages,
535
- emitSchema: ProdWorkerToChildMessages,
536
- process: child,
537
- handlers: {
538
- TASKS_READY: async (message) => {
539
- if (!resolved) {
540
- clearTimeout(timeout3);
541
- resolved = true;
542
- resolve(message.tasks);
543
- child.kill();
544
- }
545
- },
546
- UNCAUGHT_EXCEPTION: async (message) => {
547
- if (!resolved) {
548
- clearTimeout(timeout3);
549
- resolved = true;
550
- reject(new UncaughtExceptionError(message.error, message.origin));
551
- child.kill();
552
- }
553
- },
554
- TASKS_FAILED_TO_PARSE: async (message) => {
555
- if (!resolved) {
556
- clearTimeout(timeout3);
557
- resolved = true;
558
- reject(new TaskMetadataParseError(message.zodIssues, message.tasks));
559
- child.kill();
560
- }
561
- }
562
- }
563
- });
564
- });
565
- this._initialized = true;
566
- }
567
- getMetadata(workerId, version) {
568
- return {
569
- contentHash: this.params.contentHash,
570
- id: workerId,
571
- version
572
- };
573
- }
574
- // We need to notify all the task run processes that a task run has completed,
575
- // in case they are waiting for it through triggerAndWait
576
- async taskRunCompletedNotification(completion) {
577
- this._taskRunProcess?.taskRunCompletedNotification(completion);
578
- }
579
- async waitCompletedNotification() {
580
- this._taskRunProcess?.waitCompletedNotification();
581
- }
582
- async #getFreshTaskRunProcess(payload, messageId) {
583
- const metadata = this.getMetadata(
584
- payload.execution.worker.id,
585
- payload.execution.worker.version
586
- );
587
- console.log("Getting fresh task run process, setting closed to false", {
588
- closed: this._closed
589
- });
590
- this._closed = false;
591
- await this.#killCurrentTaskRunProcessBeforeAttempt();
592
- const taskRunProcess = new TaskRunProcess(
593
- payload.execution.run.id,
594
- payload.execution.run.isTest,
595
- this.path,
596
- {
597
- ...this.params.env,
598
- ...payload.environment ?? {}
599
- },
600
- metadata,
601
- this.params,
602
- messageId
603
- );
604
- taskRunProcess.onExit.attach(({ pid }) => {
605
- console.log("Task run process exited", { pid });
606
- if (this._taskRunProcess?.pid === pid) {
607
- this._taskRunProcess = void 0;
608
- }
609
- if (pid) {
610
- this._taskRunProcessesBeingKilled.delete(pid);
611
- }
612
- });
613
- taskRunProcess.onIsBeingKilled.attach((taskRunProcess2) => {
614
- if (taskRunProcess2?.pid) {
615
- this._taskRunProcessesBeingKilled.set(taskRunProcess2.pid, taskRunProcess2);
616
- }
617
- });
618
- taskRunProcess.onTaskHeartbeat.attach((id) => {
619
- this.onTaskHeartbeat.post(id);
620
- });
621
- taskRunProcess.onTaskRunHeartbeat.attach((id) => {
622
- this.onTaskRunHeartbeat.post(id);
623
- });
624
- taskRunProcess.onWaitForBatch.attach((message) => {
625
- this.onWaitForBatch.post(message);
626
- });
627
- taskRunProcess.onWaitForDuration.attach((message) => {
628
- this.onWaitForDuration.post(message);
629
- });
630
- taskRunProcess.onWaitForTask.attach((message) => {
631
- this.onWaitForTask.post(message);
632
- });
633
- await taskRunProcess.initialize();
634
- this._taskRunProcess = taskRunProcess;
635
- return this._taskRunProcess;
636
- }
637
- async forceKillOldTaskRunProcesses() {
638
- for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
639
- try {
640
- await taskRunProcess.kill("SIGKILL");
641
- } catch (error) {
642
- console.error("Error while force killing old task run processes", error);
643
- }
644
- }
645
- }
646
- async #killCurrentTaskRunProcessBeforeAttempt() {
647
- console.log("killCurrentTaskRunProcessBeforeAttempt()", {
648
- hasTaskRunProcess: !!this._taskRunProcess
649
- });
650
- if (!this._taskRunProcess) {
651
- return;
652
- }
653
- const currentTaskRunProcess = this._taskRunProcess;
654
- console.log("Killing current task run process", {
655
- isBeingKilled: currentTaskRunProcess?.isBeingKilled,
656
- totalBeingKilled: this._taskRunProcessesBeingKilled.size
657
- });
658
- if (currentTaskRunProcess.isBeingKilled) {
659
- if (this._taskRunProcessesBeingKilled.size > 1) {
660
- await this.#tryGracefulExit(currentTaskRunProcess);
661
- } else {
662
- }
663
- } else {
664
- if (this._taskRunProcessesBeingKilled.size > 0) {
665
- await this.#tryGracefulExit(currentTaskRunProcess);
666
- } else {
667
- currentTaskRunProcess.kill("SIGTERM", 5e3).catch(() => {
668
- });
669
- }
670
- }
671
- }
672
- async #tryGracefulExit(taskRunProcess, kill = false, initialSignal = "SIGTERM") {
673
- console.log("Trying graceful exit", { kill, initialSignal });
674
- try {
675
- const initialExit = taskRunProcess.onExit.waitFor(5e3);
676
- if (kill) {
677
- taskRunProcess.kill(initialSignal);
678
- }
679
- await initialExit;
680
- } catch (error) {
681
- console.error("TaskRunProcess graceful kill timeout exceeded", error);
682
- this.#tryForcefulExit(taskRunProcess);
683
- }
684
- }
685
- async #tryForcefulExit(taskRunProcess) {
686
- console.log("Trying forceful exit");
687
- try {
688
- const forcedKill = taskRunProcess.onExit.waitFor(5e3);
689
- taskRunProcess.kill("SIGKILL");
690
- await forcedKill;
691
- } catch (error) {
692
- console.error("TaskRunProcess forced kill timeout exceeded", error);
693
- throw new SigKillTimeoutProcessError();
694
- }
695
- }
696
- // We need to fork the process before we can execute any tasks, use a fresh process for each execution
697
- async executeTaskRun(payload, messageId) {
698
- try {
699
- const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
700
- console.log("executing task run", {
701
- attempt: payload.execution.attempt.id,
702
- taskRunPid: taskRunProcess.pid
703
- });
704
- const result = await taskRunProcess.executeTaskRun(payload);
705
- if (result.ok) {
706
- return result;
707
- }
708
- const error = result.error;
709
- if (error.type === "BUILT_IN_ERROR") {
710
- const mappedError = await this.#correctError(error, payload.execution);
711
- return {
712
- ...result,
713
- error: mappedError
714
- };
715
- }
716
- return result;
717
- } catch (e) {
718
- if (e instanceof CancelledProcessError) {
719
- return {
720
- id: payload.execution.attempt.id,
721
- ok: false,
722
- retry: void 0,
723
- error: {
724
- type: "INTERNAL_ERROR",
725
- code: TaskRunErrorCodes.TASK_RUN_CANCELLED
726
- }
727
- };
728
- }
729
- if (e instanceof CleanupProcessError) {
730
- return {
731
- id: payload.execution.attempt.id,
732
- ok: false,
733
- retry: void 0,
734
- error: {
735
- type: "INTERNAL_ERROR",
736
- code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED
737
- }
738
- };
739
- }
740
- if (e instanceof UnexpectedExitError) {
741
- return {
742
- id: payload.execution.attempt.id,
743
- ok: false,
744
- retry: void 0,
745
- error: {
746
- type: "INTERNAL_ERROR",
747
- code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
748
- message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
749
- stackTrace: e.stderr
750
- }
751
- };
752
- }
753
- if (e instanceof SigKillTimeoutProcessError) {
754
- return {
755
- id: payload.execution.attempt.id,
756
- ok: false,
757
- retry: void 0,
758
- error: {
759
- type: "INTERNAL_ERROR",
760
- code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT
761
- }
762
- };
763
- }
764
- if (e instanceof GracefulExitTimeoutError) {
765
- return {
766
- id: payload.execution.attempt.id,
767
- ok: false,
768
- retry: void 0,
769
- error: {
770
- type: "INTERNAL_ERROR",
771
- code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
772
- message: "Worker process killed while attempt in progress."
773
- }
774
- };
775
- }
776
- return {
777
- id: payload.execution.attempt.id,
778
- ok: false,
779
- retry: void 0,
780
- error: {
781
- type: "INTERNAL_ERROR",
782
- code: TaskRunErrorCodes.TASK_EXECUTION_FAILED
783
- }
784
- };
785
- } finally {
786
- await this.#killTaskRunProcess();
787
- }
788
- }
789
- async cancelAttempt(attemptId) {
790
- if (!this._taskRunProcess) {
791
- console.error("No task run process to cancel attempt", { attemptId });
792
- return;
793
- }
794
- await this._taskRunProcess.cancel();
795
- }
796
- async executeTaskRunLazyAttempt(payload) {
797
- this.onCreateTaskRunAttempt.post({ runId: payload.runId });
798
- let execution;
799
- try {
800
- const start = performance.now();
801
- const attemptCreated = await this.attemptCreatedNotification.waitFor(12e4);
802
- if (!attemptCreated.success) {
803
- throw new Error(`${attemptCreated.reason ?? "Unknown error"}`);
804
- }
805
- console.log("Attempt created", {
806
- number: attemptCreated.execution.attempt.number,
807
- duration: performance.now() - start
808
- });
809
- execution = attemptCreated.execution;
810
- } catch (error) {
811
- console.error("Error while creating attempt", error);
812
- throw new Error(`Failed to create attempt: ${error}`);
813
- }
814
- const completion = await this.executeTaskRun(
815
- {
816
- execution,
817
- traceContext: payload.traceContext,
818
- environment: payload.environment
819
- },
820
- payload.messageId
821
- );
822
- return { execution, completion };
823
- }
824
- async #correctError(error, execution) {
825
- return {
826
- ...error,
827
- stackTrace: correctErrorStackTrace(error.stackTrace, this.params.projectConfig.projectDir)
828
- };
829
- }
830
- };
831
- var TaskRunProcess = class {
832
- constructor(runId, isTest, path, env, metadata, worker, messageId) {
833
- this.runId = runId;
834
- this.isTest = isTest;
835
- this.path = path;
836
- this.env = env;
837
- this.metadata = metadata;
838
- this.worker = worker;
839
- this.messageId = messageId;
840
- }
841
- _ipc;
842
- _child;
843
- _childPid;
844
- _attemptPromises = /* @__PURE__ */ new Map();
845
- _attemptStatuses = /* @__PURE__ */ new Map();
846
- _currentExecution;
847
- _isBeingKilled = false;
848
- _isBeingCancelled = false;
849
- _gracefulExitTimeoutElapsed = false;
850
- _stderr = [];
851
- /**
852
- * @deprecated use onTaskRunHeartbeat instead
853
- */
854
- onTaskHeartbeat = new Evt();
855
- onTaskRunHeartbeat = new Evt();
856
- onExit = new Evt();
857
- onIsBeingKilled = new Evt();
858
- onWaitForDuration = new Evt();
859
- onWaitForTask = new Evt();
860
- onWaitForBatch = new Evt();
861
- preCheckpointNotification = Evt.create();
862
- async initialize() {
863
- this._child = fork(this.path, {
864
- stdio: [
865
- /*stdin*/
866
- "ignore",
867
- /*stdout*/
868
- "pipe",
869
- /*stderr*/
870
- "pipe",
871
- "ipc"
872
- ],
873
- env: {
874
- ...this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {},
875
- ...this.env,
876
- OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
877
- [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir
878
- }),
879
- ...this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}
880
- }
881
- });
882
- this._childPid = this._child?.pid;
883
- this._ipc = new ZodIpcConnection({
884
- listenSchema: ProdChildToWorkerMessages,
885
- emitSchema: ProdWorkerToChildMessages,
886
- process: this._child,
887
- handlers: {
888
- TASK_RUN_COMPLETED: async (message) => {
889
- const { result, execution } = message;
890
- const promiseStatus = this._attemptStatuses.get(execution.attempt.id);
891
- if (promiseStatus !== "PENDING") {
892
- return;
893
- }
894
- this._attemptStatuses.set(execution.attempt.id, "RESOLVED");
895
- const attemptPromise = this._attemptPromises.get(execution.attempt.id);
896
- if (!attemptPromise) {
897
- return;
898
- }
899
- const { resolver } = attemptPromise;
900
- resolver(result);
901
- },
902
- READY_TO_DISPOSE: async (message) => {
903
- process.exit(0);
904
- },
905
- TASK_HEARTBEAT: async (message) => {
906
- if (this.messageId) {
907
- this.onTaskRunHeartbeat.post(this.messageId);
908
- } else {
909
- console.error(
910
- "No message id for task heartbeat, falling back to (deprecated) attempt heartbeat",
911
- { id: message.id }
912
- );
913
- this.onTaskHeartbeat.post(message.id);
914
- }
915
- },
916
- TASKS_READY: async (message) => {
917
- },
918
- WAIT_FOR_TASK: async (message) => {
919
- this.onWaitForTask.post(message);
920
- },
921
- WAIT_FOR_BATCH: async (message) => {
922
- this.onWaitForBatch.post(message);
923
- },
924
- WAIT_FOR_DURATION: async (message) => {
925
- this.onWaitForDuration.post(message);
926
- }
927
- }
928
- });
929
- this._child.on("exit", this.#handleExit.bind(this));
930
- this._child.stdout?.on("data", this.#handleLog.bind(this));
931
- this._child.stderr?.on("data", this.#handleStdErr.bind(this));
932
- }
933
- async cancel() {
934
- this._isBeingCancelled = true;
935
- await this.cleanup(true);
936
- }
937
- async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
938
- console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
939
- if (kill && this._isBeingKilled) {
940
- return;
941
- }
942
- if (kill) {
943
- this._isBeingKilled = true;
944
- this.onIsBeingKilled.post(this);
945
- }
946
- const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
947
- const killParentProcess = kill && !killChildProcess;
948
- console.log("Cleaning up task run process", {
949
- killChildProcess,
950
- killParentProcess,
951
- ipc: this._ipc,
952
- childPid: this._childPid,
953
- realChildPid: this._child?.pid
954
- });
955
- try {
956
- await this._ipc?.sendWithAck(
957
- "CLEANUP",
958
- {
959
- flush: true,
960
- kill: killParentProcess
961
- },
962
- 3e4
963
- );
964
- } catch (error) {
965
- console.error("Error while cleaning up task run process", error);
966
- if (killParentProcess) {
967
- process.exit(0);
968
- }
969
- }
970
- if (killChildProcess) {
971
- this._gracefulExitTimeoutElapsed = true;
972
- await this.kill("SIGKILL");
973
- }
974
- }
975
- async executeTaskRun(payload) {
976
- let resolver;
977
- let rejecter;
978
- const promise = new Promise((resolve, reject) => {
979
- resolver = resolve;
980
- rejecter = reject;
981
- });
982
- this._attemptStatuses.set(payload.execution.attempt.id, "PENDING");
983
- this._attemptPromises.set(payload.execution.attempt.id, { resolver, rejecter });
984
- const { execution, traceContext } = payload;
985
- this._currentExecution = execution;
986
- if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
987
- await this._ipc?.send("EXECUTE_TASK_RUN", {
988
- execution,
989
- traceContext,
990
- metadata: this.metadata
991
- });
992
- }
993
- const result = await promise;
994
- this._currentExecution = void 0;
995
- return result;
996
- }
997
- taskRunCompletedNotification(completion) {
998
- if (!completion.ok && typeof completion.retry !== "undefined") {
999
- console.error(
1000
- "Task run completed with error and wants to retry, won't send task run completed notification"
1001
- );
1002
- return;
1003
- }
1004
- if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
1005
- console.error(
1006
- "Child process not connected or being killed, can't send task run completed notification"
1007
- );
1008
- return;
1009
- }
1010
- this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
1011
- version: "v2",
1012
- completion
1013
- });
1014
- }
1015
- waitCompletedNotification() {
1016
- if (!this._child?.connected || this._isBeingKilled || this._child.killed) {
1017
- console.error(
1018
- "Child process not connected or being killed, can't send wait completed notification"
1019
- );
1020
- return;
1021
- }
1022
- this._ipc?.send("WAIT_COMPLETED_NOTIFICATION", {});
1023
- }
1024
- async #handleExit(code, signal) {
1025
- console.log("handling child exit", { code, signal });
1026
- for (const [id, status] of this._attemptStatuses.entries()) {
1027
- if (status === "PENDING") {
1028
- console.log("found pending attempt", { id });
1029
- this._attemptStatuses.set(id, "REJECTED");
1030
- const attemptPromise = this._attemptPromises.get(id);
1031
- if (!attemptPromise) {
1032
- continue;
1033
- }
1034
- const { rejecter } = attemptPromise;
1035
- if (this._isBeingCancelled) {
1036
- rejecter(new CancelledProcessError());
1037
- } else if (this._gracefulExitTimeoutElapsed) {
1038
- rejecter(new GracefulExitTimeoutError());
1039
- } else if (this._isBeingKilled) {
1040
- rejecter(new CleanupProcessError());
1041
- } else {
1042
- rejecter(
1043
- new UnexpectedExitError(
1044
- code ?? -1,
1045
- signal,
1046
- this._stderr.length ? this._stderr.join("\n") : void 0
1047
- )
1048
- );
1049
- }
1050
- }
1051
- }
1052
- this.onExit.post({ code, signal, pid: this.pid });
1053
- }
1054
- #handleLog(data) {
1055
- console.log(data.toString());
1056
- }
1057
- #handleStdErr(data) {
1058
- const text = data.toString();
1059
- console.error(text);
1060
- if (this._stderr.length > 100) {
1061
- this._stderr.shift();
1062
- }
1063
- this._stderr.push(text);
1064
- }
1065
- async kill(signal, timeoutInMs) {
1066
- this._isBeingKilled = true;
1067
- const killTimeout = this.onExit.waitFor(timeoutInMs);
1068
- this.onIsBeingKilled.post(this);
1069
- this._child?.kill(signal);
1070
- if (timeoutInMs) {
1071
- await killTimeout;
1072
- }
1073
- }
1074
- get isBeingKilled() {
1075
- return this._isBeingKilled || this._child?.killed;
1076
- }
1077
- get pid() {
1078
- return this._childPid;
1079
- }
1080
- };
1081
-
1082
- // src/workers/prod/entry-point.ts
1083
- import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers";
1084
- import { randomUUID } from "node:crypto";
1085
- import { readFile } from "node:fs/promises";
1086
- import { createServer } from "node:http";
1087
- import { setTimeout as timeout2 } from "node:timers/promises";
1088
- var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
1089
- var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
1090
- var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
1091
- var MACHINE_NAME = process.env.MACHINE_NAME || "local";
1092
- var POD_NAME = process.env.POD_NAME || "some-pod";
1093
- var SHORT_HASH = process.env.TRIGGER_CONTENT_HASH.slice(0, 9);
1094
- var logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
1095
- var defaultBackoff = new ExponentialBackoff("FullJitter", {
1096
- maxRetries: 5
1097
- });
1098
- var ProdWorker = class {
1099
- constructor(port, host = "0.0.0.0") {
1100
- this.host = host;
1101
- process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
1102
- this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
1103
- this.#backgroundWorker = this.#createBackgroundWorker();
1104
- this.#httpPort = port;
1105
- this.#httpServer = this.#createHttpServer();
1106
- }
1107
- apiUrl = process.env.TRIGGER_API_URL;
1108
- apiKey = process.env.TRIGGER_SECRET_KEY;
1109
- contentHash = process.env.TRIGGER_CONTENT_HASH;
1110
- projectRef = process.env.TRIGGER_PROJECT_REF;
1111
- envId = process.env.TRIGGER_ENV_ID;
1112
- runId = process.env.TRIGGER_RUN_ID || "index-only";
1113
- deploymentId = process.env.TRIGGER_DEPLOYMENT_ID;
1114
- deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION;
1115
- runningInKubernetes = !!process.env.KUBERNETES_PORT;
1116
- executing = false;
1117
- completed = /* @__PURE__ */ new Set();
1118
- paused = false;
1119
- attemptFriendlyId;
1120
- nextResumeAfter;
1121
- waitForPostStart = false;
1122
- connectionCount = 0;
1123
- waitForTaskReplay;
1124
- waitForBatchReplay;
1125
- readyForLazyAttemptReplay;
1126
- submitAttemptCompletionReplay;
1127
- durationResumeFallback;
1128
- #httpPort;
1129
- #backgroundWorker;
1130
- #httpServer;
1131
- #coordinatorSocket;
1132
- async #handleSignal(signal) {
1133
- logger.log("Received signal", { signal });
1134
- if (signal === "SIGTERM") {
1135
- let gracefulExitTimeoutElapsed = false;
1136
- if (this.executing) {
1137
- const terminationGracePeriodSeconds = 60 * 60;
1138
- logger.log("Waiting for attempt to complete before exiting", {
1139
- terminationGracePeriodSeconds
1140
- });
1141
- await timeout2(terminationGracePeriodSeconds * 1e3 - 5e3);
1142
- gracefulExitTimeoutElapsed = true;
1143
- logger.log("Termination timeout reached, exiting gracefully.");
1144
- } else {
1145
- logger.log("Not executing, exiting immediately.");
1146
- }
1147
- await this.#exitGracefully(gracefulExitTimeoutElapsed);
1148
- return;
1149
- }
1150
- logger.log("Unhandled signal", { signal });
1151
- }
1152
- async #exitGracefully(gracefulExitTimeoutElapsed = false, exitCode = 0) {
1153
- await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
1154
- if (!gracefulExitTimeoutElapsed) {
1155
- process.exit(exitCode);
1156
- }
1157
- }
1158
- async #reconnectAfterPostStart() {
1159
- this.waitForPostStart = false;
1160
- this.#coordinatorSocket.close();
1161
- this.connectionCount = 0;
1162
- let coordinatorHost = COORDINATOR_HOST;
1163
- try {
1164
- if (this.runningInKubernetes) {
1165
- coordinatorHost = (await readFile("/etc/taskinfo/coordinator-host", "utf-8")).replace(
1166
- "\n",
1167
- ""
1168
- );
1169
- logger.log("reconnecting", {
1170
- coordinatorHost: {
1171
- fromEnv: COORDINATOR_HOST,
1172
- fromVolume: coordinatorHost,
1173
- current: this.#coordinatorSocket.socket.io.opts.hostname
1174
- }
1175
- });
1176
- }
1177
- } catch (error) {
1178
- logger.error("taskinfo read error during reconnect", {
1179
- error: error instanceof Error ? error.message : error
1180
- });
1181
- } finally {
1182
- this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
1183
- }
1184
- }
1185
- // MARK: TASK WAIT
1186
- async #waitForTaskHandler(message, replayIdempotencyKey) {
1187
- const waitForTask = await defaultBackoff.execute(async ({ retry }) => {
1188
- logger.log("Wait for task with backoff", { retry });
1189
- if (!this.attemptFriendlyId) {
1190
- logger.error("Failed to send wait message, attempt friendly ID not set", { message });
1191
- throw new ExponentialBackoff.StopRetrying("No attempt ID");
1192
- }
1193
- return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_TASK", {
1194
- version: "v2",
1195
- friendlyId: message.friendlyId,
1196
- attemptFriendlyId: this.attemptFriendlyId
1197
- });
1198
- });
1199
- if (!waitForTask.success) {
1200
- logger.error("Failed to wait for task with backoff", {
1201
- cause: waitForTask.cause,
1202
- error: waitForTask.error
1203
- });
1204
- this.#emitUnrecoverableError(
1205
- "WaitForTaskFailed",
1206
- `${waitForTask.cause}: ${waitForTask.error}`
1207
- );
1208
- return;
1209
- }
1210
- const { willCheckpointAndRestore } = waitForTask.result;
1211
- await this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
1212
- if (willCheckpointAndRestore) {
1213
- if (!this.waitForTaskReplay) {
1214
- this.waitForTaskReplay = {
1215
- message,
1216
- attempt: 1,
1217
- idempotencyKey: randomUUID()
1218
- };
1219
- } else {
1220
- if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
1221
- logger.error(
1222
- "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
1223
- );
1224
- return;
1225
- }
1226
- this.waitForTaskReplay.attempt++;
1227
- }
1228
- }
1229
- }
1230
- // MARK: BATCH WAIT
1231
- async #waitForBatchHandler(message, replayIdempotencyKey) {
1232
- const waitForBatch = await defaultBackoff.execute(async ({ retry }) => {
1233
- logger.log("Wait for batch with backoff", { retry });
1234
- if (!this.attemptFriendlyId) {
1235
- logger.error("Failed to send wait message, attempt friendly ID not set", { message });
1236
- throw new ExponentialBackoff.StopRetrying("No attempt ID");
1237
- }
1238
- return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_BATCH", {
1239
- version: "v2",
1240
- batchFriendlyId: message.batchFriendlyId,
1241
- runFriendlyIds: message.runFriendlyIds,
1242
- attemptFriendlyId: this.attemptFriendlyId
1243
- });
1244
- });
1245
- if (!waitForBatch.success) {
1246
- logger.error("Failed to wait for batch with backoff", {
1247
- cause: waitForBatch.cause,
1248
- error: waitForBatch.error
1249
- });
1250
- this.#emitUnrecoverableError(
1251
- "WaitForBatchFailed",
1252
- `${waitForBatch.cause}: ${waitForBatch.error}`
1253
- );
1254
- return;
1255
- }
1256
- const { willCheckpointAndRestore } = waitForBatch.result;
1257
- await this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
1258
- if (willCheckpointAndRestore) {
1259
- if (!this.waitForBatchReplay) {
1260
- this.waitForBatchReplay = {
1261
- message,
1262
- attempt: 1,
1263
- idempotencyKey: randomUUID()
1264
- };
1265
- } else {
1266
- if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
1267
- logger.error(
1268
- "wait for task handler called with mismatched idempotency key, won't overwrite replay request"
1269
- );
1270
- return;
1271
- }
1272
- this.waitForBatchReplay.attempt++;
1273
- }
1274
- }
1275
- }
1276
- // MARK: WORKER CREATION
1277
- #createBackgroundWorker() {
1278
- const backgroundWorker = new ProdBackgroundWorker("worker.js", {
1279
- projectConfig: __PROJECT_CONFIG__,
1280
- env: {
1281
- ...gatherProcessEnv(),
1282
- TRIGGER_API_URL: this.apiUrl,
1283
- TRIGGER_SECRET_KEY: this.apiKey,
1284
- OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"
1285
- },
1286
- contentHash: this.contentHash
1287
- });
1288
- backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
1289
- logger.log("onTaskHeartbeat", { attemptFriendlyId });
1290
- this.#coordinatorSocket.socket.volatile.emit("TASK_HEARTBEAT", {
1291
- version: "v1",
1292
- attemptFriendlyId
1293
- });
1294
- });
1295
- backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
1296
- logger.log("onTaskRunHeartbeat", { runId });
1297
- this.#coordinatorSocket.socket.volatile.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
1298
- });
1299
- backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
1300
- logger.log("onCreateTaskRunAttempt()", { message });
1301
- const createAttempt = await defaultBackoff.execute(async ({ retry }) => {
1302
- logger.log("Create task run attempt with backoff", { retry });
1303
- return await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CREATE_TASK_RUN_ATTEMPT", {
1304
- version: "v1",
1305
- runId: message.runId
1306
- });
1307
- });
1308
- if (!createAttempt.success) {
1309
- backgroundWorker.attemptCreatedNotification.post({
1310
- success: false,
1311
- reason: `Failed to create attempt with backoff due to ${createAttempt.cause}. ${createAttempt.error}`
1312
- });
1313
- return;
1314
- }
1315
- if (!createAttempt.result.success) {
1316
- backgroundWorker.attemptCreatedNotification.post({
1317
- success: false,
1318
- reason: createAttempt.result.reason
1319
- });
1320
- return;
1321
- }
1322
- backgroundWorker.attemptCreatedNotification.post({
1323
- success: true,
1324
- execution: createAttempt.result.executionPayload.execution
1325
- });
1326
- });
1327
- backgroundWorker.attemptCreatedNotification.attach((message) => {
1328
- logger.log("attemptCreatedNotification", {
1329
- success: message.success,
1330
- ...message.success ? {
1331
- attempt: message.execution.attempt,
1332
- queue: message.execution.queue,
1333
- worker: message.execution.worker,
1334
- machine: message.execution.machine
1335
- } : {
1336
- reason: message.reason
1337
- }
1338
- });
1339
- if (!message.success) {
1340
- return;
1341
- }
1342
- this.attemptFriendlyId = message.execution.attempt.id;
1343
- });
1344
- backgroundWorker.onWaitForDuration.attach(async (message) => {
1345
- logger.log("onWaitForDuration", { ...message, drift: Date.now() - message.now });
1346
- noResume: {
1347
- const { ms, waitThresholdInMs } = message;
1348
- const internalTimeout = unboundedTimeout(ms, "internal");
1349
- const checkpointSafeInternalTimeout = checkpointSafeTimeout(ms);
1350
- if (ms < waitThresholdInMs) {
1351
- await internalTimeout;
1352
- break noResume;
1353
- }
1354
- const waitForDuration = await defaultBackoff.execute(async ({ retry }) => {
1355
- logger.log("Wait for duration with backoff", { retry });
1356
- if (!this.attemptFriendlyId) {
1357
- logger.error("Failed to send wait message, attempt friendly ID not set", { message });
1358
- throw new ExponentialBackoff.StopRetrying("No attempt ID");
1359
- }
1360
- return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_DURATION", {
1361
- ...message,
1362
- attemptFriendlyId: this.attemptFriendlyId
1363
- });
1364
- });
1365
- if (!waitForDuration.success) {
1366
- logger.error("Failed to wait for duration with backoff", {
1367
- cause: waitForDuration.cause,
1368
- error: waitForDuration.error
1369
- });
1370
- this.#emitUnrecoverableError(
1371
- "WaitForDurationFailed",
1372
- `${waitForDuration.cause}: ${waitForDuration.error}`
1373
- );
1374
- return;
1375
- }
1376
- const { willCheckpointAndRestore } = waitForDuration.result;
1377
- if (!willCheckpointAndRestore) {
1378
- await internalTimeout;
1379
- break noResume;
1380
- }
1381
- await this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
1382
- await Promise.race([internalTimeout, checkpointSafeInternalTimeout]);
1383
- try {
1384
- const { checkpointCanceled } = await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CANCEL_CHECKPOINT", {
1385
- version: "v2",
1386
- reason: "WAIT_FOR_DURATION"
1387
- });
1388
- logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
1389
- if (checkpointCanceled) {
1390
- break noResume;
1391
- }
1392
- logger.log("Waiting for external duration resume as we may have been restored");
1393
- const idempotencyKey = randomUUID();
1394
- this.durationResumeFallback = { idempotencyKey };
1395
- setTimeout(() => {
1396
- if (!this.durationResumeFallback) {
1397
- logger.error("Already resumed after duration, skipping fallback");
1398
- return;
1399
- }
1400
- if (this.durationResumeFallback.idempotencyKey !== idempotencyKey) {
1401
- logger.error("Duration resume idempotency key mismatch, skipping fallback");
1402
- return;
1403
- }
1404
- logger.log("Resuming after duration with fallback");
1405
- this.#resumeAfterDuration();
1406
- }, 15e3);
1407
- } catch (error) {
1408
- logger.debug("Checkpoint cancellation timed out", { error });
1409
- break noResume;
1410
- }
1411
- return;
1412
- }
1413
- this.#resumeAfterDuration();
1414
- });
1415
- backgroundWorker.onWaitForTask.attach(this.#waitForTaskHandler.bind(this));
1416
- backgroundWorker.onWaitForBatch.attach(this.#waitForBatchHandler.bind(this));
1417
- return backgroundWorker;
1418
- }
1419
- async #prepareForWait(reason, willCheckpointAndRestore) {
1420
- logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
1421
- if (!willCheckpointAndRestore) {
1422
- return;
1423
- }
1424
- this.paused = true;
1425
- this.nextResumeAfter = reason;
1426
- this.waitForPostStart = true;
1427
- await this.#prepareForCheckpoint();
1428
- }
1429
- // MARK: RETRY PREP
1430
- async #prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode) {
1431
- logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit, exitCode });
1432
- if (shouldExit) {
1433
- if (willCheckpointAndRestore) {
1434
- logger.error("WARNING: Will checkpoint but also requested exit. This won't end well.");
1435
- }
1436
- await this.#exitGracefully(false, exitCode);
1437
- return;
1438
- }
1439
- this.paused = false;
1440
- this.waitForPostStart = false;
1441
- this.executing = false;
1442
- this.attemptFriendlyId = void 0;
1443
- if (!willCheckpointAndRestore) {
1444
- return;
1445
- }
1446
- this.waitForPostStart = true;
1447
- await this.#prepareForCheckpoint(false);
1448
- }
1449
- // MARK: CHECKPOINT PREP
1450
- async #prepareForCheckpoint(flush = true) {
1451
- if (flush) {
1452
- try {
1453
- await this.#backgroundWorker.flushTelemetry();
1454
- } catch (error) {
1455
- logger.error(
1456
- "Failed to flush telemetry while preparing for checkpoint, will proceed anyway",
1457
- { error }
1458
- );
1459
- }
1460
- }
1461
- try {
1462
- await this.#backgroundWorker.forceKillOldTaskRunProcesses();
1463
- } catch (error) {
1464
- logger.error(
1465
- "Failed to kill previous worker while preparing for checkpoint, will proceed anyway",
1466
- { error }
1467
- );
1468
- }
1469
- this.#readyForCheckpoint();
1470
- }
1471
- #resumeAfterDuration() {
1472
- this.paused = false;
1473
- this.nextResumeAfter = void 0;
1474
- this.waitForPostStart = false;
1475
- this.#backgroundWorker.waitCompletedNotification();
1476
- }
1477
- async #readyForLazyAttempt() {
1478
- const idempotencyKey = randomUUID();
1479
- this.readyForLazyAttemptReplay = {
1480
- idempotencyKey
1481
- };
1482
- for await (const { delay, retry } of defaultBackoff.min(10).maxRetries(3)) {
1483
- if (retry > 0) {
1484
- logger.log("retrying ready for lazy attempt", { retry });
1485
- }
1486
- this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
1487
- version: "v1",
1488
- runId: this.runId,
1489
- totalCompletions: this.completed.size
1490
- });
1491
- await timeout2(delay.milliseconds);
1492
- if (!this.readyForLazyAttemptReplay) {
1493
- logger.error("replay ready for lazy attempt cancelled, discarding", {
1494
- idempotencyKey
1495
- });
1496
- return;
1497
- }
1498
- if (idempotencyKey !== this.readyForLazyAttemptReplay.idempotencyKey) {
1499
- logger.error("replay ready for lazy attempt idempotency key mismatch, discarding", {
1500
- idempotencyKey,
1501
- newIdempotencyKey: this.readyForLazyAttemptReplay.idempotencyKey
1502
- });
1503
- return;
1504
- }
1505
- }
1506
- this.#failRun(this.runId, "Failed to receive execute request in a reasonable time");
1507
- }
1508
- #readyForCheckpoint() {
1509
- this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
1510
- }
1511
- #failRun(anyRunId, error) {
1512
- logger.error("Failing run", { anyRunId, error });
1513
- const completion = {
1514
- ok: false,
1515
- id: anyRunId,
1516
- retry: void 0,
1517
- error: error instanceof Error ? {
1518
- type: "BUILT_IN_ERROR",
1519
- name: error.name,
1520
- message: error.message,
1521
- stackTrace: error.stack ?? ""
1522
- } : {
1523
- type: "BUILT_IN_ERROR",
1524
- name: "UnknownError",
1525
- message: String(error),
1526
- stackTrace: ""
1527
- }
1528
- };
1529
- this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
1530
- version: "v1",
1531
- completion
1532
- });
1533
- }
1534
- // MARK: ATTEMPT COMPLETION
1535
- async #submitAttemptCompletion(execution, completion, replayIdempotencyKey) {
1536
- const taskRunCompleted = await defaultBackoff.execute(async ({ retry }) => {
1537
- logger.log("Submit attempt completion with backoff", { retry });
1538
- return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("TASK_RUN_COMPLETED", {
1539
- version: "v1",
1540
- execution,
1541
- completion
1542
- });
1543
- });
1544
- if (!taskRunCompleted.success) {
1545
- logger.error("Failed to complete lazy attempt with backoff", {
1546
- cause: taskRunCompleted.cause,
1547
- error: taskRunCompleted.error
1548
- });
1549
- this.#failRun(execution.run.id, taskRunCompleted.error);
1550
- return;
1551
- }
1552
- const { willCheckpointAndRestore, shouldExit } = taskRunCompleted.result;
1553
- logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
1554
- const exitCode = !completion.ok && completion.error.type === "INTERNAL_ERROR" && completion.error.code === TaskRunErrorCodes2.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE ? EXIT_CODE_CHILD_NONZERO : 0;
1555
- await this.#prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode);
1556
- if (willCheckpointAndRestore) {
1557
- if (!this.submitAttemptCompletionReplay) {
1558
- this.submitAttemptCompletionReplay = {
1559
- message: {
1560
- execution,
1561
- completion
1562
- },
1563
- attempt: 1,
1564
- idempotencyKey: randomUUID()
1565
- };
1566
- } else {
1567
- if (replayIdempotencyKey && replayIdempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
1568
- logger.error(
1569
- "attempt completion handler called with mismatched idempotency key, won't overwrite replay request"
1570
- );
1571
- return;
1572
- }
1573
- this.submitAttemptCompletionReplay.attempt++;
1574
- }
1575
- }
1576
- }
1577
- #returnValidatedExtraHeaders(headers) {
1578
- for (const [key, value] of Object.entries(headers)) {
1579
- if (value === void 0) {
1580
- throw new Error(`Extra header is undefined: ${key}`);
1581
- }
1582
- }
1583
- return headers;
1584
- }
1585
- // MARK: COORDINATOR SOCKET
1586
- #createCoordinatorSocket(host) {
1587
- const extraHeaders = this.#returnValidatedExtraHeaders({
1588
- "x-machine-name": MACHINE_NAME,
1589
- "x-pod-name": POD_NAME,
1590
- "x-trigger-content-hash": this.contentHash,
1591
- "x-trigger-project-ref": this.projectRef,
1592
- "x-trigger-env-id": this.envId,
1593
- "x-trigger-deployment-id": this.deploymentId,
1594
- "x-trigger-run-id": this.runId,
1595
- "x-trigger-deployment-version": this.deploymentVersion
1596
- });
1597
- if (this.attemptFriendlyId) {
1598
- extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
1599
- }
1600
- logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
1601
- logger.debug(`connecting with extra headers`, { extraHeaders });
1602
- const coordinatorConnection = new ZodSocketConnection({
1603
- namespace: "prod-worker",
1604
- host,
1605
- port: COORDINATOR_PORT,
1606
- clientMessages: ProdWorkerToCoordinatorMessages,
1607
- serverMessages: CoordinatorToProdWorkerMessages,
1608
- extraHeaders,
1609
- ioOptions: {
1610
- reconnectionDelay: 1e3,
1611
- reconnectionDelayMax: 3e3
1612
- },
1613
- handlers: {
1614
- RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
1615
- if (!this.paused) {
1616
- logger.error("Failed to resume after dependency: Worker not paused");
1617
- return;
1618
- }
1619
- if (completions.length === 0) {
1620
- logger.error("Failed to resume after dependency: No completions");
1621
- return;
1622
- }
1623
- if (this.nextResumeAfter !== "WAIT_FOR_TASK" && this.nextResumeAfter !== "WAIT_FOR_BATCH") {
1624
- logger.error("Failed to resume after dependency: Invalid next resume", {
1625
- nextResumeAfter: this.nextResumeAfter
1626
- });
1627
- return;
1628
- }
1629
- if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
1630
- logger.error(
1631
- "Failed to resume after dependency: Waiting for single task but got multiple completions",
1632
- {
1633
- completions
1634
- }
1635
- );
1636
- return;
1637
- }
1638
- switch (this.nextResumeAfter) {
1639
- case "WAIT_FOR_TASK": {
1640
- this.waitForTaskReplay = void 0;
1641
- break;
1642
- }
1643
- case "WAIT_FOR_BATCH": {
1644
- this.waitForBatchReplay = void 0;
1645
- break;
1646
- }
1647
- }
1648
- this.paused = false;
1649
- this.nextResumeAfter = void 0;
1650
- this.waitForPostStart = false;
1651
- for (let i = 0; i < completions.length; i++) {
1652
- const completion = completions[i];
1653
- if (!completion)
1654
- continue;
1655
- this.#backgroundWorker.taskRunCompletedNotification(completion);
1656
- }
1657
- },
1658
- RESUME_AFTER_DURATION: async (message) => {
1659
- if (!this.paused) {
1660
- logger.error("worker not paused", {
1661
- attemptId: message.attemptId
1662
- });
1663
- return;
1664
- }
1665
- if (this.nextResumeAfter !== "WAIT_FOR_DURATION") {
1666
- logger.error("not waiting to resume after duration", {
1667
- nextResumeAfter: this.nextResumeAfter
1668
- });
1669
- return;
1670
- }
1671
- this.durationResumeFallback = void 0;
1672
- this.#resumeAfterDuration();
1673
- },
1674
- // Deprecated: This will never get called as this worker supports lazy attempts. It's only here for a quick view of the flow old workers use.
1675
- EXECUTE_TASK_RUN: async ({ executionPayload }) => {
1676
- if (this.executing) {
1677
- logger.error("dropping execute request, already executing");
1678
- return;
1679
- }
1680
- if (this.completed.has(executionPayload.execution.attempt.id)) {
1681
- logger.error("dropping execute request, already completed");
1682
- return;
1683
- }
1684
- this.executing = true;
1685
- this.attemptFriendlyId = executionPayload.execution.attempt.id;
1686
- const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
1687
- logger.log("completed", completion);
1688
- this.completed.add(executionPayload.execution.attempt.id);
1689
- const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
1690
- version: "v1",
1691
- execution: executionPayload.execution,
1692
- completion
1693
- });
1694
- logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
1695
- await this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
1696
- },
1697
- EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
1698
- this.readyForLazyAttemptReplay = void 0;
1699
- if (this.executing) {
1700
- logger.error("dropping execute request, already executing");
1701
- return;
1702
- }
1703
- const attemptCount = message.lazyPayload.attemptCount ?? 0;
1704
- logger.log("execute attempt counts", { attemptCount, completed: this.completed.size });
1705
- if (this.completed.size > 0 && this.completed.size >= attemptCount + 1) {
1706
- logger.error("dropping execute request, already completed");
1707
- return;
1708
- }
1709
- this.executing = true;
1710
- try {
1711
- const { completion, execution } = await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
1712
- logger.log("completed", completion);
1713
- this.completed.add(execution.attempt.id);
1714
- await this.#submitAttemptCompletion(execution, completion);
1715
- } catch (error) {
1716
- logger.error("Failed to complete lazy attempt", {
1717
- error
1718
- });
1719
- this.#failRun(message.lazyPayload.runId, error);
1720
- }
1721
- },
1722
- REQUEST_ATTEMPT_CANCELLATION: async (message) => {
1723
- if (!this.executing) {
1724
- logger.log("dropping cancel request, not executing", { status: this.#status });
1725
- return;
1726
- }
1727
- logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
1728
- await this.#backgroundWorker.cancelAttempt(message.attemptId);
1729
- },
1730
- REQUEST_EXIT: async (message) => {
1731
- if (message.version === "v2" && message.delayInMs) {
1732
- logger.log("exit requested with delay", { delayInMs: message.delayInMs });
1733
- await timeout2(message.delayInMs);
1734
- }
1735
- this.#coordinatorSocket.close();
1736
- process.exit(0);
1737
- },
1738
- READY_FOR_RETRY: async (message) => {
1739
- if (this.completed.size < 1) {
1740
- logger.error("Received READY_FOR_RETRY but no completions yet. This is a bug.");
1741
- return;
1742
- }
1743
- this.submitAttemptCompletionReplay = void 0;
1744
- await this.#readyForLazyAttempt();
1745
- }
1746
- },
1747
- // MARK: ON CONNECTION
1748
- onConnection: async (socket, handler, sender, logger2) => {
1749
- logger2.log("connected to coordinator", {
1750
- status: this.#status,
1751
- connectionCount: ++this.connectionCount
1752
- });
1753
- socket.emit("SET_STATE", { version: "v1", attemptFriendlyId: this.attemptFriendlyId });
1754
- try {
1755
- if (this.waitForPostStart) {
1756
- logger2.log("skip connection handler, waiting for post start hook");
1757
- return;
1758
- }
1759
- if (this.paused) {
1760
- if (!this.nextResumeAfter) {
1761
- logger2.error("Missing next resume reason", { status: this.#status });
1762
- this.#emitUnrecoverableError(
1763
- "NoNextResume",
1764
- "Next resume reason not set while resuming from paused state"
1765
- );
1766
- return;
1767
- }
1768
- if (!this.attemptFriendlyId) {
1769
- logger2.error("Missing friendly ID", { status: this.#status });
1770
- this.#emitUnrecoverableError(
1771
- "NoAttemptId",
1772
- "Attempt ID not set while resuming from paused state"
1773
- );
1774
- return;
1775
- }
1776
- socket.emit("READY_FOR_RESUME", {
1777
- version: "v1",
1778
- attemptFriendlyId: this.attemptFriendlyId,
1779
- type: this.nextResumeAfter
1780
- });
1781
- return;
1782
- }
1783
- if (process.env.INDEX_TASKS === "true") {
1784
- const failIndex = (error) => {
1785
- socket.emit("INDEXING_FAILED", {
1786
- version: "v1",
1787
- deploymentId: this.deploymentId,
1788
- error
1789
- });
1790
- };
1791
- process.removeAllListeners("uncaughtException");
1792
- process.on("uncaughtException", (error) => {
1793
- console.error("Uncaught exception while indexing", error);
1794
- failIndex(error);
1795
- });
1796
- try {
1797
- const taskResources = await this.#initializeWorker();
1798
- const indexTasks = await defaultBackoff.maxRetries(3).execute(async () => {
1799
- return await socket.timeout(2e4).emitWithAck("INDEX_TASKS", {
1800
- version: "v2",
1801
- deploymentId: this.deploymentId,
1802
- ...taskResources,
1803
- supportsLazyAttempts: true
1804
- });
1805
- });
1806
- if (!indexTasks.success || !indexTasks.result.success) {
1807
- logger2.error("indexing failure, shutting down..", { indexTasks });
1808
- process.exit(1);
1809
- } else {
1810
- logger2.info("indexing done, shutting down..");
1811
- process.exit(0);
1812
- }
1813
- } catch (e) {
1814
- const stderr = this.#backgroundWorker.stderr.join("\n");
1815
- if (e instanceof TaskMetadataParseError) {
1816
- logger2.error("tasks metadata parse error", {
1817
- zodIssues: e.zodIssues,
1818
- tasks: e.tasks
1819
- });
1820
- failIndex({
1821
- name: "TaskMetadataParseError",
1822
- message: "There was an error parsing the task metadata",
1823
- stack: JSON.stringify({ zodIssues: e.zodIssues, tasks: e.tasks }),
1824
- stderr
1825
- });
1826
- } else if (e instanceof UncaughtExceptionError) {
1827
- const error = {
1828
- name: e.originalError.name,
1829
- message: e.originalError.message,
1830
- stack: e.originalError.stack,
1831
- stderr
1832
- };
1833
- logger2.error("uncaught exception", { originalError: error });
1834
- failIndex(error);
1835
- } else if (e instanceof Error) {
1836
- const error = {
1837
- name: e.name,
1838
- message: e.message,
1839
- stack: e.stack,
1840
- stderr
1841
- };
1842
- logger2.error("error", { error });
1843
- failIndex(error);
1844
- } else if (typeof e === "string") {
1845
- logger2.error("string error", { error: { message: e } });
1846
- failIndex({
1847
- name: "Error",
1848
- message: e,
1849
- stderr
1850
- });
1851
- } else {
1852
- logger2.error("unknown error", { error: e });
1853
- failIndex({
1854
- name: "Error",
1855
- message: "Unknown error",
1856
- stderr
1857
- });
1858
- }
1859
- await timeout2(1e3);
1860
- process.exit(EXIT_CODE_ALREADY_HANDLED);
1861
- }
1862
- }
1863
- if (this.executing) {
1864
- return;
1865
- }
1866
- process.removeAllListeners("uncaughtException");
1867
- process.on("uncaughtException", (error) => {
1868
- console.error("Uncaught exception during run", error);
1869
- this.#failRun(this.runId, error);
1870
- });
1871
- await this.#readyForLazyAttempt();
1872
- } catch (error) {
1873
- logger2.error("connection handler error", { error });
1874
- } finally {
1875
- if (this.connectionCount === 1) {
1876
- return;
1877
- }
1878
- this.#handleReplays();
1879
- }
1880
- },
1881
- onError: async (socket, err, logger2) => {
1882
- logger2.error("onError", {
1883
- error: {
1884
- name: err.name,
1885
- message: err.message
1886
- }
1887
- });
1888
- }
1889
- });
1890
- return coordinatorConnection;
1891
- }
1892
- // MARK: REPLAYS
1893
- async #handleReplays() {
1894
- const backoff = new ExponentialBackoff().type("FullJitter").maxRetries(3);
1895
- const replayCancellationDelay = 2e4;
1896
- if (this.waitForTaskReplay) {
1897
- logger.log("replaying wait for task", { ...this.waitForTaskReplay });
1898
- const { idempotencyKey, message, attempt } = this.waitForTaskReplay;
1899
- await timeout2(replayCancellationDelay);
1900
- if (!this.waitForTaskReplay) {
1901
- logger.error("wait for task replay cancelled, discarding", {
1902
- originalMessage: { idempotencyKey, message, attempt }
1903
- });
1904
- return;
1905
- }
1906
- if (idempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
1907
- logger.error("wait for task replay idempotency key mismatch, discarding", {
1908
- originalMessage: { idempotencyKey, message, attempt },
1909
- newMessage: this.waitForTaskReplay
1910
- });
1911
- return;
1912
- }
1913
- try {
1914
- await backoff.wait(attempt + 1);
1915
- await this.#waitForTaskHandler(message);
1916
- } catch (error) {
1917
- if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
1918
- logger.error("wait for task replay retry limit exceeded", { error });
1919
- } else {
1920
- logger.error("wait for task replay error", { error });
1921
- }
1922
- }
1923
- return;
1924
- }
1925
- if (this.waitForBatchReplay) {
1926
- logger.log("replaying wait for batch", {
1927
- ...this.waitForBatchReplay,
1928
- cancellationDelay: replayCancellationDelay
1929
- });
1930
- const { idempotencyKey, message, attempt } = this.waitForBatchReplay;
1931
- await timeout2(replayCancellationDelay);
1932
- if (!this.waitForBatchReplay) {
1933
- logger.error("wait for batch replay cancelled, discarding", {
1934
- originalMessage: { idempotencyKey, message, attempt }
1935
- });
1936
- return;
1937
- }
1938
- if (idempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
1939
- logger.error("wait for batch replay idempotency key mismatch, discarding", {
1940
- originalMessage: { idempotencyKey, message, attempt },
1941
- newMessage: this.waitForBatchReplay
1942
- });
1943
- return;
1944
- }
1945
- try {
1946
- await backoff.wait(attempt + 1);
1947
- await this.#waitForBatchHandler(message);
1948
- } catch (error) {
1949
- if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
1950
- logger.error("wait for batch replay retry limit exceeded", { error });
1951
- } else {
1952
- logger.error("wait for batch replay error", { error });
1953
- }
1954
- }
1955
- return;
1956
- }
1957
- if (this.submitAttemptCompletionReplay) {
1958
- logger.log("replaying attempt completion", {
1959
- ...this.submitAttemptCompletionReplay,
1960
- cancellationDelay: replayCancellationDelay
1961
- });
1962
- const { idempotencyKey, message, attempt } = this.submitAttemptCompletionReplay;
1963
- await timeout2(replayCancellationDelay);
1964
- if (!this.submitAttemptCompletionReplay) {
1965
- logger.error("attempt completion replay cancelled, discarding", {
1966
- originalMessage: { idempotencyKey, message, attempt }
1967
- });
1968
- return;
1969
- }
1970
- if (idempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
1971
- logger.error("attempt completion replay idempotency key mismatch, discarding", {
1972
- originalMessage: { idempotencyKey, message, attempt },
1973
- newMessage: this.submitAttemptCompletionReplay
1974
- });
1975
- return;
1976
- }
1977
- try {
1978
- await backoff.wait(attempt + 1);
1979
- await this.#submitAttemptCompletion(message.execution, message.completion, idempotencyKey);
1980
- } catch (error) {
1981
- if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
1982
- logger.error("attempt completion replay retry limit exceeded", { error });
1983
- } else {
1984
- logger.error("attempt completion replay error", { error });
1985
- }
1986
- }
1987
- return;
1988
- }
1989
- }
1990
- // MARK: HTTP SERVER
1991
- #createHttpServer() {
1992
- const httpServer = createServer(async (req, res) => {
1993
- logger.log(`[${req.method}]`, req.url);
1994
- const reply = new HttpReply(res);
1995
- try {
1996
- const url = new URL(req.url ?? "", `http://${req.headers.host}`);
1997
- switch (url.pathname) {
1998
- case "/health": {
1999
- return reply.text("ok");
2000
- }
2001
- case "/status": {
2002
- return reply.json(this.#status);
2003
- }
2004
- case "/connect": {
2005
- this.#coordinatorSocket.connect();
2006
- return reply.text("Connected to coordinator");
2007
- }
2008
- case "/close": {
2009
- this.#coordinatorSocket.close();
2010
- this.connectionCount = 0;
2011
- return reply.text("Disconnected from coordinator");
2012
- }
2013
- case "/test": {
2014
- await this.#coordinatorSocket.socket.timeout(1e4).emitWithAck("TEST", {
2015
- version: "v1"
2016
- });
2017
- return reply.text("Received ACK from coordinator");
2018
- }
2019
- case "/preStop": {
2020
- const cause = PreStopCauses.safeParse(url.searchParams.get("cause"));
2021
- if (!cause.success) {
2022
- logger.error("Failed to parse cause", { cause });
2023
- return reply.text("Failed to parse cause", 400);
2024
- }
2025
- switch (cause.data) {
2026
- case "terminate": {
2027
- break;
2028
- }
2029
- default: {
2030
- logger.error("Unhandled cause", { cause: cause.data });
2031
- break;
2032
- }
2033
- }
2034
- return reply.text("preStop ok");
2035
- }
2036
- case "/postStart": {
2037
- const cause = PostStartCauses.safeParse(url.searchParams.get("cause"));
2038
- if (!cause.success) {
2039
- logger.error("Failed to parse cause", { cause });
2040
- return reply.text("Failed to parse cause", 400);
2041
- }
2042
- switch (cause.data) {
2043
- case "index": {
2044
- break;
2045
- }
2046
- case "create": {
2047
- break;
2048
- }
2049
- case "restore": {
2050
- await this.#reconnectAfterPostStart();
2051
- break;
2052
- }
2053
- default: {
2054
- logger.error("Unhandled cause", { cause: cause.data });
2055
- break;
2056
- }
2057
- }
2058
- return reply.text("postStart ok");
2059
- }
2060
- default: {
2061
- return reply.empty(404);
2062
- }
2063
- }
2064
- } catch (error) {
2065
- logger.error("HTTP server error", { error });
2066
- reply.empty(500);
2067
- }
2068
- });
2069
- httpServer.on("clientError", (err, socket) => {
2070
- socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
2071
- });
2072
- httpServer.on("listening", () => {
2073
- logger.log("http server listening on port", this.#httpPort);
2074
- });
2075
- httpServer.on("error", async (error) => {
2076
- if (error.code != "EADDRINUSE") {
2077
- return;
2078
- }
2079
- logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
2080
- this.#httpPort = getRandomPortNumber();
2081
- await timeout2(100);
2082
- this.start();
2083
- });
2084
- return httpServer;
2085
- }
2086
- async #initializeWorker() {
2087
- const envVars = await this.#fetchEnvironmentVariables();
2088
- await this.#backgroundWorker.initialize({ env: envVars });
2089
- let packageVersion;
2090
- const taskResources = [];
2091
- if (!this.#backgroundWorker.tasks || this.#backgroundWorker.tasks.length === 0) {
2092
- throw new Error(
2093
- `Background Worker started without tasks. Searched in: ${__PROJECT_CONFIG__.triggerDirectories?.join(
2094
- ", "
2095
- )}`
2096
- );
2097
- }
2098
- for (const task of this.#backgroundWorker.tasks) {
2099
- taskResources.push(task);
2100
- packageVersion = task.packageVersion;
2101
- }
2102
- if (!packageVersion) {
2103
- throw new Error(`Background Worker started without package version`);
2104
- }
2105
- return {
2106
- packageVersion,
2107
- tasks: taskResources
2108
- };
2109
- }
2110
- async #fetchEnvironmentVariables() {
2111
- const response = await fetch(`${this.apiUrl}/api/v1/projects/${this.projectRef}/envvars`, {
2112
- method: "GET",
2113
- headers: {
2114
- Authorization: `Bearer ${this.apiKey}`
2115
- }
2116
- });
2117
- if (!response.ok) {
2118
- return {};
2119
- }
2120
- const data = await response.json();
2121
- return data?.variables ?? {};
2122
- }
2123
- get #status() {
2124
- return {
2125
- executing: this.executing,
2126
- paused: this.paused,
2127
- completed: this.completed.size,
2128
- nextResumeAfter: this.nextResumeAfter,
2129
- waitForPostStart: this.waitForPostStart,
2130
- attemptFriendlyId: this.attemptFriendlyId,
2131
- waitForTaskReplay: this.waitForTaskReplay,
2132
- waitForBatchReplay: this.waitForBatchReplay
2133
- };
2134
- }
2135
- #emitUnrecoverableError(name, message) {
2136
- this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
2137
- version: "v1",
2138
- error: {
2139
- name,
2140
- message
2141
- }
2142
- });
2143
- }
2144
- start() {
2145
- this.#httpServer.listen(this.#httpPort, this.host);
2146
- }
2147
- };
2148
- var prodWorker = new ProdWorker(HTTP_SERVER_PORT);
2149
- prodWorker.start();
2150
- function gatherProcessEnv() {
2151
- const env = {
2152
- NODE_ENV: process.env.NODE_ENV ?? "production",
2153
- PATH: process.env.PATH,
2154
- USER: process.env.USER,
2155
- SHELL: process.env.SHELL,
2156
- LANG: process.env.LANG,
2157
- TERM: process.env.TERM,
2158
- NODE_PATH: process.env.NODE_PATH,
2159
- HOME: process.env.HOME,
2160
- NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS
2161
- };
2162
- return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== void 0));
2163
- }