trigger.dev 3.0.0-beta.54 → 3.0.0-beta.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,7 +8,7 @@ import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
|
|
|
8
8
|
import "source-map-support/register.js";
|
|
9
9
|
|
|
10
10
|
// package.json
|
|
11
|
-
var version = "3.0.0-beta.
|
|
11
|
+
var version = "3.0.0-beta.56";
|
|
12
12
|
|
|
13
13
|
// src/workers/dev/worker-setup.ts
|
|
14
14
|
__SETUP_IMPORTED_PROJECT_CONFIG__;
|
|
@@ -1085,6 +1085,7 @@ import { randomUUID } from "node:crypto";
|
|
|
1085
1085
|
import { readFile } from "node:fs/promises";
|
|
1086
1086
|
import { createServer } from "node:http";
|
|
1087
1087
|
import { setTimeout as timeout2 } from "node:timers/promises";
|
|
1088
|
+
import { Evt as Evt2 } from "evt";
|
|
1088
1089
|
var HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
|
|
1089
1090
|
var COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
|
1090
1091
|
var COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
|
|
@@ -1117,13 +1118,14 @@ var ProdWorker = class {
|
|
|
1117
1118
|
completed = /* @__PURE__ */ new Set();
|
|
1118
1119
|
paused = false;
|
|
1119
1120
|
attemptFriendlyId;
|
|
1121
|
+
attemptNumber;
|
|
1120
1122
|
nextResumeAfter;
|
|
1121
1123
|
waitForPostStart = false;
|
|
1122
1124
|
connectionCount = 0;
|
|
1125
|
+
restoreNotification = Evt2.create();
|
|
1123
1126
|
waitForTaskReplay;
|
|
1124
1127
|
waitForBatchReplay;
|
|
1125
1128
|
readyForLazyAttemptReplay;
|
|
1126
|
-
submitAttemptCompletionReplay;
|
|
1127
1129
|
durationResumeFallback;
|
|
1128
1130
|
#httpPort;
|
|
1129
1131
|
#backgroundWorker;
|
|
@@ -1183,98 +1185,118 @@ var ProdWorker = class {
|
|
|
1183
1185
|
}
|
|
1184
1186
|
}
|
|
1185
1187
|
// MARK: TASK WAIT
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
logger.log("
|
|
1189
|
-
if (
|
|
1190
|
-
logger.error("
|
|
1191
|
-
|
|
1188
|
+
#waitForTaskHandlerFactory(workerId) {
|
|
1189
|
+
return async (message, replayIdempotencyKey) => {
|
|
1190
|
+
logger.log("onWaitForTask", { workerId, message });
|
|
1191
|
+
if (this.nextResumeAfter) {
|
|
1192
|
+
logger.error("Already waiting for resume, skipping wait for task", {
|
|
1193
|
+
nextResumeAfter: this.nextResumeAfter
|
|
1194
|
+
});
|
|
1195
|
+
return;
|
|
1192
1196
|
}
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1197
|
+
const waitForTask = await defaultBackoff.execute(async ({ retry }) => {
|
|
1198
|
+
logger.log("Wait for task with backoff", { retry });
|
|
1199
|
+
if (!this.attemptFriendlyId) {
|
|
1200
|
+
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1201
|
+
throw new ExponentialBackoff.StopRetrying("No attempt ID");
|
|
1202
|
+
}
|
|
1203
|
+
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_TASK", {
|
|
1204
|
+
version: "v2",
|
|
1205
|
+
friendlyId: message.friendlyId,
|
|
1206
|
+
attemptFriendlyId: this.attemptFriendlyId
|
|
1207
|
+
});
|
|
1203
1208
|
});
|
|
1204
|
-
|
|
1205
|
-
"
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1209
|
+
if (!waitForTask.success) {
|
|
1210
|
+
logger.error("Failed to wait for task with backoff", {
|
|
1211
|
+
cause: waitForTask.cause,
|
|
1212
|
+
error: waitForTask.error
|
|
1213
|
+
});
|
|
1214
|
+
this.#emitUnrecoverableError(
|
|
1215
|
+
"WaitForTaskFailed",
|
|
1216
|
+
`${waitForTask.cause}: ${waitForTask.error}`
|
|
1217
|
+
);
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1220
|
+
const { willCheckpointAndRestore } = waitForTask.result;
|
|
1221
|
+
await this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
|
|
1222
|
+
if (willCheckpointAndRestore) {
|
|
1223
|
+
if (!this.waitForTaskReplay) {
|
|
1224
|
+
this.waitForTaskReplay = {
|
|
1225
|
+
message,
|
|
1226
|
+
attempt: 1,
|
|
1227
|
+
idempotencyKey: randomUUID()
|
|
1228
|
+
};
|
|
1229
|
+
} else {
|
|
1230
|
+
if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForTaskReplay.idempotencyKey) {
|
|
1231
|
+
logger.error(
|
|
1232
|
+
"wait for task handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1233
|
+
);
|
|
1234
|
+
return;
|
|
1235
|
+
}
|
|
1236
|
+
this.waitForTaskReplay.attempt++;
|
|
1225
1237
|
}
|
|
1226
|
-
this.waitForTaskReplay.attempt++;
|
|
1227
1238
|
}
|
|
1228
|
-
}
|
|
1239
|
+
};
|
|
1229
1240
|
}
|
|
1230
1241
|
// MARK: BATCH WAIT
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
logger.log("
|
|
1234
|
-
if (
|
|
1235
|
-
logger.error("
|
|
1236
|
-
|
|
1242
|
+
#waitForBatchHandlerFactory(workerId) {
|
|
1243
|
+
return async (message, replayIdempotencyKey) => {
|
|
1244
|
+
logger.log("onWaitForBatch", { workerId, message });
|
|
1245
|
+
if (this.nextResumeAfter) {
|
|
1246
|
+
logger.error("Already waiting for resume, skipping wait for batch", {
|
|
1247
|
+
nextResumeAfter: this.nextResumeAfter
|
|
1248
|
+
});
|
|
1249
|
+
return;
|
|
1237
1250
|
}
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1251
|
+
const waitForBatch = await defaultBackoff.execute(async ({ retry }) => {
|
|
1252
|
+
logger.log("Wait for batch with backoff", { retry });
|
|
1253
|
+
if (!this.attemptFriendlyId) {
|
|
1254
|
+
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
|
1255
|
+
throw new ExponentialBackoff.StopRetrying("No attempt ID");
|
|
1256
|
+
}
|
|
1257
|
+
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("WAIT_FOR_BATCH", {
|
|
1258
|
+
version: "v2",
|
|
1259
|
+
batchFriendlyId: message.batchFriendlyId,
|
|
1260
|
+
runFriendlyIds: message.runFriendlyIds,
|
|
1261
|
+
attemptFriendlyId: this.attemptFriendlyId
|
|
1262
|
+
});
|
|
1249
1263
|
});
|
|
1250
|
-
|
|
1251
|
-
"
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1264
|
+
if (!waitForBatch.success) {
|
|
1265
|
+
logger.error("Failed to wait for batch with backoff", {
|
|
1266
|
+
cause: waitForBatch.cause,
|
|
1267
|
+
error: waitForBatch.error
|
|
1268
|
+
});
|
|
1269
|
+
this.#emitUnrecoverableError(
|
|
1270
|
+
"WaitForBatchFailed",
|
|
1271
|
+
`${waitForBatch.cause}: ${waitForBatch.error}`
|
|
1272
|
+
);
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
const { willCheckpointAndRestore } = waitForBatch.result;
|
|
1276
|
+
await this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
|
|
1277
|
+
if (willCheckpointAndRestore) {
|
|
1278
|
+
if (!this.waitForBatchReplay) {
|
|
1279
|
+
this.waitForBatchReplay = {
|
|
1280
|
+
message,
|
|
1281
|
+
attempt: 1,
|
|
1282
|
+
idempotencyKey: randomUUID()
|
|
1283
|
+
};
|
|
1284
|
+
} else {
|
|
1285
|
+
if (replayIdempotencyKey && replayIdempotencyKey !== this.waitForBatchReplay.idempotencyKey) {
|
|
1286
|
+
logger.error(
|
|
1287
|
+
"wait for task handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1288
|
+
);
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
this.waitForBatchReplay.attempt++;
|
|
1271
1292
|
}
|
|
1272
|
-
this.waitForBatchReplay.attempt++;
|
|
1273
1293
|
}
|
|
1274
|
-
}
|
|
1294
|
+
};
|
|
1275
1295
|
}
|
|
1276
1296
|
// MARK: WORKER CREATION
|
|
1277
1297
|
#createBackgroundWorker() {
|
|
1298
|
+
const workerId = randomUUID();
|
|
1299
|
+
logger.log("Creating background worker", { workerId });
|
|
1278
1300
|
const backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
|
1279
1301
|
projectConfig: __PROJECT_CONFIG__,
|
|
1280
1302
|
env: {
|
|
@@ -1286,18 +1308,27 @@ var ProdWorker = class {
|
|
|
1286
1308
|
contentHash: this.contentHash
|
|
1287
1309
|
});
|
|
1288
1310
|
backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
|
|
1289
|
-
logger.log("onTaskHeartbeat", {
|
|
1311
|
+
logger.log("onTaskHeartbeat", {
|
|
1312
|
+
workerId,
|
|
1313
|
+
attemptFriendlyId
|
|
1314
|
+
});
|
|
1290
1315
|
this.#coordinatorSocket.socket.volatile.emit("TASK_HEARTBEAT", {
|
|
1291
1316
|
version: "v1",
|
|
1292
1317
|
attemptFriendlyId
|
|
1293
1318
|
});
|
|
1294
1319
|
});
|
|
1295
1320
|
backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
|
|
1296
|
-
logger.log("onTaskRunHeartbeat", {
|
|
1321
|
+
logger.log("onTaskRunHeartbeat", {
|
|
1322
|
+
workerId,
|
|
1323
|
+
runId
|
|
1324
|
+
});
|
|
1297
1325
|
this.#coordinatorSocket.socket.volatile.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
|
|
1298
1326
|
});
|
|
1299
1327
|
backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
|
|
1300
|
-
logger.log("onCreateTaskRunAttempt()", {
|
|
1328
|
+
logger.log("onCreateTaskRunAttempt()", {
|
|
1329
|
+
workerId,
|
|
1330
|
+
message
|
|
1331
|
+
});
|
|
1301
1332
|
const createAttempt = await defaultBackoff.execute(async ({ retry }) => {
|
|
1302
1333
|
logger.log("Create task run attempt with backoff", { retry });
|
|
1303
1334
|
return await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CREATE_TASK_RUN_ATTEMPT", {
|
|
@@ -1326,6 +1357,7 @@ var ProdWorker = class {
|
|
|
1326
1357
|
});
|
|
1327
1358
|
backgroundWorker.attemptCreatedNotification.attach((message) => {
|
|
1328
1359
|
logger.log("attemptCreatedNotification", {
|
|
1360
|
+
workerId,
|
|
1329
1361
|
success: message.success,
|
|
1330
1362
|
...message.success ? {
|
|
1331
1363
|
attempt: message.execution.attempt,
|
|
@@ -1340,9 +1372,20 @@ var ProdWorker = class {
|
|
|
1340
1372
|
return;
|
|
1341
1373
|
}
|
|
1342
1374
|
this.attemptFriendlyId = message.execution.attempt.id;
|
|
1375
|
+
this.attemptNumber = message.execution.attempt.number;
|
|
1343
1376
|
});
|
|
1344
1377
|
backgroundWorker.onWaitForDuration.attach(async (message) => {
|
|
1345
|
-
logger.log("onWaitForDuration", {
|
|
1378
|
+
logger.log("onWaitForDuration", {
|
|
1379
|
+
workerId,
|
|
1380
|
+
...message,
|
|
1381
|
+
drift: Date.now() - message.now
|
|
1382
|
+
});
|
|
1383
|
+
if (this.nextResumeAfter) {
|
|
1384
|
+
logger.error("Already waiting for resume, skipping wait for duration", {
|
|
1385
|
+
nextResumeAfter: this.nextResumeAfter
|
|
1386
|
+
});
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1346
1389
|
noResume: {
|
|
1347
1390
|
const { ms, waitThresholdInMs } = message;
|
|
1348
1391
|
const internalTimeout = unboundedTimeout(ms, "internal");
|
|
@@ -1380,6 +1423,15 @@ var ProdWorker = class {
|
|
|
1380
1423
|
}
|
|
1381
1424
|
await this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
|
|
1382
1425
|
await Promise.race([internalTimeout, checkpointSafeInternalTimeout]);
|
|
1426
|
+
const idempotencyKey = randomUUID();
|
|
1427
|
+
this.durationResumeFallback = { idempotencyKey };
|
|
1428
|
+
try {
|
|
1429
|
+
await this.restoreNotification.waitFor(5e3);
|
|
1430
|
+
} catch (error) {
|
|
1431
|
+
logger.error("Did not receive restore notification in time", {
|
|
1432
|
+
error
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1383
1435
|
try {
|
|
1384
1436
|
const { checkpointCanceled } = await this.#coordinatorSocket.socket.timeout(15e3).emitWithAck("CANCEL_CHECKPOINT", {
|
|
1385
1437
|
version: "v2",
|
|
@@ -1390,8 +1442,6 @@ var ProdWorker = class {
|
|
|
1390
1442
|
break noResume;
|
|
1391
1443
|
}
|
|
1392
1444
|
logger.log("Waiting for external duration resume as we may have been restored");
|
|
1393
|
-
const idempotencyKey = randomUUID();
|
|
1394
|
-
this.durationResumeFallback = { idempotencyKey };
|
|
1395
1445
|
setTimeout(() => {
|
|
1396
1446
|
if (!this.durationResumeFallback) {
|
|
1397
1447
|
logger.error("Already resumed after duration, skipping fallback");
|
|
@@ -1405,19 +1455,32 @@ var ProdWorker = class {
|
|
|
1405
1455
|
this.#resumeAfterDuration();
|
|
1406
1456
|
}, 15e3);
|
|
1407
1457
|
} catch (error) {
|
|
1408
|
-
logger.debug("Checkpoint cancellation timed out", {
|
|
1409
|
-
|
|
1458
|
+
logger.debug("Checkpoint cancellation timed out", {
|
|
1459
|
+
workerId,
|
|
1460
|
+
message,
|
|
1461
|
+
error
|
|
1462
|
+
});
|
|
1410
1463
|
}
|
|
1411
1464
|
return;
|
|
1412
1465
|
}
|
|
1413
1466
|
this.#resumeAfterDuration();
|
|
1414
1467
|
});
|
|
1415
|
-
backgroundWorker.onWaitForTask.attach(this.#
|
|
1416
|
-
backgroundWorker.onWaitForBatch.attach(this.#
|
|
1468
|
+
backgroundWorker.onWaitForTask.attach(this.#waitForTaskHandlerFactory(workerId).bind(this));
|
|
1469
|
+
backgroundWorker.onWaitForBatch.attach(this.#waitForBatchHandlerFactory(workerId).bind(this));
|
|
1417
1470
|
return backgroundWorker;
|
|
1418
1471
|
}
|
|
1419
1472
|
async #prepareForWait(reason, willCheckpointAndRestore) {
|
|
1420
1473
|
logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
|
|
1474
|
+
if (this.nextResumeAfter) {
|
|
1475
|
+
logger.error("Already waiting for resume, skipping prepare for wait", {
|
|
1476
|
+
nextResumeAfter: this.nextResumeAfter,
|
|
1477
|
+
params: {
|
|
1478
|
+
reason,
|
|
1479
|
+
willCheckpointAndRestore
|
|
1480
|
+
}
|
|
1481
|
+
});
|
|
1482
|
+
return;
|
|
1483
|
+
}
|
|
1421
1484
|
if (!willCheckpointAndRestore) {
|
|
1422
1485
|
return;
|
|
1423
1486
|
}
|
|
@@ -1427,12 +1490,9 @@ var ProdWorker = class {
|
|
|
1427
1490
|
await this.#prepareForCheckpoint();
|
|
1428
1491
|
}
|
|
1429
1492
|
// MARK: RETRY PREP
|
|
1430
|
-
async #prepareForRetry(
|
|
1431
|
-
logger.log("prepare for retry", {
|
|
1493
|
+
async #prepareForRetry(shouldExit, exitCode) {
|
|
1494
|
+
logger.log("prepare for retry", { shouldExit, exitCode });
|
|
1432
1495
|
if (shouldExit) {
|
|
1433
|
-
if (willCheckpointAndRestore) {
|
|
1434
|
-
logger.error("WARNING: Will checkpoint but also requested exit. This won't end well.");
|
|
1435
|
-
}
|
|
1436
1496
|
await this.#exitGracefully(false, exitCode);
|
|
1437
1497
|
return;
|
|
1438
1498
|
}
|
|
@@ -1440,11 +1500,7 @@ var ProdWorker = class {
|
|
|
1440
1500
|
this.waitForPostStart = false;
|
|
1441
1501
|
this.executing = false;
|
|
1442
1502
|
this.attemptFriendlyId = void 0;
|
|
1443
|
-
|
|
1444
|
-
return;
|
|
1445
|
-
}
|
|
1446
|
-
this.waitForPostStart = true;
|
|
1447
|
-
await this.#prepareForCheckpoint(false);
|
|
1503
|
+
this.attemptNumber = void 0;
|
|
1448
1504
|
}
|
|
1449
1505
|
// MARK: CHECKPOINT PREP
|
|
1450
1506
|
async #prepareForCheckpoint(flush = true) {
|
|
@@ -1472,6 +1528,7 @@ var ProdWorker = class {
|
|
|
1472
1528
|
this.paused = false;
|
|
1473
1529
|
this.nextResumeAfter = void 0;
|
|
1474
1530
|
this.waitForPostStart = false;
|
|
1531
|
+
this.durationResumeFallback = void 0;
|
|
1475
1532
|
this.#backgroundWorker.waitCompletedNotification();
|
|
1476
1533
|
}
|
|
1477
1534
|
async #readyForLazyAttempt() {
|
|
@@ -1536,7 +1593,7 @@ var ProdWorker = class {
|
|
|
1536
1593
|
const taskRunCompleted = await defaultBackoff.execute(async ({ retry }) => {
|
|
1537
1594
|
logger.log("Submit attempt completion with backoff", { retry });
|
|
1538
1595
|
return await this.#coordinatorSocket.socket.timeout(2e4).emitWithAck("TASK_RUN_COMPLETED", {
|
|
1539
|
-
version: "
|
|
1596
|
+
version: "v2",
|
|
1540
1597
|
execution,
|
|
1541
1598
|
completion
|
|
1542
1599
|
});
|
|
@@ -1552,26 +1609,9 @@ var ProdWorker = class {
|
|
|
1552
1609
|
const { willCheckpointAndRestore, shouldExit } = taskRunCompleted.result;
|
|
1553
1610
|
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1554
1611
|
const exitCode = !completion.ok && completion.error.type === "INTERNAL_ERROR" && completion.error.code === TaskRunErrorCodes2.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE ? EXIT_CODE_CHILD_NONZERO : 0;
|
|
1555
|
-
await this.#prepareForRetry(
|
|
1612
|
+
await this.#prepareForRetry(shouldExit, exitCode);
|
|
1556
1613
|
if (willCheckpointAndRestore) {
|
|
1557
|
-
|
|
1558
|
-
this.submitAttemptCompletionReplay = {
|
|
1559
|
-
message: {
|
|
1560
|
-
execution,
|
|
1561
|
-
completion
|
|
1562
|
-
},
|
|
1563
|
-
attempt: 1,
|
|
1564
|
-
idempotencyKey: randomUUID()
|
|
1565
|
-
};
|
|
1566
|
-
} else {
|
|
1567
|
-
if (replayIdempotencyKey && replayIdempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
|
|
1568
|
-
logger.error(
|
|
1569
|
-
"attempt completion handler called with mismatched idempotency key, won't overwrite replay request"
|
|
1570
|
-
);
|
|
1571
|
-
return;
|
|
1572
|
-
}
|
|
1573
|
-
this.submitAttemptCompletionReplay.attempt++;
|
|
1574
|
-
}
|
|
1614
|
+
logger.error("This worker should never be checkpointed between attempts. This is a bug.");
|
|
1575
1615
|
}
|
|
1576
1616
|
}
|
|
1577
1617
|
#returnValidatedExtraHeaders(headers) {
|
|
@@ -1597,6 +1637,9 @@ var ProdWorker = class {
|
|
|
1597
1637
|
if (this.attemptFriendlyId) {
|
|
1598
1638
|
extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
|
|
1599
1639
|
}
|
|
1640
|
+
if (this.attemptNumber !== void 0) {
|
|
1641
|
+
extraHeaders["x-trigger-attempt-number"] = String(this.attemptNumber);
|
|
1642
|
+
}
|
|
1600
1643
|
logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
|
|
1601
1644
|
logger.debug(`connecting with extra headers`, { extraHeaders });
|
|
1602
1645
|
const coordinatorConnection = new ZodSocketConnection({
|
|
@@ -1668,31 +1711,13 @@ var ProdWorker = class {
|
|
|
1668
1711
|
});
|
|
1669
1712
|
return;
|
|
1670
1713
|
}
|
|
1671
|
-
this.durationResumeFallback = void 0;
|
|
1672
1714
|
this.#resumeAfterDuration();
|
|
1673
1715
|
},
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
}
|
|
1680
|
-
if (this.completed.has(executionPayload.execution.attempt.id)) {
|
|
1681
|
-
logger.error("dropping execute request, already completed");
|
|
1682
|
-
return;
|
|
1683
|
-
}
|
|
1684
|
-
this.executing = true;
|
|
1685
|
-
this.attemptFriendlyId = executionPayload.execution.attempt.id;
|
|
1686
|
-
const completion = await this.#backgroundWorker.executeTaskRun(executionPayload);
|
|
1687
|
-
logger.log("completed", completion);
|
|
1688
|
-
this.completed.add(executionPayload.execution.attempt.id);
|
|
1689
|
-
const { willCheckpointAndRestore, shouldExit } = await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
|
1690
|
-
version: "v1",
|
|
1691
|
-
execution: executionPayload.execution,
|
|
1692
|
-
completion
|
|
1693
|
-
});
|
|
1694
|
-
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
|
1695
|
-
await this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
|
1716
|
+
EXECUTE_TASK_RUN: async () => {
|
|
1717
|
+
this.#failRun(
|
|
1718
|
+
this.runId,
|
|
1719
|
+
"Received deprecated EXECUTE_TASK_RUN message. Please contact us if you see this error."
|
|
1720
|
+
);
|
|
1696
1721
|
},
|
|
1697
1722
|
EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
|
|
1698
1723
|
this.readyForLazyAttemptReplay = void 0;
|
|
@@ -1740,7 +1765,6 @@ var ProdWorker = class {
|
|
|
1740
1765
|
logger.error("Received READY_FOR_RETRY but no completions yet. This is a bug.");
|
|
1741
1766
|
return;
|
|
1742
1767
|
}
|
|
1743
|
-
this.submitAttemptCompletionReplay = void 0;
|
|
1744
1768
|
await this.#readyForLazyAttempt();
|
|
1745
1769
|
}
|
|
1746
1770
|
},
|
|
@@ -1750,7 +1774,11 @@ var ProdWorker = class {
|
|
|
1750
1774
|
status: this.#status,
|
|
1751
1775
|
connectionCount: ++this.connectionCount
|
|
1752
1776
|
});
|
|
1753
|
-
socket.emit("SET_STATE", {
|
|
1777
|
+
socket.emit("SET_STATE", {
|
|
1778
|
+
version: "v1",
|
|
1779
|
+
attemptFriendlyId: this.attemptFriendlyId,
|
|
1780
|
+
attemptNumber: this.attemptNumber ? String(this.attemptNumber) : void 0
|
|
1781
|
+
});
|
|
1754
1782
|
try {
|
|
1755
1783
|
if (this.waitForPostStart) {
|
|
1756
1784
|
logger2.log("skip connection handler, waiting for post start hook");
|
|
@@ -1766,16 +1794,25 @@ var ProdWorker = class {
|
|
|
1766
1794
|
return;
|
|
1767
1795
|
}
|
|
1768
1796
|
if (!this.attemptFriendlyId) {
|
|
1769
|
-
logger2.error("Missing friendly ID", { status: this.#status });
|
|
1797
|
+
logger2.error("Missing attempt friendly ID", { status: this.#status });
|
|
1770
1798
|
this.#emitUnrecoverableError(
|
|
1771
1799
|
"NoAttemptId",
|
|
1772
1800
|
"Attempt ID not set while resuming from paused state"
|
|
1773
1801
|
);
|
|
1774
1802
|
return;
|
|
1775
1803
|
}
|
|
1804
|
+
if (!this.attemptNumber) {
|
|
1805
|
+
logger2.error("Missing attempt number", { status: this.#status });
|
|
1806
|
+
this.#emitUnrecoverableError(
|
|
1807
|
+
"NoAttemptNumber",
|
|
1808
|
+
"Attempt number not set while resuming from paused state"
|
|
1809
|
+
);
|
|
1810
|
+
return;
|
|
1811
|
+
}
|
|
1776
1812
|
socket.emit("READY_FOR_RESUME", {
|
|
1777
|
-
version: "
|
|
1813
|
+
version: "v2",
|
|
1778
1814
|
attemptFriendlyId: this.attemptFriendlyId,
|
|
1815
|
+
attemptNumber: this.attemptNumber,
|
|
1779
1816
|
type: this.nextResumeAfter
|
|
1780
1817
|
});
|
|
1781
1818
|
return;
|
|
@@ -1912,7 +1949,7 @@ var ProdWorker = class {
|
|
|
1912
1949
|
}
|
|
1913
1950
|
try {
|
|
1914
1951
|
await backoff.wait(attempt + 1);
|
|
1915
|
-
await this.#
|
|
1952
|
+
await this.#waitForTaskHandlerFactory("replay")(message, idempotencyKey);
|
|
1916
1953
|
} catch (error) {
|
|
1917
1954
|
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1918
1955
|
logger.error("wait for task replay retry limit exceeded", { error });
|
|
@@ -1944,7 +1981,7 @@ var ProdWorker = class {
|
|
|
1944
1981
|
}
|
|
1945
1982
|
try {
|
|
1946
1983
|
await backoff.wait(attempt + 1);
|
|
1947
|
-
await this.#
|
|
1984
|
+
await this.#waitForBatchHandlerFactory("replay")(message, idempotencyKey);
|
|
1948
1985
|
} catch (error) {
|
|
1949
1986
|
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1950
1987
|
logger.error("wait for batch replay retry limit exceeded", { error });
|
|
@@ -1954,38 +1991,6 @@ var ProdWorker = class {
|
|
|
1954
1991
|
}
|
|
1955
1992
|
return;
|
|
1956
1993
|
}
|
|
1957
|
-
if (this.submitAttemptCompletionReplay) {
|
|
1958
|
-
logger.log("replaying attempt completion", {
|
|
1959
|
-
...this.submitAttemptCompletionReplay,
|
|
1960
|
-
cancellationDelay: replayCancellationDelay
|
|
1961
|
-
});
|
|
1962
|
-
const { idempotencyKey, message, attempt } = this.submitAttemptCompletionReplay;
|
|
1963
|
-
await timeout2(replayCancellationDelay);
|
|
1964
|
-
if (!this.submitAttemptCompletionReplay) {
|
|
1965
|
-
logger.error("attempt completion replay cancelled, discarding", {
|
|
1966
|
-
originalMessage: { idempotencyKey, message, attempt }
|
|
1967
|
-
});
|
|
1968
|
-
return;
|
|
1969
|
-
}
|
|
1970
|
-
if (idempotencyKey !== this.submitAttemptCompletionReplay.idempotencyKey) {
|
|
1971
|
-
logger.error("attempt completion replay idempotency key mismatch, discarding", {
|
|
1972
|
-
originalMessage: { idempotencyKey, message, attempt },
|
|
1973
|
-
newMessage: this.submitAttemptCompletionReplay
|
|
1974
|
-
});
|
|
1975
|
-
return;
|
|
1976
|
-
}
|
|
1977
|
-
try {
|
|
1978
|
-
await backoff.wait(attempt + 1);
|
|
1979
|
-
await this.#submitAttemptCompletion(message.execution, message.completion, idempotencyKey);
|
|
1980
|
-
} catch (error) {
|
|
1981
|
-
if (error instanceof ExponentialBackoff.RetryLimitExceeded) {
|
|
1982
|
-
logger.error("attempt completion replay retry limit exceeded", { error });
|
|
1983
|
-
} else {
|
|
1984
|
-
logger.error("attempt completion replay error", { error });
|
|
1985
|
-
}
|
|
1986
|
-
}
|
|
1987
|
-
return;
|
|
1988
|
-
}
|
|
1989
1994
|
}
|
|
1990
1995
|
// MARK: HTTP SERVER
|
|
1991
1996
|
#createHttpServer() {
|
|
@@ -2048,6 +2053,7 @@ var ProdWorker = class {
|
|
|
2048
2053
|
}
|
|
2049
2054
|
case "restore": {
|
|
2050
2055
|
await this.#reconnectAfterPostStart();
|
|
2056
|
+
this.restoreNotification.post();
|
|
2051
2057
|
break;
|
|
2052
2058
|
}
|
|
2053
2059
|
default: {
|
|
@@ -2128,6 +2134,7 @@ var ProdWorker = class {
|
|
|
2128
2134
|
nextResumeAfter: this.nextResumeAfter,
|
|
2129
2135
|
waitForPostStart: this.waitForPostStart,
|
|
2130
2136
|
attemptFriendlyId: this.attemptFriendlyId,
|
|
2137
|
+
attemptNumber: this.attemptNumber,
|
|
2131
2138
|
waitForTaskReplay: this.waitForTaskReplay,
|
|
2132
2139
|
waitForBatchReplay: this.waitForBatchReplay
|
|
2133
2140
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trigger.dev",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.56",
|
|
4
4
|
"description": "A Command-Line Interface for Trigger.dev (v3) projects",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"vitest": "^1.6.0",
|
|
55
55
|
"xdg-app-paths": "^8.3.0",
|
|
56
56
|
"@trigger.dev/tsconfig": "0.0.0",
|
|
57
|
-
"@trigger.dev/core-apps": "3.0.0-beta.
|
|
57
|
+
"@trigger.dev/core-apps": "3.0.0-beta.56"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
60
|
"@anatine/esbuild-decorators": "^0.2.19",
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
"@opentelemetry/sdk-trace-base": "^1.22.0",
|
|
73
73
|
"@opentelemetry/sdk-trace-node": "^1.22.0",
|
|
74
74
|
"@opentelemetry/semantic-conventions": "^1.22.0",
|
|
75
|
-
"@trigger.dev/core": "3.0.0-beta.
|
|
75
|
+
"@trigger.dev/core": "3.0.0-beta.56",
|
|
76
76
|
"@types/degit": "^2.8.3",
|
|
77
77
|
"chalk": "^5.2.0",
|
|
78
78
|
"chokidar": "^3.5.3",
|