witnora 0.20.10 → 0.20.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +7 -3
- package/dist/gateway-service.js +11 -3
- package/dist/gateway.js +101 -14
- package/dist/internal/control-client/durable-action-worker.d.ts.map +1 -1
- package/dist/internal/control-client/durable-action-worker.js +17 -1
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts +2 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts.map +1 -1
- package/dist/vendor/onegent-runtime/failure-runtime-watch.js +79 -62
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -40,8 +40,8 @@ import { runOnboard } from "./onboard.js";
|
|
|
40
40
|
import { inspectRepository } from "./onboard.js";
|
|
41
41
|
import { renderReleaseEvaluation, runReleaseEvaluation } from "./release-evaluation.js";
|
|
42
42
|
import { runPrivateCapabilityDiscovery } from "./private-discovery.js";
|
|
43
|
-
import { configureManagedWorkflowHarness, doctorCustomerGateway, initializeCustomerGateway, isGatewayDoctorReady, readManagedCustomerGatewayLogs, renderGatewayDoctor, renderManagedGatewayStatus,
|
|
44
|
-
import { installCurrentGatewayService, stopCurrentGatewayOwners, uninstallCurrentGatewayService } from "./gateway-service.js";
|
|
43
|
+
import { awaitManagedCustomerGateway, configureManagedWorkflowHarness, doctorCustomerGateway, initializeCustomerGateway, isGatewayDoctorReady, readManagedCustomerGatewayLogs, renderGatewayDoctor, renderManagedGatewayStatus, runCustomerGateway, startManagedCustomerGateway, statusManagedCustomerGateway, stopManagedCustomerGateway, superviseManagedCustomerGateway, } from "./gateway.js";
|
|
44
|
+
import { installCurrentGatewayService, restartCurrentGatewayOwners, stopCurrentGatewayOwners, uninstallCurrentGatewayService } from "./gateway-service.js";
|
|
45
45
|
import { runCustomerGatewayCommand } from "./gateway-exec.js";
|
|
46
46
|
import { verifyEvidencePacketV02 } from "./evidence-v02.js";
|
|
47
47
|
process.on("uncaughtException", reportFatalError);
|
|
@@ -234,7 +234,11 @@ else if (command === "gateway") {
|
|
|
234
234
|
process.stdout.write(`${await readManagedCustomerGatewayLogs({ repository: readFlag("--repo") ?? process.cwd(), dir: readFlag("--dir"), lines })}\n`);
|
|
235
235
|
}
|
|
236
236
|
else if (action === "restart") {
|
|
237
|
-
const
|
|
237
|
+
const repository = readFlag("--repo") ?? process.cwd();
|
|
238
|
+
const dir = readFlag("--dir");
|
|
239
|
+
const configHome = readFlag("--config-home");
|
|
240
|
+
const serviceOptions = { repository, gatewayDirectory: dir, cliEntry: fileURLToPath(import.meta.url), configHome };
|
|
241
|
+
const result = await restartCurrentGatewayOwners(serviceOptions, () => stopManagedCustomerGateway({ repository, dir, configHome }), () => awaitManagedCustomerGateway({ repository, dir, configHome }));
|
|
238
242
|
process.stdout.write(readBoolFlag("--json") ? `${JSON.stringify(result, null, 2)}\n` : renderManagedGatewayStatus(result));
|
|
239
243
|
}
|
|
240
244
|
else if (action === "repair") {
|
package/dist/gateway-service.js
CHANGED
|
@@ -12,9 +12,11 @@ export async function installGatewayService(input) {
|
|
|
12
12
|
// before replacing the definition so onboarding never leaves two owners
|
|
13
13
|
// racing for the same localhost port.
|
|
14
14
|
if (plan.kind === "WINDOWS_TASK") {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
if (!input.existingOwnerStopped) {
|
|
16
|
+
await input.run(plan.stop.command, plan.stop.args).catch(() => undefined);
|
|
17
|
+
await (input.removeFile ?? removeFile)(plan.serviceLease.path);
|
|
18
|
+
await (input.sleep ?? wait)(WINDOWS_PARENT_EXIT_GRACE_MS);
|
|
19
|
+
}
|
|
18
20
|
await input.writeDefinition(plan.serviceLease.path, `${plan.serviceLease.generation}\n`);
|
|
19
21
|
}
|
|
20
22
|
if (plan.launcher)
|
|
@@ -40,6 +42,7 @@ export async function installCurrentGatewayService(options) {
|
|
|
40
42
|
run: options.run ?? runCommand,
|
|
41
43
|
sleep: options.sleep,
|
|
42
44
|
removeFile: options.removeFile,
|
|
45
|
+
existingOwnerStopped: options.existingOwnerStopped,
|
|
43
46
|
});
|
|
44
47
|
}
|
|
45
48
|
export async function uninstallCurrentGatewayService(options) {
|
|
@@ -69,6 +72,11 @@ export async function restartCurrentGatewayService(options) {
|
|
|
69
72
|
await run(plan.start.command, plan.start.args);
|
|
70
73
|
return plan;
|
|
71
74
|
}
|
|
75
|
+
export async function restartCurrentGatewayOwners(options, stopManagedGateway, awaitManagedGateway) {
|
|
76
|
+
await stopCurrentGatewayOwners(options, stopManagedGateway);
|
|
77
|
+
await restartCurrentGatewayService({ ...options, existingOwnerStopped: true });
|
|
78
|
+
return awaitManagedGateway();
|
|
79
|
+
}
|
|
72
80
|
export async function stopCurrentGatewayService(options) {
|
|
73
81
|
const plan = createGatewayServicePlan(currentPlanInput(options));
|
|
74
82
|
await (options.run ?? runCommand)(plan.stop.command, plan.stop.args).catch(() => undefined);
|
package/dist/gateway.js
CHANGED
|
@@ -574,6 +574,7 @@ export async function runCustomerGateway(options) {
|
|
|
574
574
|
directory, config, connection, configHome: options.configHome,
|
|
575
575
|
DurableApprovedActionWorker: (await durableWorker).DurableApprovedActionWorker,
|
|
576
576
|
FileActionCheckpointStore: (await durableWorker).FileActionCheckpointStore,
|
|
577
|
+
onReceipt: () => assuranceController.wakeRuntimeWatch(),
|
|
577
578
|
})
|
|
578
579
|
: undefined;
|
|
579
580
|
const probeCredentialReady = createBoundedRuntimeProbeReadiness({
|
|
@@ -738,6 +739,10 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
738
739
|
: undefined;
|
|
739
740
|
const basePollIntervalMs = input.config.pollIntervalMs ?? 5_000;
|
|
740
741
|
const maxIdlePollIntervalMs = Math.max(basePollIntervalMs, 15 * 60_000);
|
|
742
|
+
// Receipt events wake the read-only Watch; this modest periodic check covers
|
|
743
|
+
// restart recovery, new cases, and results produced by another local worker.
|
|
744
|
+
const watchPollIntervalMs = 60_000;
|
|
745
|
+
let nextWatchAt = Date.now() + watchPollIntervalMs;
|
|
741
746
|
const runtimeWatch = runtimeWatchConfiguration
|
|
742
747
|
&& input.managed.ManagedFailureRuntimeWatch
|
|
743
748
|
&& input.managed.FileFailureRuntimeWatchCheckpointStore
|
|
@@ -751,7 +756,8 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
751
756
|
fetch: requestFetch,
|
|
752
757
|
}),
|
|
753
758
|
checkpoints: new input.managed.FileFailureRuntimeWatchCheckpointStore(join(input.directory, "data", "failure-runtime-watch")),
|
|
754
|
-
intervalMs:
|
|
759
|
+
intervalMs: watchPollIntervalMs,
|
|
760
|
+
nextCheckAt: () => new Date(nextWatchAt).toISOString(),
|
|
755
761
|
...(activation ? { bindings: [{ taskContractId: activation.taskContractId, actionPathId: activation.actionPathIds[0], environment: "sandbox" }] } : {}),
|
|
756
762
|
readOutcome: async ({ resourceId, actions, receipts }) => input.managed.readLocalSandboxRuntimeWatchOutcome({
|
|
757
763
|
path: join(input.directory, "data", "runtime-sandbox", "fixture", "audit.jsonl"),
|
|
@@ -764,19 +770,66 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
764
770
|
})
|
|
765
771
|
: undefined;
|
|
766
772
|
let timer;
|
|
773
|
+
let watchTimer;
|
|
774
|
+
let wakeTimer;
|
|
775
|
+
let watchInFlight;
|
|
776
|
+
let watchPending = false;
|
|
767
777
|
let closing = false;
|
|
768
778
|
let idleTicks = 0;
|
|
769
779
|
let lastTickAt;
|
|
770
780
|
let lastError;
|
|
771
781
|
let lastProviderHealthAt = 0;
|
|
772
|
-
const
|
|
782
|
+
const runWatch = () => {
|
|
783
|
+
if (!runtimeWatch || closing)
|
|
784
|
+
return Promise.resolve(undefined);
|
|
785
|
+
if (watchInFlight)
|
|
786
|
+
return watchInFlight;
|
|
787
|
+
watchInFlight = runtimeWatch.tick().finally(() => {
|
|
788
|
+
watchInFlight = undefined;
|
|
789
|
+
if (watchPending)
|
|
790
|
+
wakeRuntimeWatch();
|
|
791
|
+
});
|
|
792
|
+
return watchInFlight;
|
|
793
|
+
};
|
|
794
|
+
const wakeRuntimeWatch = () => {
|
|
795
|
+
if (!runtimeWatch || closing)
|
|
796
|
+
return;
|
|
797
|
+
watchPending = true;
|
|
798
|
+
if (wakeTimer || watchInFlight)
|
|
799
|
+
return;
|
|
800
|
+
wakeTimer = setTimeout(() => {
|
|
801
|
+
wakeTimer = undefined;
|
|
802
|
+
// A Receipt arriving during a scan requires one following scan: the
|
|
803
|
+
// in-flight snapshot may already have read its Action/Receipt list.
|
|
804
|
+
if (watchInFlight)
|
|
805
|
+
return;
|
|
806
|
+
watchPending = false;
|
|
807
|
+
void runWatch().catch((error) => {
|
|
808
|
+
process.stderr.write(`Runtime Watch check failed: ${message(error)}\n`);
|
|
809
|
+
});
|
|
810
|
+
}, 100);
|
|
811
|
+
wakeTimer.unref?.();
|
|
812
|
+
};
|
|
813
|
+
const scheduleWatch = () => {
|
|
814
|
+
if (!runtimeWatch || closing)
|
|
815
|
+
return;
|
|
816
|
+
nextWatchAt = Date.now() + watchPollIntervalMs;
|
|
817
|
+
watchTimer = setTimeout(() => {
|
|
818
|
+
scheduleWatch();
|
|
819
|
+
wakeRuntimeWatch();
|
|
820
|
+
}, watchPollIntervalMs);
|
|
821
|
+
watchTimer.unref?.();
|
|
822
|
+
};
|
|
823
|
+
const tick = async (includeWatch = true) => {
|
|
773
824
|
try {
|
|
774
825
|
if (managedEvaluator?.healthCheck && Date.now() - lastProviderHealthAt >= 30_000) {
|
|
775
826
|
lastProviderHealthAt = Date.now();
|
|
776
827
|
await managedEvaluator.healthCheck();
|
|
777
828
|
}
|
|
778
|
-
const result = await
|
|
779
|
-
|
|
829
|
+
const [result, watchResult] = await Promise.all([
|
|
830
|
+
worker.tick(),
|
|
831
|
+
includeWatch ? runWatch() : Promise.resolve(undefined),
|
|
832
|
+
]);
|
|
780
833
|
lastTickAt = new Date().toISOString();
|
|
781
834
|
lastError = (result.evaluationsFailed ?? 0) > 0 ? result.limitations?.[0] ?? "One or more evaluations failed closed." : undefined;
|
|
782
835
|
if ((result.evaluationsFailed ?? 0) > 0) {
|
|
@@ -800,6 +853,8 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
800
853
|
start() {
|
|
801
854
|
if (timer || closing)
|
|
802
855
|
return;
|
|
856
|
+
scheduleWatch();
|
|
857
|
+
wakeRuntimeWatch();
|
|
803
858
|
const schedule = (delayMs) => {
|
|
804
859
|
if (closing)
|
|
805
860
|
return;
|
|
@@ -809,7 +864,7 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
809
864
|
const run = async () => {
|
|
810
865
|
let didWork = false;
|
|
811
866
|
try {
|
|
812
|
-
const result = await tick();
|
|
867
|
+
const result = await tick(false);
|
|
813
868
|
didWork = workflowHarnessTickDidWork(result);
|
|
814
869
|
}
|
|
815
870
|
catch (error) {
|
|
@@ -823,11 +878,19 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
823
878
|
};
|
|
824
879
|
schedule(0);
|
|
825
880
|
},
|
|
881
|
+
wakeRuntimeWatch,
|
|
826
882
|
async close() {
|
|
827
883
|
closing = true;
|
|
828
884
|
if (timer)
|
|
829
885
|
clearTimeout(timer);
|
|
886
|
+
if (watchTimer)
|
|
887
|
+
clearTimeout(watchTimer);
|
|
888
|
+
if (wakeTimer)
|
|
889
|
+
clearTimeout(wakeTimer);
|
|
830
890
|
timer = undefined;
|
|
891
|
+
watchTimer = undefined;
|
|
892
|
+
wakeTimer = undefined;
|
|
893
|
+
await watchInFlight?.catch(() => undefined);
|
|
831
894
|
await managedEvaluator?.close();
|
|
832
895
|
},
|
|
833
896
|
async status() {
|
|
@@ -934,6 +997,8 @@ export async function createContinuousAssuranceController(input) {
|
|
|
934
997
|
timer.unref?.();
|
|
935
998
|
},
|
|
936
999
|
reconcile,
|
|
1000
|
+
wakeRuntimeWatch: () => { if (!closing)
|
|
1001
|
+
harness?.wakeRuntimeWatch(); },
|
|
937
1002
|
status: () => harness?.status() ?? Promise.resolve(undefined),
|
|
938
1003
|
async close() {
|
|
939
1004
|
closing = true;
|
|
@@ -1191,10 +1256,25 @@ export async function createConfiguredRuntimeActionWorker(input) {
|
|
|
1191
1256
|
},
|
|
1192
1257
|
store: new input.FileActionCheckpointStore(resolve(input.directory, "data", "runtime-actions", "checkpoints")),
|
|
1193
1258
|
hosted: {
|
|
1194
|
-
getAction: (actionId) =>
|
|
1259
|
+
getAction: async (actionId) => {
|
|
1260
|
+
const action = await primary(`actions/${encodeURIComponent(actionId)}`);
|
|
1261
|
+
if (action.status === "VERIFIED" && action.verificationSuccess === true && action.receiptId && Number(action.receiptSignatureCount) > 0)
|
|
1262
|
+
input.onReceipt?.();
|
|
1263
|
+
return action;
|
|
1264
|
+
},
|
|
1195
1265
|
issueExecutionGrant: (actionId, body, idempotencyKey) => primary(`actions/${encodeURIComponent(actionId)}/execution-grant`, { method: "POST", body, idempotencyKey }),
|
|
1196
|
-
verifyAction: (actionId, body, idempotencyKey) =>
|
|
1197
|
-
|
|
1266
|
+
verifyAction: async (actionId, body, idempotencyKey) => {
|
|
1267
|
+
const action = await verifier(`actions/${encodeURIComponent(actionId)}/verify`, { method: "POST", body, idempotencyKey });
|
|
1268
|
+
if (action.status === "VERIFIED" && action.verificationSuccess === true && action.receiptId && Number(action.receiptSignatureCount) > 0)
|
|
1269
|
+
input.onReceipt?.();
|
|
1270
|
+
return action;
|
|
1271
|
+
},
|
|
1272
|
+
listActionReceipts: async (actionId) => {
|
|
1273
|
+
const receipts = (await primary(`actions/${encodeURIComponent(actionId)}/receipts`)).receipts ?? [];
|
|
1274
|
+
if (receipts.some((receipt) => (receipt.receipt?.signatureSet?.length ?? 0) > 0))
|
|
1275
|
+
input.onReceipt?.();
|
|
1276
|
+
return receipts;
|
|
1277
|
+
},
|
|
1198
1278
|
claimExecutionGrant: (executionGrantId, claim, idempotencyKey) => claimHostedExecutionGrant(input.connection, requestFetch, executionGrantId, claim, idempotencyKey),
|
|
1199
1279
|
},
|
|
1200
1280
|
runtime: { reconcileReadOnly: true, prepareClaim: runtime.prepareClaim, execute: runtime.execute, reconcile: runtime.reconcile, submitEvidence: runtime.submitEvidence },
|
|
@@ -1424,12 +1504,19 @@ export async function stopManagedCustomerGateway(options = {}) {
|
|
|
1424
1504
|
const after = await statusManagedCustomerGateway(options);
|
|
1425
1505
|
return { ...after, stopped: !after.healthy };
|
|
1426
1506
|
}
|
|
1427
|
-
export async function
|
|
1428
|
-
const
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1507
|
+
export async function awaitManagedCustomerGateway(options = {}) {
|
|
1508
|
+
const sleep = options.sleep ?? wait;
|
|
1509
|
+
const deadline = Date.now() + (options.timeoutMs ?? 12_000);
|
|
1510
|
+
let status = await statusManagedCustomerGateway(options);
|
|
1511
|
+
while (Date.now() < deadline) {
|
|
1512
|
+
if (status.state === "RUNNING_MANAGED")
|
|
1513
|
+
return { ...status, started: true };
|
|
1514
|
+
if (status.state === "RUNNING_EXTERNAL" || status.state === "CONFLICT")
|
|
1515
|
+
throw new Error(status.detail);
|
|
1516
|
+
await sleep(120);
|
|
1517
|
+
status = await statusManagedCustomerGateway(options);
|
|
1518
|
+
}
|
|
1519
|
+
throw new Error(`The operating-system Gateway supervisor did not restore a healthy managed process before the startup deadline. ${status.detail}`);
|
|
1433
1520
|
}
|
|
1434
1521
|
export async function superviseManagedCustomerGateway(options = {}) {
|
|
1435
1522
|
const inspect = options.inspect ?? (() => statusManagedCustomerGateway({ repository: options.repository, dir: options.dir }));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"durable-action-worker.d.ts","sourceRoot":"","sources":["../../../../agentcert-sdk/src/durable-action-worker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"durable-action-worker.d.ts","sourceRoot":"","sources":["../../../../agentcert-sdk/src/durable-action-worker.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,cAAc,GACd,mBAAmB,GACnB,gBAAgB,GAChB,UAAU,GACV,QAAQ,GACR,oBAAoB,GACpB,WAAW,CAAC;AAEhB,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,eAAe,EAAE;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,wBAAwB,EAAE,MAAM,CAAC;QACjC,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,gBAAgB,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;QACxB,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,kBAAkB,GAAG,UAAU,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,kBAAkB,GAAG,eAAe,GAAG,UAAU,GAAG,MAAM,CAAC;IACzI,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,mBAAmB,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IACjG,gBAAgB,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE;YACrC,SAAS,EAAE,MAAM,CAAC;YAClB,wBAAwB,EAAE,MAAM,CAAC;YACjC,cAAc,EAAE,MAAM,EAAE,CAAC;YACzB,gBAAgB,EAAE,MAAM,CAAC;YACzB,eAAe,EAAE,MAAM,CAAC;YACxB,wBAAwB,EAAE,MAAM,CAAC;YACjC,sBAAsB,EAAE,MAAM,CAAC;YAC/B,YAAY,EAAE,MAAM,CAAC;YACrB,gBAAgB,EAAE,MAAM,CAAC;SAC1B,CAAA;KAAE,CAAC;IACJ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IACrE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IAC3C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,8BAA8B;IAC7C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,iBAAiB,EAAE,YAAY,GAAG,WAAW,GAAG,kBAAkB,GAAG,SAAS,GAAG,gBAAgB,GAAG,0BAA0B,CAAC;IAC/H,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,wCAAwC,CAAC;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,kBAAkB,CAAC;IAC1B,QAAQ,EAAE,qBAAqB,CAAC;IAChC,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,EAAE,4BAA4B,CAAC;IACzC,WAAW,CAAC,EAAE,wBAAwB,CAAC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,8BAA8B,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC,CAAC;IACrE,IAAI,CAAC,UAAU,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,IAAI,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IAC3C,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvH;AAED,qBAAa,yBAA0B,YAAW,qBAAqB;IACrE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,SAAS,EAAE,MAAM;IAEvB,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAAC;IAQpE,IAAI,CAAC,UAAU,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBxD,IAAI,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC;IAQ1C,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAmB3H,OAAO,CAAC,IAAI;IAIZ,OAAO,CAAC,SAAS;CAIlB;AAED,MAAM,WAAW,kCAAkC;IACjD,MAAM,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,KAAK,EAAE,qBAAqB,CAAC;IAC7B,MAAM,EAAE;QACN,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;QAC1D,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC3H,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAC;YAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACvM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE;gBAAE,YAAY,CAAC,EAAE,OAAO,EAAE,CAAA;aAAE,CAAA;SAAE,CAAC,CAAC,CAAC;QAC7G,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC;YAAE,QAAQ,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KACvI,CAAC;IACF,OAAO,EAAE;QACP,iBAAiB,EAAE,IAAI,CAAC;QACxB,YAAY,CAAC,KAAK,EAAE;YAAE,MAAM,EAAE,mBAAmB,CAAC;YAAC,QAAQ,EAAE,qBAAqB,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAA;SAAE,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;QAC1J,OAAO,CAAC,KAAK,EAAE;YAAE,MAAM,EAAE,mBAAmB,CAAC;YAAC,QAAQ,EAAE,qBAAqB,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAC;YAAC,iBAAiB,EAAE,8BAA8B,CAAA;SAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;QACtM,SAAS,CAAC,KAAK,EAAE;YAAE,MAAM,EAAE,mBAAmB,CAAC;YAAC,QAAQ,EAAE,qBAAqB,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAA;SAAE,GAAG,OAAO,CAAC,4BAA4B,GAAG,SAAS,CAAC,CAAC;QACjK,cAAc,CAAC,KAAK,EAAE;YAAE,MAAM,EAAE,mBAAmB,CAAC;YAAC,QAAQ,EAAE,qBAAqB,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAC;YAAC,iBAAiB,EAAE,8BAA8B,CAAC;YAAC,SAAS,EAAE,4BAA4B,CAAC;YAAC,WAAW,EAAE,wBAAwB,CAAA;SAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACtQ,CAAC;IACF,KAAK,EAAE;QACL,EAAE,EAAE,MAAM,CAAC;QACX,gBAAgB,EAAE,MAAM,CAAC;QACzB,QAAQ,EAAE,IAAI,CAAC;QACf,OAAO,CAAC,KAAK,EAAE;YAAE,MAAM,EAAE,mBAAmB,CAAC;YAAC,QAAQ,EAAE,qBAAqB,CAAC;YAAC,KAAK,EAAE,kBAAkB,CAAC;YAAC,SAAS,EAAE,4BAA4B,CAAA;SAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;KACzL,CAAC;IACF,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,2CAA2C,CAAC;IAC3D,KAAK,EAAE,OAAO,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,2BAA2B;IAQ1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuD;IAC9E,OAAO,CAAC,KAAK,CAAC,CAAiC;IAC/C,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgB;gBAEZ,OAAO,EAAE,kCAAkC;IAUlE,KAAK,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,qBAAqB,CAAA;KAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAgB3G,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAOxD,KAAK,IAAI,IAAI;IAcb,IAAI,IAAI,IAAI;IAEN,MAAM,IAAI,OAAO,CAAC,yBAAyB,CAAC;YAapC,QAAQ;YA6IR,OAAO;CAMtB;AA2DD,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAY3F"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { mkdir, open, readFile, readdir, rename } from "node:fs/promises";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
4
5
|
import { canonicalJson } from "./canonical.js";
|
|
5
6
|
export class FileActionCheckpointStore {
|
|
6
7
|
directory;
|
|
@@ -27,7 +28,22 @@ export class FileActionCheckpointStore {
|
|
|
27
28
|
finally {
|
|
28
29
|
await handle.close();
|
|
29
30
|
}
|
|
30
|
-
|
|
31
|
+
// Windows readers and scanners may briefly deny atomic replacement. Retry
|
|
32
|
+
// publication of this same synced file, never the action or its execution claim.
|
|
33
|
+
const retryDelaysMs = [10, 25, 50, 100, 200, 400];
|
|
34
|
+
for (let attempt = 0;; attempt++) {
|
|
35
|
+
try {
|
|
36
|
+
await rename(temporary, path);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
const code = error && typeof error === "object" && "code" in error ? error.code : undefined;
|
|
41
|
+
const retryDelay = retryDelaysMs[attempt];
|
|
42
|
+
if (retryDelay === undefined || (code !== "EPERM" && code !== "EACCES" && code !== "EBUSY"))
|
|
43
|
+
throw error;
|
|
44
|
+
await delay(retryDelay);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
31
47
|
}
|
|
32
48
|
async list() {
|
|
33
49
|
let names;
|
|
@@ -118,6 +118,7 @@ export declare class ManagedFailureRuntimeWatch {
|
|
|
118
118
|
checkpoints: FailureRuntimeWatchCheckpointStore;
|
|
119
119
|
intervalMs?: number;
|
|
120
120
|
now?: () => Date;
|
|
121
|
+
nextCheckAt?: () => string;
|
|
121
122
|
readOutcome: FailureRuntimeWatchOutcomeReader;
|
|
122
123
|
bindings?: Array<{
|
|
123
124
|
taskContractId: string;
|
|
@@ -126,6 +127,7 @@ export declare class ManagedFailureRuntimeWatch {
|
|
|
126
127
|
}>;
|
|
127
128
|
});
|
|
128
129
|
tick(): Promise<ManagedFailureRuntimeWatchResult>;
|
|
130
|
+
private nextCheckAt;
|
|
129
131
|
private runTick;
|
|
130
132
|
}
|
|
131
133
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"failure-runtime-watch.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-runtime-watch.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2B,KAAK,uBAAuB,EAAE,KAAK,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAE1I,MAAM,MAAM,yBAAyB,GACjC,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,GAAG,MAAM,CAAC;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,uBAAuB,CAAC;IACpC,SAAS,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,SAAS,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,YAAY,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE,CAAC;CACpG;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,mBAAmB,CAAC,EAAE;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE;YAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IACzG,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,0BAA0B;IACzC,cAAc,EAAE,SAAS,GAAG,YAAY,GAAG,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAC;IACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE;IACrD,WAAW,EAAE,uBAAuB,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,yBAAyB,EAAE,CAAC;IACrC,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACxC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,yBAAyB,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,+BAA+B;IAC9C,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACvD,WAAW,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAAC;IACpD,YAAY,CAAC,OAAO,CAAC,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IAC3F,iBAAiB,CAAC,KAAK,EAAE,4BAA4B,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClG,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9F;AAED,wBAAgB,qCAAqC,CAAC,OAAO,EAAE;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,GAAG,+BAA+B,CAoClC;AAED,MAAM,WAAW,6BAA6B;IAC5C,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E;AAED,qBAAa,wCAAyC,YAAW,kCAAkC;;IAG3F,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IAKrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;CAGlF;AAED,qBAAa,sCAAuC,YAAW,kCAAkC;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAIxC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IASrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IASjF,OAAO,CAAC,IAAI;CAIb;AAED,MAAM,WAAW,gCAAgC;IAC/C,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,0BAA0B;;
|
|
1
|
+
{"version":3,"file":"failure-runtime-watch.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-runtime-watch.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2B,KAAK,uBAAuB,EAAE,KAAK,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAE1I,MAAM,MAAM,yBAAyB,GACjC,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,GAAG,MAAM,CAAC;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,uBAAuB,CAAC;IACpC,SAAS,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,SAAS,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,YAAY,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE,CAAC;CACpG;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,mBAAmB,CAAC,EAAE;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE;YAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IACzG,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,0BAA0B;IACzC,cAAc,EAAE,SAAS,GAAG,YAAY,GAAG,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAC;IACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE;IACrD,WAAW,EAAE,uBAAuB,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,yBAAyB,EAAE,CAAC;IACrC,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACxC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,yBAAyB,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,+BAA+B;IAC9C,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACvD,WAAW,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAAC;IACpD,YAAY,CAAC,OAAO,CAAC,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IAC3F,iBAAiB,CAAC,KAAK,EAAE,4BAA4B,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClG,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9F;AAED,wBAAgB,qCAAqC,CAAC,OAAO,EAAE;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,GAAG,+BAA+B,CAoClC;AAED,MAAM,WAAW,6BAA6B;IAC5C,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E;AAED,qBAAa,wCAAyC,YAAW,kCAAkC;;IAG3F,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IAKrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;CAGlF;AAED,qBAAa,sCAAuC,YAAW,kCAAkC;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAIxC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IASrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IASjF,OAAO,CAAC,IAAI;CAIb;AAED,MAAM,WAAW,gCAAgC;IAC/C,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,0BAA0B;;gBAUzB,OAAO,EAAE;QACnB,MAAM,EAAE,+BAA+B,CAAC;QACxC,WAAW,EAAE,kCAAkC,CAAC;QAChD,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,MAAM,CAAC;QAC3B,WAAW,EAAE,gCAAgC,CAAC;QAC9C,QAAQ,CAAC,EAAE,KAAK,CAAC;YAAE,cAAc,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,SAAS,CAAA;SAAE,CAAC,CAAC;KAC5F;IAaD,IAAI,IAAI,OAAO,CAAC,gCAAgC,CAAC;IAMjD,OAAO,CAAC,WAAW;YAKL,OAAO;CAqJtB;AAID;;;GAGG;AACH,wBAAsB,mCAAmC,CAAC,KAAK,EAAE;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAiEtC"}
|
|
@@ -77,6 +77,7 @@ export class ManagedFailureRuntimeWatch {
|
|
|
77
77
|
#checkpoints;
|
|
78
78
|
#intervalMs;
|
|
79
79
|
#now;
|
|
80
|
+
#scheduledCheck;
|
|
80
81
|
#readOutcome;
|
|
81
82
|
#bindings;
|
|
82
83
|
#activeTick;
|
|
@@ -85,6 +86,7 @@ export class ManagedFailureRuntimeWatch {
|
|
|
85
86
|
this.#checkpoints = options.checkpoints;
|
|
86
87
|
this.#intervalMs = options.intervalMs ?? 5_000;
|
|
87
88
|
this.#now = options.now ?? (() => new Date());
|
|
89
|
+
this.#scheduledCheck = options.nextCheckAt;
|
|
88
90
|
this.#readOutcome = options.readOutcome;
|
|
89
91
|
this.#bindings = options.bindings ? new Set(options.bindings.map(bindingKey)) : undefined;
|
|
90
92
|
if (!Number.isSafeInteger(this.#intervalMs) || this.#intervalMs < 1_000 || this.#intervalMs > 15 * 60_000) {
|
|
@@ -97,6 +99,10 @@ export class ManagedFailureRuntimeWatch {
|
|
|
97
99
|
this.#activeTick = this.runTick().finally(() => { this.#activeTick = undefined; });
|
|
98
100
|
return this.#activeTick;
|
|
99
101
|
}
|
|
102
|
+
nextCheckAt(checkedAt) {
|
|
103
|
+
const scheduled = this.#scheduledCheck?.();
|
|
104
|
+
return scheduled ?? new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString();
|
|
105
|
+
}
|
|
100
106
|
async runTick() {
|
|
101
107
|
const failureCases = await this.#client.listFailureCases();
|
|
102
108
|
const candidates = failureCases.filter((failureCase) => failureCase.definition.environment === "sandbox"
|
|
@@ -126,7 +132,7 @@ export class ManagedFailureRuntimeWatch {
|
|
|
126
132
|
return this.#client.updateStatus(failureCase.id, {
|
|
127
133
|
health: "CONNECTION_UNAVAILABLE",
|
|
128
134
|
lastCheckedAt: checkedAt,
|
|
129
|
-
nextCheckAt:
|
|
135
|
+
nextCheckAt: this.nextCheckAt(checkedAt),
|
|
130
136
|
detail,
|
|
131
137
|
});
|
|
132
138
|
}));
|
|
@@ -142,7 +148,7 @@ export class ManagedFailureRuntimeWatch {
|
|
|
142
148
|
return this.#client.updateStatus(failureCase.id, {
|
|
143
149
|
health: "INSUFFICIENT_EVIDENCE",
|
|
144
150
|
lastCheckedAt: checkedAt,
|
|
145
|
-
nextCheckAt:
|
|
151
|
+
nextCheckAt: this.nextCheckAt(checkedAt),
|
|
146
152
|
detail,
|
|
147
153
|
});
|
|
148
154
|
}));
|
|
@@ -156,9 +162,8 @@ export class ManagedFailureRuntimeWatch {
|
|
|
156
162
|
const limitations = [];
|
|
157
163
|
for (const failureCase of eligible) {
|
|
158
164
|
const checkedAt = this.#now().toISOString();
|
|
159
|
-
const nextCheckAt = new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString();
|
|
160
165
|
const storedCheckpoint = await this.#checkpoints.load(failureCase.id);
|
|
161
|
-
|
|
166
|
+
let checkpoint = storedCheckpoint ?? {
|
|
162
167
|
processedReceiptIds: failureCase.runtimeWatch?.receiptIds ?? failureCase.runtimeWatch?.latest?.receiptIds ?? [],
|
|
163
168
|
updatedAt: checkedAt,
|
|
164
169
|
};
|
|
@@ -168,75 +173,87 @@ export class ManagedFailureRuntimeWatch {
|
|
|
168
173
|
await this.#client.updateStatus(failureCase.id, {
|
|
169
174
|
health: failureCase.status === "REGRESSION_DETECTED" ? "REGRESSION" : "HEALTHY",
|
|
170
175
|
lastCheckedAt: checkedAt,
|
|
171
|
-
nextCheckAt,
|
|
176
|
+
nextCheckAt: this.nextCheckAt(checkedAt),
|
|
172
177
|
});
|
|
173
178
|
if (!storedCheckpoint)
|
|
174
179
|
await this.#checkpoints.save(failureCase.id, checkpoint);
|
|
175
180
|
continue;
|
|
176
181
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
await this.#client.updateStatus(failureCase.id, {
|
|
189
|
-
health: outcome.classification === "SCHEMA_DIGEST_DRIFT" ? "INSUFFICIENT_EVIDENCE" : outcome.classification,
|
|
190
|
-
lastCheckedAt: checkedAt,
|
|
191
|
-
nextCheckAt,
|
|
192
|
-
...(outcome.reason ? { detail: outcome.reason } : {}),
|
|
182
|
+
// One wake can represent several resources. Drain that snapshot in order
|
|
183
|
+
// so a coalesced Receipt does not have to wait for the periodic fallback.
|
|
184
|
+
let batchStatus = { health: "HEALTHY" };
|
|
185
|
+
for (const resourceId of unique(pending.map((item) => item.resourceId))) {
|
|
186
|
+
const checkedAt = this.#now().toISOString();
|
|
187
|
+
const group = covered.filter((item) => item.resourceId === resourceId);
|
|
188
|
+
const outcome = await this.#readOutcome({
|
|
189
|
+
failureCase,
|
|
190
|
+
resourceId,
|
|
191
|
+
actions: group.map((item) => item.action),
|
|
192
|
+
receipts: group.map((item) => item.receipt),
|
|
193
193
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
: "A covered sandbox result remained healthy after the candidate was verified.",
|
|
220
|
-
measurements: {
|
|
194
|
+
if (outcome.classification !== "HEALTHY" && outcome.classification !== "REGRESSION") {
|
|
195
|
+
outcomesDeferred += 1;
|
|
196
|
+
limitations.push(outcome.reason ?? `Failure Runtime Watch deferred ${failureCase.id}: ${outcome.classification}.`);
|
|
197
|
+
if (batchStatus.health !== "REGRESSION")
|
|
198
|
+
batchStatus = {
|
|
199
|
+
health: outcome.classification === "SCHEMA_DIGEST_DRIFT" ? "INSUFFICIENT_EVIDENCE" : outcome.classification,
|
|
200
|
+
...(outcome.reason ? { detail: outcome.reason } : {}),
|
|
201
|
+
};
|
|
202
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
203
|
+
...batchStatus,
|
|
204
|
+
lastCheckedAt: checkedAt,
|
|
205
|
+
nextCheckAt: this.nextCheckAt(checkedAt),
|
|
206
|
+
});
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
const receiptIds = group.map((item) => item.receipt.id);
|
|
210
|
+
const agentVersion = requiredAgentVersion(failureCase, group.map((item) => item.action));
|
|
211
|
+
await this.#client.uploadObservation({
|
|
212
|
+
discoveryKey: failureCase.discovery?.discoveryKey ?? failureCase.id,
|
|
213
|
+
phase: "RUNTIME",
|
|
214
|
+
observedAt: outcome.observedAt,
|
|
215
|
+
definition: structuredClone(failureCase.definition),
|
|
216
|
+
result: {
|
|
217
|
+
agentVersion,
|
|
218
|
+
attempts: outcome.attempts ?? group.length,
|
|
221
219
|
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
222
220
|
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
223
221
|
failureCount: outcome.failureCount ?? 0,
|
|
224
222
|
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
225
|
-
|
|
223
|
+
observation: "RECORDED",
|
|
224
|
+
executionControl: "ENFORCED",
|
|
225
|
+
outcome: "VERIFIED",
|
|
226
|
+
review: "NOT_REVIEWED",
|
|
227
|
+
receiptIds,
|
|
226
228
|
},
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
229
|
+
evidence: {
|
|
230
|
+
summary: outcome.classification === "REGRESSION"
|
|
231
|
+
? "A covered sandbox result reproduced the defined failure after the candidate was verified."
|
|
232
|
+
: "A covered sandbox result remained healthy after the candidate was verified.",
|
|
233
|
+
measurements: {
|
|
234
|
+
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
235
|
+
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
236
|
+
failureCount: outcome.failureCount ?? 0,
|
|
237
|
+
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
238
|
+
receiptCount: receiptIds.length,
|
|
239
|
+
},
|
|
240
|
+
limitations: ["This automated observation covers the exact customer-owned sandbox task and action path only; it does not establish production reliability."],
|
|
241
|
+
},
|
|
242
|
+
});
|
|
243
|
+
observationsUploaded += 1;
|
|
244
|
+
if (outcome.classification === "REGRESSION")
|
|
245
|
+
batchStatus = { health: "REGRESSION" };
|
|
246
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
247
|
+
...batchStatus,
|
|
248
|
+
lastCheckedAt: checkedAt,
|
|
249
|
+
nextCheckAt: this.nextCheckAt(checkedAt),
|
|
250
|
+
});
|
|
251
|
+
checkpoint = {
|
|
252
|
+
processedReceiptIds: unique([...checkpoint.processedReceiptIds, ...receiptIds]),
|
|
253
|
+
updatedAt: checkedAt,
|
|
254
|
+
};
|
|
255
|
+
await this.#checkpoints.save(failureCase.id, checkpoint);
|
|
256
|
+
}
|
|
240
257
|
}
|
|
241
258
|
return { casesInspected: eligible.length, observationsUploaded, outcomesDeferred, limitations };
|
|
242
259
|
}
|