witnora 0.18.12 → 0.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gateway-service.js +5 -0
- package/dist/gateway.js +52 -3
- package/dist/onboard.js +31 -11
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.d.ts +65 -0
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.d.ts.map +1 -0
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.js +146 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts +144 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts.map +1 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.js +457 -0
- package/package.json +1 -1
package/dist/gateway-service.js
CHANGED
|
@@ -53,6 +53,11 @@ export async function restartCurrentGatewayService(options) {
|
|
|
53
53
|
await run(plan.start.command, plan.start.args);
|
|
54
54
|
return plan;
|
|
55
55
|
}
|
|
56
|
+
export async function stopCurrentGatewayService(options) {
|
|
57
|
+
const plan = createGatewayServicePlan(currentPlanInput(options));
|
|
58
|
+
await (options.run ?? runCommand)(plan.stop.command, plan.stop.args).catch(() => undefined);
|
|
59
|
+
return plan;
|
|
60
|
+
}
|
|
56
61
|
export function createGatewayServicePlan(input) {
|
|
57
62
|
const suffix = createHash("sha256").update(`${input.repository}\n${input.gatewayDirectory}`).digest("hex").slice(0, 12);
|
|
58
63
|
const id = `witnora-gateway-${suffix}`;
|
package/dist/gateway.js
CHANGED
|
@@ -609,8 +609,12 @@ export async function runCustomerGateway(options) {
|
|
|
609
609
|
throw error;
|
|
610
610
|
}
|
|
611
611
|
}
|
|
612
|
-
function configManagedWorkflowHarnessImport() {
|
|
613
|
-
|
|
612
|
+
async function configManagedWorkflowHarnessImport() {
|
|
613
|
+
const [managed, runtimeWatch] = await Promise.all([
|
|
614
|
+
import(new URL("./vendor/onegent-runtime/managed-workflow-harness.js", import.meta.url).href),
|
|
615
|
+
import(new URL("./vendor/onegent-runtime/failure-runtime-watch.js", import.meta.url).href),
|
|
616
|
+
]);
|
|
617
|
+
return { ...managed, ...runtimeWatch };
|
|
614
618
|
}
|
|
615
619
|
function configBusinessTaskEvaluatorImport() {
|
|
616
620
|
return import(new URL("./vendor/onegent-runtime/business-task-evaluator.js", import.meta.url).href);
|
|
@@ -671,6 +675,43 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
671
675
|
return body.report;
|
|
672
676
|
},
|
|
673
677
|
});
|
|
678
|
+
const runtimeWatchConfiguration = input.config.schemaVersion === MANAGED_WORKFLOW_HARNESS_SCHEMA
|
|
679
|
+
&& input.runtimeWorker?.adapterId === LOCAL_SANDBOX_ADAPTER_ID
|
|
680
|
+
&& (!input.config.realPathActivations || (input.config.realPathActivations.length === 1
|
|
681
|
+
&& input.config.realPathActivations[0]?.environment === "sandbox"
|
|
682
|
+
&& input.config.realPathActivations[0].actionPathIds.length === 1));
|
|
683
|
+
const activation = runtimeWatchConfiguration
|
|
684
|
+
&& input.config.schemaVersion === MANAGED_WORKFLOW_HARNESS_SCHEMA
|
|
685
|
+
&& input.config.realPathActivations?.length === 1
|
|
686
|
+
&& input.config.realPathActivations[0]?.environment === "sandbox"
|
|
687
|
+
&& input.config.realPathActivations[0].actionPathIds.length === 1
|
|
688
|
+
? input.config.realPathActivations[0]
|
|
689
|
+
: undefined;
|
|
690
|
+
const runtimeWatch = runtimeWatchConfiguration
|
|
691
|
+
&& input.managed.ManagedFailureRuntimeWatch
|
|
692
|
+
&& input.managed.FileFailureRuntimeWatchCheckpointStore
|
|
693
|
+
&& input.managed.createFailureRuntimeWatchHostedClient
|
|
694
|
+
&& input.managed.readLocalSandboxRuntimeWatchOutcome
|
|
695
|
+
? new input.managed.ManagedFailureRuntimeWatch({
|
|
696
|
+
client: input.managed.createFailureRuntimeWatchHostedClient({
|
|
697
|
+
baseUrl: input.server,
|
|
698
|
+
projectId: input.projectId,
|
|
699
|
+
apiKey: input.apiKey,
|
|
700
|
+
fetch: requestFetch,
|
|
701
|
+
}),
|
|
702
|
+
checkpoints: new input.managed.FileFailureRuntimeWatchCheckpointStore(join(input.directory, "data", "failure-runtime-watch")),
|
|
703
|
+
intervalMs: input.config.pollIntervalMs ?? 5_000,
|
|
704
|
+
...(activation ? { bindings: [{ taskContractId: activation.taskContractId, actionPathId: activation.actionPathIds[0], environment: "sandbox" }] } : {}),
|
|
705
|
+
readOutcome: async ({ resourceId, actions, receipts }) => input.managed.readLocalSandboxRuntimeWatchOutcome({
|
|
706
|
+
path: join(input.directory, "data", "runtime-sandbox", "fixture", "audit.jsonl"),
|
|
707
|
+
resourceId,
|
|
708
|
+
actions: actions.map((action) => {
|
|
709
|
+
const executionSessionId = receipts.find((receipt) => receipt.actionId === action.id)?.receipt?.core?.executionSessionId;
|
|
710
|
+
return { id: action.id, ...(executionSessionId ? { executionSessionId } : {}) };
|
|
711
|
+
}),
|
|
712
|
+
}),
|
|
713
|
+
})
|
|
714
|
+
: undefined;
|
|
674
715
|
let timer;
|
|
675
716
|
let closing = false;
|
|
676
717
|
let lastTickAt;
|
|
@@ -683,12 +724,19 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
683
724
|
await managedEvaluator.healthCheck();
|
|
684
725
|
}
|
|
685
726
|
const result = await worker.tick();
|
|
727
|
+
const watchResult = runtimeWatch ? await runtimeWatch.tick() : undefined;
|
|
686
728
|
lastTickAt = new Date().toISOString();
|
|
687
729
|
lastError = (result.evaluationsFailed ?? 0) > 0 ? result.limitations?.[0] ?? "One or more evaluations failed closed." : undefined;
|
|
688
730
|
if ((result.evaluationsFailed ?? 0) > 0) {
|
|
689
731
|
process.stderr.write(`Managed Workflow Harness stopped ${result.evaluationsFailed} evaluation(s) fail-closed. ${lastError}\n`);
|
|
690
732
|
}
|
|
691
|
-
return
|
|
733
|
+
return watchResult ? {
|
|
734
|
+
...result,
|
|
735
|
+
runtimeWatchCasesInspected: watchResult.casesInspected ?? 0,
|
|
736
|
+
runtimeWatchObservationsUploaded: watchResult.observationsUploaded ?? 0,
|
|
737
|
+
runtimeWatchOutcomesDeferred: watchResult.outcomesDeferred ?? 0,
|
|
738
|
+
runtimeWatchLimitations: watchResult.limitations ?? [],
|
|
739
|
+
} : result;
|
|
692
740
|
}
|
|
693
741
|
catch (error) {
|
|
694
742
|
lastTickAt = new Date().toISOString();
|
|
@@ -751,6 +799,7 @@ export async function createContinuousAssuranceController(input) {
|
|
|
751
799
|
server: input.server,
|
|
752
800
|
apiKey: input.apiKey,
|
|
753
801
|
config,
|
|
802
|
+
runtimeWorker: input.config.runtimeWorker,
|
|
754
803
|
managed,
|
|
755
804
|
evaluatorKit: input.evaluatorKit,
|
|
756
805
|
fetch: request,
|
package/dist/onboard.js
CHANGED
|
@@ -12,7 +12,7 @@ import { doctorCustomerGateway, activateManagedWorkflowHarness, ensureCustomerGa
|
|
|
12
12
|
import { discoverRuntimeReferences, planRuntimeSandboxModules } from "./runtime-sandbox-kit.js";
|
|
13
13
|
import { automaticRuntimeReferencesUsable, bootstrapLocalRuntime, verifyAutomaticRuntimeProbe } from "./runtime-bootstrap.js";
|
|
14
14
|
import { activateRealPathIntegrations } from "./real-path-activation.js";
|
|
15
|
-
import { installCurrentGatewayService } from "./gateway-service.js";
|
|
15
|
+
import { installCurrentGatewayService, stopCurrentGatewayService } from "./gateway-service.js";
|
|
16
16
|
export async function runOnboard(options) {
|
|
17
17
|
const requestFetch = options.fetch ?? fetch;
|
|
18
18
|
const output = options.output ?? ((message) => process.stdout.write(message));
|
|
@@ -91,6 +91,19 @@ export async function runOnboard(options) {
|
|
|
91
91
|
start: startManagedCustomerGateway,
|
|
92
92
|
stop: stopManagedCustomerGateway,
|
|
93
93
|
};
|
|
94
|
+
const gatewayServiceLifecycle = options.gatewayServiceLifecycle ?? (!options.gatewayLifecycle ? {
|
|
95
|
+
stop: async ({ repository, configHome }) => {
|
|
96
|
+
await stopCurrentGatewayService({
|
|
97
|
+
repository,
|
|
98
|
+
configHome,
|
|
99
|
+
cliEntry: fileURLToPath(new URL("./cli.js", import.meta.url)),
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
} : undefined);
|
|
103
|
+
const stopGatewayOwners = async () => {
|
|
104
|
+
await gatewayServiceLifecycle?.stop({ repository: repositoryPath, configHome: options.configHome });
|
|
105
|
+
return gatewayLifecycle.stop({ repository: repositoryPath, configHome: options.configHome, fetch: requestFetch });
|
|
106
|
+
};
|
|
94
107
|
let gatewayMigration;
|
|
95
108
|
let managedGateway;
|
|
96
109
|
let runtimeUpgrade;
|
|
@@ -184,7 +197,7 @@ export async function runOnboard(options) {
|
|
|
184
197
|
try {
|
|
185
198
|
runtimeUpgrade = await prepareRuntime(() => upgradeCustomerGatewayRuntime({ repository: repositoryPath, authorization: token, runtimeReferences: localRuntime.references, agentIdentity, configHome: options.configHome, fetch: requestFetch }));
|
|
186
199
|
if (runtimeUpgrade.changed) {
|
|
187
|
-
const stopped = await
|
|
200
|
+
const stopped = await stopGatewayOwners();
|
|
188
201
|
if (stopped.state !== "STOPPED" || stopped.healthy)
|
|
189
202
|
throw new Error("The prior managed Gateway process did not stop before Runtime generation replacement.");
|
|
190
203
|
}
|
|
@@ -205,11 +218,11 @@ export async function runOnboard(options) {
|
|
|
205
218
|
}
|
|
206
219
|
}
|
|
207
220
|
else {
|
|
208
|
-
const stopped = await
|
|
221
|
+
const stopped = await stopGatewayOwners();
|
|
209
222
|
if (stopped.state === "STOPPED" || stopped.state === "STALE") {
|
|
210
223
|
output(`\nStopped the managed Gateway for project ${binding.projectId} before rebinding this repository.\n`);
|
|
211
224
|
}
|
|
212
|
-
await assertGatewayStopped(requestFetch, binding.host, binding.port, binding.projectId, token.projectId);
|
|
225
|
+
await assertGatewayStopped(requestFetch, binding.host, binding.port, binding.collectorId, binding.projectId, token.projectId);
|
|
213
226
|
gatewayMigration = await archiveGateway(gatewayState.directory, repositoryPath, binding.projectId);
|
|
214
227
|
output(`\nExisting Gateway belongs to project ${binding.projectId}; archived it at ${gatewayMigration.archiveDirectory}.\n`);
|
|
215
228
|
const gateway = await prepareRuntime(() => initializeCustomerGateway({
|
|
@@ -239,14 +252,14 @@ export async function runOnboard(options) {
|
|
|
239
252
|
assuranceHarnessReadiness = { state: "WAITING_FOR_CUSTOMER_HARNESS", limitation: detail };
|
|
240
253
|
}
|
|
241
254
|
if (assuranceHarnessChanged && gatewayState.status !== "absent") {
|
|
242
|
-
const stopped = await
|
|
255
|
+
const stopped = await stopGatewayOwners();
|
|
243
256
|
if (stopped.state !== "STOPPED" && stopped.state !== "STALE")
|
|
244
257
|
throw new Error("The prior managed Gateway process did not stop before Assurance Harness activation.");
|
|
245
258
|
}
|
|
246
259
|
if (!options.gatewayLifecycle) {
|
|
247
260
|
// Stop any detached Gateway left by an earlier CLI before the OS-owned
|
|
248
261
|
// supervisor is installed. The service becomes the single process owner.
|
|
249
|
-
await
|
|
262
|
+
await stopGatewayOwners().catch(() => undefined);
|
|
250
263
|
const port = await ensureCustomerGatewayPortAvailable({ repository: repositoryPath });
|
|
251
264
|
if (port.changed)
|
|
252
265
|
output(`Rebound this repository's generated Gateway to available localhost port ${port.port}; another project remains untouched.\n`);
|
|
@@ -304,7 +317,7 @@ export async function runOnboard(options) {
|
|
|
304
317
|
catch (error) {
|
|
305
318
|
const diagnosis = error instanceof Error ? error.message : String(error);
|
|
306
319
|
if (managedGateway?.started) {
|
|
307
|
-
await
|
|
320
|
+
await stopGatewayOwners().catch(() => undefined);
|
|
308
321
|
}
|
|
309
322
|
await realPathActivation?.rollback().catch(() => undefined);
|
|
310
323
|
await rollbackGeneratedFiles(generatedFiles);
|
|
@@ -349,18 +362,25 @@ async function inspectGatewayBinding(directory, projectId, server) {
|
|
|
349
362
|
const existingProjectId = typeof value.projectId === "string" ? value.projectId.trim() : "";
|
|
350
363
|
const existingServer = typeof value.server === "string" ? normalizeServer(value.server) : "";
|
|
351
364
|
const connectionName = typeof value.connectionName === "string" ? value.connectionName.trim() : "";
|
|
365
|
+
const collectorId = typeof value.collectorId === "string" ? value.collectorId.trim() : "";
|
|
352
366
|
const host = typeof value.host === "string" && value.host.trim() ? value.host.trim() : "127.0.0.1";
|
|
353
367
|
const port = Number(value.port);
|
|
354
|
-
if (!existingProjectId || !existingServer || !connectionName || !Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
355
|
-
throw new Error("Existing Gateway configuration is missing a valid project, server, host, or port. Preserve it and repair the setup in Advanced mode.");
|
|
368
|
+
if (!existingProjectId || !existingServer || !connectionName || !collectorId || !Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
369
|
+
throw new Error("Existing Gateway configuration is missing a valid project, server, connection, collector, host, or port. Preserve it and repair the setup in Advanced mode.");
|
|
356
370
|
}
|
|
357
|
-
return { matches: existingProjectId === projectId && existingServer === server, projectId: existingProjectId, server: existingServer, connectionName, host, port };
|
|
371
|
+
return { matches: existingProjectId === projectId && existingServer === server, projectId: existingProjectId, server: existingServer, connectionName, collectorId, host, port };
|
|
358
372
|
}
|
|
359
|
-
async function assertGatewayStopped(requestFetch, host, port, oldProjectId, newProjectId) {
|
|
373
|
+
async function assertGatewayStopped(requestFetch, host, port, oldCollectorId, oldProjectId, newProjectId) {
|
|
360
374
|
try {
|
|
361
375
|
const response = await requestFetch(`http://${host}:${port}/healthz`, { signal: AbortSignal.timeout(800) });
|
|
362
376
|
if (!response.ok)
|
|
363
377
|
return;
|
|
378
|
+
const health = await response.json().catch(() => ({}));
|
|
379
|
+
// A different repository may legitimately own the old localhost port.
|
|
380
|
+
// It must remain untouched; this repository will select a free port after
|
|
381
|
+
// its prior Gateway directory is archived and regenerated.
|
|
382
|
+
if (health.collectorId !== oldCollectorId)
|
|
383
|
+
return;
|
|
364
384
|
}
|
|
365
385
|
catch {
|
|
366
386
|
return;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export declare const FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION: "witnora.failure_experiment_submission.v0.1";
|
|
2
|
+
export interface HostedFailureDefinition {
|
|
3
|
+
title: string;
|
|
4
|
+
taskContractId: string;
|
|
5
|
+
actionPathId: string;
|
|
6
|
+
provider: string;
|
|
7
|
+
environment: "sandbox" | "production";
|
|
8
|
+
failureSignal: string;
|
|
9
|
+
successSignal: string;
|
|
10
|
+
hardInvariants: string[];
|
|
11
|
+
outcomeCheck: {
|
|
12
|
+
kind: "READ_ONLY_PROVIDER_QUERY" | "READ_ONLY_DATABASE_QUERY" | "SIGNED_WEBHOOK_OBSERVATION" | "CUSTOMER_PROBE";
|
|
13
|
+
providerState: string;
|
|
14
|
+
expectedValue: string;
|
|
15
|
+
};
|
|
16
|
+
reproduction: {
|
|
17
|
+
kind: "DETERMINISTIC";
|
|
18
|
+
requiredRuns: number;
|
|
19
|
+
} | {
|
|
20
|
+
kind: "STATISTICAL";
|
|
21
|
+
runs: number;
|
|
22
|
+
minimumFailureRate: number;
|
|
23
|
+
allowedError: number;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export interface HostedFailureExperimentInput {
|
|
27
|
+
discoveryKey: string;
|
|
28
|
+
phase: "BASELINE" | "CANDIDATE" | "RUNTIME";
|
|
29
|
+
/** Stable source observation time; callers should reuse it on retry. */
|
|
30
|
+
observedAt?: string;
|
|
31
|
+
definition: HostedFailureDefinition;
|
|
32
|
+
result: {
|
|
33
|
+
agentVersion: string;
|
|
34
|
+
attempts: number;
|
|
35
|
+
providerCommitCount: number;
|
|
36
|
+
duplicateOutcomeCount: number;
|
|
37
|
+
failureCount: number;
|
|
38
|
+
hardInvariantViolations?: number;
|
|
39
|
+
observation: "REPORTED" | "RECORDED";
|
|
40
|
+
executionControl: "ENFORCED" | "NOT_ENFORCED" | "UNKNOWN";
|
|
41
|
+
outcome: "VERIFIED" | "NOT_VERIFIED" | "INCONCLUSIVE";
|
|
42
|
+
review: "NOT_REVIEWED" | "INTERNALLY_REVIEWED" | "INDEPENDENTLY_REVIEWED";
|
|
43
|
+
receiptIds?: string[];
|
|
44
|
+
};
|
|
45
|
+
evidence: {
|
|
46
|
+
summary: string;
|
|
47
|
+
measurements?: Record<string, string | number | boolean | null>;
|
|
48
|
+
limitations?: string[];
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export interface HostedFailureExperimentOptions {
|
|
52
|
+
baseUrl: string;
|
|
53
|
+
projectId: string;
|
|
54
|
+
apiKey: string;
|
|
55
|
+
fetch?: typeof fetch;
|
|
56
|
+
}
|
|
57
|
+
export interface HostedFailureExperimentResult {
|
|
58
|
+
run: Record<string, unknown>;
|
|
59
|
+
evidence: Record<string, unknown>;
|
|
60
|
+
completion: Record<string, unknown>;
|
|
61
|
+
failureCase: Record<string, unknown>;
|
|
62
|
+
verifiedFixReport: Record<string, unknown>;
|
|
63
|
+
}
|
|
64
|
+
export declare function uploadFailureExperiment(input: HostedFailureExperimentInput, options: HostedFailureExperimentOptions): Promise<HostedFailureExperimentResult>;
|
|
65
|
+
//# sourceMappingURL=failure-experiment-hosted.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-experiment-hosted.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-experiment-hosted.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,4CAA4C,EAAG,4CAAqD,CAAC;AAElH,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,SAAS,GAAG,YAAY,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE;QACZ,IAAI,EAAE,0BAA0B,GAAG,0BAA0B,GAAG,4BAA4B,GAAG,gBAAgB,CAAC;QAChH,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,YAAY,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GACzD;QAAE,IAAI,EAAE,aAAa,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7F;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,uBAAuB,CAAC;IACpC,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,YAAY,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,WAAW,EAAE,UAAU,GAAG,UAAU,CAAC;QACrC,gBAAgB,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC;QAC1D,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,cAAc,CAAC;QACtD,MAAM,EAAE,cAAc,GAAG,qBAAqB,GAAG,wBAAwB,CAAC;QAC1E,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QAChE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;CACH;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,MAAM,WAAW,6BAA6B;IAC5C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,4BAA4B,EACnC,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,6BAA6B,CAAC,CAsFxC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION = "witnora.failure_experiment_submission.v0.1";
|
|
3
|
+
export async function uploadFailureExperiment(input, options) {
|
|
4
|
+
if (input.definition.environment !== "sandbox") {
|
|
5
|
+
throw new Error("Automatic Failure Case progression is limited to the customer sandbox.");
|
|
6
|
+
}
|
|
7
|
+
validateExperimentCounts(input.result);
|
|
8
|
+
const baseUrl = required(options.baseUrl, "baseUrl").replace(/\/$/, "");
|
|
9
|
+
const projectId = required(options.projectId, "projectId");
|
|
10
|
+
const apiKey = required(options.apiKey, "apiKey");
|
|
11
|
+
const requestFetch = options.fetch ?? fetch;
|
|
12
|
+
const projectUrl = `${baseUrl}/v1/projects/${encodeURIComponent(projectId)}`;
|
|
13
|
+
const submittedAt = input.observedAt === undefined ? new Date().toISOString() : normalizedTimestamp(input.observedAt, "observedAt");
|
|
14
|
+
const boundedEvidence = {
|
|
15
|
+
schemaVersion: "witnora.failure_experiment_evidence.v0.1",
|
|
16
|
+
discoveryKey: input.discoveryKey,
|
|
17
|
+
phase: input.phase,
|
|
18
|
+
taskContractId: input.definition.taskContractId,
|
|
19
|
+
actionPathId: input.definition.actionPathId,
|
|
20
|
+
environment: input.definition.environment,
|
|
21
|
+
provider: input.definition.provider,
|
|
22
|
+
summary: bounded(input.evidence.summary, "evidence.summary", 1, 2_000),
|
|
23
|
+
measurements: boundedMeasurements(input.evidence.measurements),
|
|
24
|
+
limitations: (input.evidence.limitations ?? []).slice(0, 20).map((item) => bounded(item, "evidence.limitation", 1, 500)),
|
|
25
|
+
result: structuredClone(input.result),
|
|
26
|
+
observedAt: submittedAt,
|
|
27
|
+
rawPayloadRetainedByCustomer: true,
|
|
28
|
+
};
|
|
29
|
+
const bytes = Buffer.from(`${JSON.stringify(boundedEvidence, null, 2)}\n`, "utf8");
|
|
30
|
+
if (bytes.byteLength > 64 * 1024)
|
|
31
|
+
throw new Error("Failure experiment evidence exceeds the 64 KiB hosted boundary.");
|
|
32
|
+
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
33
|
+
const operation = `failure-experiment-${digest.slice(0, 32)}`;
|
|
34
|
+
const run = await jsonRequest(requestFetch, `${projectUrl}/runs`, apiKey, {
|
|
35
|
+
method: "POST",
|
|
36
|
+
headers: { "idempotency-key": `${operation}:run` },
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
externalId: `${input.discoveryKey}:${input.phase.toLowerCase()}:${digest.slice(0, 16)}`,
|
|
39
|
+
kind: "custom",
|
|
40
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
41
|
+
startedAt: submittedAt,
|
|
42
|
+
metadata: { failureExperiment: {
|
|
43
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
44
|
+
discoveryKey: input.discoveryKey,
|
|
45
|
+
phase: input.phase,
|
|
46
|
+
taskContractId: input.definition.taskContractId,
|
|
47
|
+
actionPathId: input.definition.actionPathId,
|
|
48
|
+
evidenceDigestSha256: digest,
|
|
49
|
+
rawPayloadRetainedByCustomer: true,
|
|
50
|
+
} },
|
|
51
|
+
}),
|
|
52
|
+
});
|
|
53
|
+
const runId = id(run, "Run");
|
|
54
|
+
const query = new URLSearchParams({
|
|
55
|
+
fileName: `failure-experiment-${input.phase.toLowerCase()}-evidence.json`,
|
|
56
|
+
kind: "failure_experiment",
|
|
57
|
+
schemaVersion: "witnora.failure_experiment_evidence.v0.1",
|
|
58
|
+
runId,
|
|
59
|
+
});
|
|
60
|
+
const evidence = await jsonRequest(requestFetch, `${projectUrl}/evidence?${query}`, apiKey, {
|
|
61
|
+
method: "POST",
|
|
62
|
+
headers: { "content-type": "application/json", "idempotency-key": `${operation}:evidence` },
|
|
63
|
+
body: new Uint8Array(bytes).buffer,
|
|
64
|
+
});
|
|
65
|
+
const evidenceId = id(evidence, "Evidence");
|
|
66
|
+
const completion = await jsonRequest(requestFetch, `${projectUrl}/runs/${encodeURIComponent(runId)}/complete`, apiKey, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { "idempotency-key": `${operation}:complete` },
|
|
69
|
+
body: JSON.stringify({
|
|
70
|
+
status: input.phase === "BASELINE" ? "manual_review" : input.result.failureCount ? "failed" : "passed",
|
|
71
|
+
score: input.result.failureCount ? 0 : 1,
|
|
72
|
+
summary: boundedEvidence.summary,
|
|
73
|
+
metadata: { evidenceId, evidenceSha256: evidence.sha256, failureExperimentDigestSha256: digest },
|
|
74
|
+
}),
|
|
75
|
+
});
|
|
76
|
+
const failureCase = await jsonRequest(requestFetch, `${projectUrl}/failure-cases/ingest`, apiKey, {
|
|
77
|
+
method: "POST",
|
|
78
|
+
headers: { "idempotency-key": `${operation}:ingest` },
|
|
79
|
+
body: JSON.stringify({
|
|
80
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
81
|
+
discoveryKey: input.discoveryKey,
|
|
82
|
+
phase: input.phase,
|
|
83
|
+
definition: input.definition,
|
|
84
|
+
result: { ...input.result, runIds: [runId], evidenceIds: [evidenceId] },
|
|
85
|
+
}),
|
|
86
|
+
});
|
|
87
|
+
const failureCaseId = id(failureCase, "Failure Case");
|
|
88
|
+
const verifiedFixReport = await jsonRequest(requestFetch, `${projectUrl}/failure-cases/${encodeURIComponent(failureCaseId)}/report`, apiKey, { method: "GET" });
|
|
89
|
+
return { run, evidence, completion, failureCase, verifiedFixReport };
|
|
90
|
+
}
|
|
91
|
+
function validateExperimentCounts(result) {
|
|
92
|
+
const counts = [
|
|
93
|
+
result.attempts,
|
|
94
|
+
result.providerCommitCount,
|
|
95
|
+
result.duplicateOutcomeCount,
|
|
96
|
+
result.failureCount,
|
|
97
|
+
result.hardInvariantViolations ?? 0,
|
|
98
|
+
];
|
|
99
|
+
if (counts.some((value) => !Number.isInteger(value) || value < 0)) {
|
|
100
|
+
throw new Error("Failure experiment counts must be non-negative integers.");
|
|
101
|
+
}
|
|
102
|
+
if (result.attempts < 1
|
|
103
|
+
|| result.providerCommitCount > result.attempts
|
|
104
|
+
|| result.failureCount > result.attempts
|
|
105
|
+
|| result.duplicateOutcomeCount > result.providerCommitCount) {
|
|
106
|
+
throw new Error("Failure experiment counts are internally inconsistent.");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function jsonRequest(requestFetch, url, apiKey, init) {
|
|
110
|
+
const headers = new Headers(init.headers);
|
|
111
|
+
headers.set("authorization", `Bearer ${apiKey}`);
|
|
112
|
+
if (typeof init.body === "string")
|
|
113
|
+
headers.set("content-type", "application/json");
|
|
114
|
+
const response = await requestFetch(url, { ...init, headers, redirect: "error" });
|
|
115
|
+
const value = await response.json().catch(() => ({}));
|
|
116
|
+
if (!response.ok)
|
|
117
|
+
throw new Error(typeof value.error === "string" ? value.error : `Witnora Control Plane request failed (${response.status}).`);
|
|
118
|
+
return value;
|
|
119
|
+
}
|
|
120
|
+
function required(value, name) {
|
|
121
|
+
return bounded(value.trim(), name, 1, 2_000);
|
|
122
|
+
}
|
|
123
|
+
function normalizedTimestamp(value, name) {
|
|
124
|
+
if (!value.trim() || Number.isNaN(Date.parse(value)))
|
|
125
|
+
throw new Error(`Failure experiment ${name} must be an ISO timestamp.`);
|
|
126
|
+
return new Date(value).toISOString();
|
|
127
|
+
}
|
|
128
|
+
function bounded(value, name, minimum, maximum) {
|
|
129
|
+
if (value.length < minimum || value.length > maximum)
|
|
130
|
+
throw new Error(`Failure experiment ${name} must contain ${minimum}-${maximum} characters.`);
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
function boundedMeasurements(value) {
|
|
134
|
+
const entries = Object.entries(value ?? {}).slice(0, 50);
|
|
135
|
+
for (const [key, item] of entries) {
|
|
136
|
+
bounded(key, "evidence.measurement key", 1, 100);
|
|
137
|
+
if (typeof item === "string")
|
|
138
|
+
bounded(item, "evidence.measurement value", 0, 500);
|
|
139
|
+
}
|
|
140
|
+
return Object.fromEntries(entries);
|
|
141
|
+
}
|
|
142
|
+
function id(value, kind) {
|
|
143
|
+
if (typeof value.id !== "string" || !value.id)
|
|
144
|
+
throw new Error(`Witnora Control Plane returned ${kind} without an id.`);
|
|
145
|
+
return value.id;
|
|
146
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { type HostedFailureDefinition, type HostedFailureExperimentInput } from "./failure-experiment-hosted.js";
|
|
2
|
+
export type FailureRuntimeWatchHealth = "HEALTHY" | "REGRESSION" | "INSUFFICIENT_EVIDENCE" | "CONNECTION_UNAVAILABLE";
|
|
3
|
+
export interface FailureRuntimeWatchCase {
|
|
4
|
+
id: string;
|
|
5
|
+
projectId: string;
|
|
6
|
+
status: "FIX_VERIFIED" | "RUNTIME_PROTECTED" | "REGRESSION_DETECTED" | string;
|
|
7
|
+
updatedAt?: string;
|
|
8
|
+
definition: HostedFailureDefinition;
|
|
9
|
+
discovery?: {
|
|
10
|
+
discoveryKey?: string;
|
|
11
|
+
};
|
|
12
|
+
candidate?: {
|
|
13
|
+
agentVersion: string;
|
|
14
|
+
recordedAt?: string;
|
|
15
|
+
};
|
|
16
|
+
runtimeWatch?: {
|
|
17
|
+
activatedAt?: string;
|
|
18
|
+
receiptIds?: string[];
|
|
19
|
+
latest?: {
|
|
20
|
+
receiptIds?: string[];
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export interface FailureRuntimeWatchAction {
|
|
25
|
+
id: string;
|
|
26
|
+
status: string;
|
|
27
|
+
updatedAt?: string;
|
|
28
|
+
verificationSuccess?: boolean;
|
|
29
|
+
expectedState?: Record<string, unknown>;
|
|
30
|
+
businessTaskBinding?: {
|
|
31
|
+
taskContractId?: string;
|
|
32
|
+
actionPathId?: string;
|
|
33
|
+
environment?: string;
|
|
34
|
+
agentVersion?: string;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export interface FailureRuntimeWatchReceipt {
|
|
38
|
+
id: string;
|
|
39
|
+
actionId: string;
|
|
40
|
+
currentStatus: string;
|
|
41
|
+
createdAt: string;
|
|
42
|
+
receipt: {
|
|
43
|
+
core: {
|
|
44
|
+
evidenceStrength?: string;
|
|
45
|
+
enforcementLevel?: string;
|
|
46
|
+
executionSessionId?: string;
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
discoveryBoundary?: "SIGNED_VERIFIED_ACTION";
|
|
50
|
+
}
|
|
51
|
+
export interface FailureRuntimeWatchOutcome {
|
|
52
|
+
classification: "HEALTHY" | "REGRESSION" | "INSUFFICIENT_EVIDENCE" | "CONNECTION_UNAVAILABLE" | "SCHEMA_DIGEST_DRIFT";
|
|
53
|
+
attempts?: number;
|
|
54
|
+
providerCommitCount?: number;
|
|
55
|
+
duplicateOutcomeCount?: number;
|
|
56
|
+
failureCount?: number;
|
|
57
|
+
hardInvariantViolations?: number;
|
|
58
|
+
observedAt: string;
|
|
59
|
+
reason?: string;
|
|
60
|
+
}
|
|
61
|
+
export type FailureRuntimeWatchOutcomeReader = (input: {
|
|
62
|
+
failureCase: FailureRuntimeWatchCase;
|
|
63
|
+
resourceId: string;
|
|
64
|
+
actions: FailureRuntimeWatchAction[];
|
|
65
|
+
receipts: FailureRuntimeWatchReceipt[];
|
|
66
|
+
}) => Promise<FailureRuntimeWatchOutcome>;
|
|
67
|
+
export interface FailureRuntimeWatchStatusInput {
|
|
68
|
+
health: FailureRuntimeWatchHealth;
|
|
69
|
+
lastCheckedAt: string;
|
|
70
|
+
nextCheckAt: string;
|
|
71
|
+
detail?: string;
|
|
72
|
+
}
|
|
73
|
+
export interface FailureRuntimeWatchHostedClient {
|
|
74
|
+
listFailureCases(): Promise<FailureRuntimeWatchCase[]>;
|
|
75
|
+
listActions(): Promise<FailureRuntimeWatchAction[]>;
|
|
76
|
+
listReceipts(actions?: FailureRuntimeWatchAction[]): Promise<FailureRuntimeWatchReceipt[]>;
|
|
77
|
+
uploadObservation(input: HostedFailureExperimentInput & {
|
|
78
|
+
observedAt: string;
|
|
79
|
+
}): Promise<unknown>;
|
|
80
|
+
updateStatus(failureCaseId: string, input: FailureRuntimeWatchStatusInput): Promise<unknown>;
|
|
81
|
+
}
|
|
82
|
+
export declare function createFailureRuntimeWatchHostedClient(options: {
|
|
83
|
+
baseUrl: string;
|
|
84
|
+
projectId: string;
|
|
85
|
+
apiKey: string;
|
|
86
|
+
fetch?: typeof fetch;
|
|
87
|
+
}): FailureRuntimeWatchHostedClient;
|
|
88
|
+
export interface FailureRuntimeWatchCheckpoint {
|
|
89
|
+
processedReceiptIds: string[];
|
|
90
|
+
updatedAt: string;
|
|
91
|
+
}
|
|
92
|
+
export interface FailureRuntimeWatchCheckpointStore {
|
|
93
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
94
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
95
|
+
}
|
|
96
|
+
export declare class MemoryFailureRuntimeWatchCheckpointStore implements FailureRuntimeWatchCheckpointStore {
|
|
97
|
+
#private;
|
|
98
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
99
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
100
|
+
}
|
|
101
|
+
export declare class FileFailureRuntimeWatchCheckpointStore implements FailureRuntimeWatchCheckpointStore {
|
|
102
|
+
private readonly directory;
|
|
103
|
+
constructor(directory: string);
|
|
104
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
105
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
106
|
+
private path;
|
|
107
|
+
}
|
|
108
|
+
export interface ManagedFailureRuntimeWatchResult {
|
|
109
|
+
casesInspected: number;
|
|
110
|
+
observationsUploaded: number;
|
|
111
|
+
outcomesDeferred: number;
|
|
112
|
+
limitations: string[];
|
|
113
|
+
}
|
|
114
|
+
export declare class ManagedFailureRuntimeWatch {
|
|
115
|
+
#private;
|
|
116
|
+
constructor(options: {
|
|
117
|
+
client: FailureRuntimeWatchHostedClient;
|
|
118
|
+
checkpoints: FailureRuntimeWatchCheckpointStore;
|
|
119
|
+
intervalMs?: number;
|
|
120
|
+
now?: () => Date;
|
|
121
|
+
readOutcome: FailureRuntimeWatchOutcomeReader;
|
|
122
|
+
bindings?: Array<{
|
|
123
|
+
taskContractId: string;
|
|
124
|
+
actionPathId: string;
|
|
125
|
+
environment: "sandbox";
|
|
126
|
+
}>;
|
|
127
|
+
});
|
|
128
|
+
tick(): Promise<ManagedFailureRuntimeWatchResult>;
|
|
129
|
+
private runTick;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Reads only the local sandbox result fields required by Runtime Watch. Raw
|
|
133
|
+
* provider payloads and unbound audit records never leave the customer Harness.
|
|
134
|
+
*/
|
|
135
|
+
export declare function readLocalSandboxRuntimeWatchOutcome(input: {
|
|
136
|
+
path: string;
|
|
137
|
+
resourceId: string;
|
|
138
|
+
actions: Array<{
|
|
139
|
+
id: string;
|
|
140
|
+
executionSessionId?: string;
|
|
141
|
+
}>;
|
|
142
|
+
now?: () => Date;
|
|
143
|
+
}): Promise<FailureRuntimeWatchOutcome>;
|
|
144
|
+
//# sourceMappingURL=failure-runtime-watch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-runtime-watch.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-runtime-watch.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2B,KAAK,uBAAuB,EAAE,KAAK,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAE1I,MAAM,MAAM,yBAAyB,GACjC,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,GAAG,MAAM,CAAC;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,uBAAuB,CAAC;IACpC,SAAS,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,SAAS,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,YAAY,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE,CAAC;CACpG;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,mBAAmB,CAAC,EAAE;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE;YAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IACzG,iBAAiB,CAAC,EAAE,wBAAwB,CAAC;CAC9C;AAED,MAAM,WAAW,0BAA0B;IACzC,cAAc,EAAE,SAAS,GAAG,YAAY,GAAG,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAC;IACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE;IACrD,WAAW,EAAE,uBAAuB,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,yBAAyB,EAAE,CAAC;IACrC,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACxC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,yBAAyB,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,+BAA+B;IAC9C,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACvD,WAAW,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAAC;IACpD,YAAY,CAAC,OAAO,CAAC,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IAC3F,iBAAiB,CAAC,KAAK,EAAE,4BAA4B,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClG,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9F;AAED,wBAAgB,qCAAqC,CAAC,OAAO,EAAE;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,GAAG,+BAA+B,CAoClC;AAED,MAAM,WAAW,6BAA6B;IAC5C,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E;AAED,qBAAa,wCAAyC,YAAW,kCAAkC;;IAG3F,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IAKrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;CAGlF;AAED,qBAAa,sCAAuC,YAAW,kCAAkC;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAIxC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IASrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IASjF,OAAO,CAAC,IAAI;CAIb;AAED,MAAM,WAAW,gCAAgC;IAC/C,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,0BAA0B;;gBASzB,OAAO,EAAE;QACnB,MAAM,EAAE,+BAA+B,CAAC;QACxC,WAAW,EAAE,kCAAkC,CAAC;QAChD,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;QACjB,WAAW,EAAE,gCAAgC,CAAC;QAC9C,QAAQ,CAAC,EAAE,KAAK,CAAC;YAAE,cAAc,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,SAAS,CAAA;SAAE,CAAC,CAAC;KAC5F;IAYD,IAAI,IAAI,OAAO,CAAC,gCAAgC,CAAC;YAMnC,OAAO;CA4ItB;AAID;;;GAGG;AACH,wBAAsB,mCAAmC,CAAC,KAAK,EAAE;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAiEtC"}
|
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { uploadFailureExperiment } from "./failure-experiment-hosted.js";
|
|
5
|
+
export function createFailureRuntimeWatchHostedClient(options) {
|
|
6
|
+
const baseUrl = required(options.baseUrl, "baseUrl").replace(/\/$/u, "");
|
|
7
|
+
const projectId = required(options.projectId, "projectId");
|
|
8
|
+
const apiKey = required(options.apiKey, "apiKey");
|
|
9
|
+
const requestFetch = options.fetch ?? fetch;
|
|
10
|
+
const projectUrl = `${baseUrl}/v1/projects/${encodeURIComponent(projectId)}`;
|
|
11
|
+
const receiptCache = new Map();
|
|
12
|
+
return {
|
|
13
|
+
listFailureCases: async () => list(await hostedJson(requestFetch, `${projectUrl}/failure-cases`, apiKey), "failureCases"),
|
|
14
|
+
listActions: async () => list(await hostedJson(requestFetch, `${projectUrl}/actions`, apiKey), "actions"),
|
|
15
|
+
listReceipts: async (actions = []) => {
|
|
16
|
+
const eligible = actions.filter((action) => action.status === "VERIFIED"
|
|
17
|
+
&& action.verificationSuccess === true
|
|
18
|
+
&& action.businessTaskBinding?.environment === "sandbox");
|
|
19
|
+
const discovered = await Promise.all(eligible.map(async (action) => {
|
|
20
|
+
const cached = receiptCache.get(action.id);
|
|
21
|
+
if (cached && cached.actionUpdatedAt === action.updatedAt)
|
|
22
|
+
return cached.receipts;
|
|
23
|
+
const detail = await hostedJson(requestFetch, `${projectUrl}/actions/${encodeURIComponent(required(action.id, "actionId"))}`, apiKey);
|
|
24
|
+
const receipts = signedVerifiedActionReceipt(detail);
|
|
25
|
+
receiptCache.set(action.id, { actionUpdatedAt: action.updatedAt, receipts });
|
|
26
|
+
return receipts;
|
|
27
|
+
}));
|
|
28
|
+
return discovered.flat();
|
|
29
|
+
},
|
|
30
|
+
uploadObservation: async (input) => uploadFailureExperiment(input, { baseUrl, projectId, apiKey, fetch: requestFetch }),
|
|
31
|
+
updateStatus: async (failureCaseId, input) => hostedJson(requestFetch, `${projectUrl}/failure-cases/${encodeURIComponent(required(failureCaseId, "failureCaseId"))}/runtime-watch-status`, apiKey, { method: "POST", body: JSON.stringify(input) }),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export class MemoryFailureRuntimeWatchCheckpointStore {
|
|
35
|
+
#checkpoints = new Map();
|
|
36
|
+
async load(key) {
|
|
37
|
+
const checkpoint = this.#checkpoints.get(key);
|
|
38
|
+
return checkpoint ? structuredClone(checkpoint) : undefined;
|
|
39
|
+
}
|
|
40
|
+
async save(key, checkpoint) {
|
|
41
|
+
this.#checkpoints.set(key, structuredClone(checkpoint));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export class FileFailureRuntimeWatchCheckpointStore {
|
|
45
|
+
directory;
|
|
46
|
+
constructor(directory) {
|
|
47
|
+
this.directory = directory;
|
|
48
|
+
if (!directory.trim())
|
|
49
|
+
throw new Error("Failure Runtime Watch checkpoint directory is required.");
|
|
50
|
+
}
|
|
51
|
+
async load(key) {
|
|
52
|
+
try {
|
|
53
|
+
return validateCheckpoint(JSON.parse(await readFile(this.path(key), "utf8")));
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
if (error.code === "ENOENT")
|
|
57
|
+
return undefined;
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async save(key, checkpoint) {
|
|
62
|
+
const value = validateCheckpoint(checkpoint);
|
|
63
|
+
await mkdir(this.directory, { recursive: true });
|
|
64
|
+
const target = this.path(key);
|
|
65
|
+
const temporary = `${target}.${randomUUID()}.tmp`;
|
|
66
|
+
await writeFile(temporary, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
67
|
+
await rename(temporary, target);
|
|
68
|
+
}
|
|
69
|
+
path(key) {
|
|
70
|
+
if (!key.trim())
|
|
71
|
+
throw new Error("Failure Runtime Watch checkpoint key is required.");
|
|
72
|
+
return join(this.directory, `${createHash("sha256").update(key).digest("hex")}.json`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export class ManagedFailureRuntimeWatch {
|
|
76
|
+
#client;
|
|
77
|
+
#checkpoints;
|
|
78
|
+
#intervalMs;
|
|
79
|
+
#now;
|
|
80
|
+
#readOutcome;
|
|
81
|
+
#bindings;
|
|
82
|
+
#activeTick;
|
|
83
|
+
constructor(options) {
|
|
84
|
+
this.#client = options.client;
|
|
85
|
+
this.#checkpoints = options.checkpoints;
|
|
86
|
+
this.#intervalMs = options.intervalMs ?? 5_000;
|
|
87
|
+
this.#now = options.now ?? (() => new Date());
|
|
88
|
+
this.#readOutcome = options.readOutcome;
|
|
89
|
+
this.#bindings = options.bindings ? new Set(options.bindings.map(bindingKey)) : undefined;
|
|
90
|
+
if (!Number.isSafeInteger(this.#intervalMs) || this.#intervalMs < 1_000 || this.#intervalMs > 60_000) {
|
|
91
|
+
throw new Error("Failure Runtime Watch intervalMs must be between 1000 and 60000.");
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
tick() {
|
|
95
|
+
if (this.#activeTick)
|
|
96
|
+
return this.#activeTick;
|
|
97
|
+
this.#activeTick = this.runTick().finally(() => { this.#activeTick = undefined; });
|
|
98
|
+
return this.#activeTick;
|
|
99
|
+
}
|
|
100
|
+
async runTick() {
|
|
101
|
+
const failureCases = await this.#client.listFailureCases();
|
|
102
|
+
const candidates = failureCases.filter((failureCase) => failureCase.definition.environment === "sandbox"
|
|
103
|
+
&& (!this.#bindings || this.#bindings.has(bindingKey(failureCase.definition)))
|
|
104
|
+
&& ["FIX_VERIFIED", "RUNTIME_PROTECTED", "REGRESSION_DETECTED"].includes(failureCase.status));
|
|
105
|
+
if (!candidates.length)
|
|
106
|
+
return { casesInspected: 0, observationsUploaded: 0, outcomesDeferred: 0, limitations: [] };
|
|
107
|
+
let actions;
|
|
108
|
+
let receipts;
|
|
109
|
+
try {
|
|
110
|
+
actions = await this.#client.listActions();
|
|
111
|
+
const candidateBindings = new Set(candidates.map((failureCase) => bindingKey(failureCase.definition)));
|
|
112
|
+
receipts = await this.#client.listReceipts(actions.filter((action) => {
|
|
113
|
+
const binding = action.businessTaskBinding;
|
|
114
|
+
return Boolean(binding?.taskContractId && binding.actionPathId && binding.environment
|
|
115
|
+
&& candidateBindings.has(bindingKey({
|
|
116
|
+
taskContractId: binding.taskContractId,
|
|
117
|
+
actionPathId: binding.actionPathId,
|
|
118
|
+
environment: binding.environment,
|
|
119
|
+
})));
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
const detail = "Runtime Watch could not load new Action and Receipt metadata from Witnora.";
|
|
124
|
+
await Promise.all(candidates.map((failureCase) => {
|
|
125
|
+
const checkedAt = this.#now().toISOString();
|
|
126
|
+
return this.#client.updateStatus(failureCase.id, {
|
|
127
|
+
health: "CONNECTION_UNAVAILABLE",
|
|
128
|
+
lastCheckedAt: checkedAt,
|
|
129
|
+
nextCheckAt: new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString(),
|
|
130
|
+
detail,
|
|
131
|
+
});
|
|
132
|
+
}));
|
|
133
|
+
return { casesInspected: candidates.length, observationsUploaded: 0, outcomesDeferred: candidates.length, limitations: [detail] };
|
|
134
|
+
}
|
|
135
|
+
let eligible = candidates;
|
|
136
|
+
if (!this.#bindings) {
|
|
137
|
+
const discoveredBindings = coveredBindingKeys(actions, receipts);
|
|
138
|
+
if (discoveredBindings.size !== 1) {
|
|
139
|
+
const detail = "Runtime Watch requires one exact sandbox Task and action path with an active enforced Receipt.";
|
|
140
|
+
await Promise.all(candidates.map((failureCase) => {
|
|
141
|
+
const checkedAt = this.#now().toISOString();
|
|
142
|
+
return this.#client.updateStatus(failureCase.id, {
|
|
143
|
+
health: "INSUFFICIENT_EVIDENCE",
|
|
144
|
+
lastCheckedAt: checkedAt,
|
|
145
|
+
nextCheckAt: new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString(),
|
|
146
|
+
detail,
|
|
147
|
+
});
|
|
148
|
+
}));
|
|
149
|
+
return { casesInspected: candidates.length, observationsUploaded: 0, outcomesDeferred: candidates.length, limitations: [detail] };
|
|
150
|
+
}
|
|
151
|
+
const discoveredBinding = [...discoveredBindings][0];
|
|
152
|
+
eligible = candidates.filter((failureCase) => bindingKey(failureCase.definition) === discoveredBinding);
|
|
153
|
+
}
|
|
154
|
+
let observationsUploaded = 0;
|
|
155
|
+
let outcomesDeferred = 0;
|
|
156
|
+
const limitations = [];
|
|
157
|
+
for (const failureCase of eligible) {
|
|
158
|
+
const checkedAt = this.#now().toISOString();
|
|
159
|
+
const nextCheckAt = new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString();
|
|
160
|
+
const storedCheckpoint = await this.#checkpoints.load(failureCase.id);
|
|
161
|
+
const checkpoint = storedCheckpoint ?? {
|
|
162
|
+
processedReceiptIds: failureCase.runtimeWatch?.receiptIds ?? failureCase.runtimeWatch?.latest?.receiptIds ?? [],
|
|
163
|
+
updatedAt: checkedAt,
|
|
164
|
+
};
|
|
165
|
+
const covered = coveredOutcomes(failureCase, actions, receipts);
|
|
166
|
+
const pending = covered.filter(({ receipt }) => !checkpoint.processedReceiptIds.includes(receipt.id));
|
|
167
|
+
if (!pending.length) {
|
|
168
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
169
|
+
health: failureCase.status === "REGRESSION_DETECTED" ? "REGRESSION" : "HEALTHY",
|
|
170
|
+
lastCheckedAt: checkedAt,
|
|
171
|
+
nextCheckAt,
|
|
172
|
+
});
|
|
173
|
+
if (!storedCheckpoint)
|
|
174
|
+
await this.#checkpoints.save(failureCase.id, checkpoint);
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const resourceId = pending[0].resourceId;
|
|
178
|
+
const group = covered.filter((item) => item.resourceId === resourceId);
|
|
179
|
+
const outcome = await this.#readOutcome({
|
|
180
|
+
failureCase,
|
|
181
|
+
resourceId,
|
|
182
|
+
actions: group.map((item) => item.action),
|
|
183
|
+
receipts: group.map((item) => item.receipt),
|
|
184
|
+
});
|
|
185
|
+
if (outcome.classification !== "HEALTHY" && outcome.classification !== "REGRESSION") {
|
|
186
|
+
outcomesDeferred += 1;
|
|
187
|
+
limitations.push(outcome.reason ?? `Failure Runtime Watch deferred ${failureCase.id}: ${outcome.classification}.`);
|
|
188
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
189
|
+
health: outcome.classification === "SCHEMA_DIGEST_DRIFT" ? "INSUFFICIENT_EVIDENCE" : outcome.classification,
|
|
190
|
+
lastCheckedAt: checkedAt,
|
|
191
|
+
nextCheckAt,
|
|
192
|
+
...(outcome.reason ? { detail: outcome.reason } : {}),
|
|
193
|
+
});
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const receiptIds = group.map((item) => item.receipt.id);
|
|
197
|
+
const agentVersion = requiredAgentVersion(failureCase, group.map((item) => item.action));
|
|
198
|
+
await this.#client.uploadObservation({
|
|
199
|
+
discoveryKey: failureCase.discovery?.discoveryKey ?? failureCase.id,
|
|
200
|
+
phase: "RUNTIME",
|
|
201
|
+
observedAt: outcome.observedAt,
|
|
202
|
+
definition: structuredClone(failureCase.definition),
|
|
203
|
+
result: {
|
|
204
|
+
agentVersion,
|
|
205
|
+
attempts: outcome.attempts ?? group.length,
|
|
206
|
+
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
207
|
+
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
208
|
+
failureCount: outcome.failureCount ?? 0,
|
|
209
|
+
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
210
|
+
observation: "RECORDED",
|
|
211
|
+
executionControl: "ENFORCED",
|
|
212
|
+
outcome: "VERIFIED",
|
|
213
|
+
review: "NOT_REVIEWED",
|
|
214
|
+
receiptIds,
|
|
215
|
+
},
|
|
216
|
+
evidence: {
|
|
217
|
+
summary: outcome.classification === "REGRESSION"
|
|
218
|
+
? "A covered sandbox result reproduced the defined failure after the candidate was verified."
|
|
219
|
+
: "A covered sandbox result remained healthy after the candidate was verified.",
|
|
220
|
+
measurements: {
|
|
221
|
+
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
222
|
+
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
223
|
+
failureCount: outcome.failureCount ?? 0,
|
|
224
|
+
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
225
|
+
receiptCount: receiptIds.length,
|
|
226
|
+
},
|
|
227
|
+
limitations: ["This automated observation covers the exact customer-owned sandbox task and action path only; it does not establish production reliability."],
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
observationsUploaded += 1;
|
|
231
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
232
|
+
health: outcome.classification === "REGRESSION" ? "REGRESSION" : "HEALTHY",
|
|
233
|
+
lastCheckedAt: checkedAt,
|
|
234
|
+
nextCheckAt,
|
|
235
|
+
});
|
|
236
|
+
await this.#checkpoints.save(failureCase.id, {
|
|
237
|
+
processedReceiptIds: unique([...checkpoint.processedReceiptIds, ...receiptIds]),
|
|
238
|
+
updatedAt: checkedAt,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
return { casesInspected: eligible.length, observationsUploaded, outcomesDeferred, limitations };
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
const MAX_LOCAL_SANDBOX_AUDIT_BYTES = 10 * 1024 * 1024;
|
|
245
|
+
/**
|
|
246
|
+
* Reads only the local sandbox result fields required by Runtime Watch. Raw
|
|
247
|
+
* provider payloads and unbound audit records never leave the customer Harness.
|
|
248
|
+
*/
|
|
249
|
+
export async function readLocalSandboxRuntimeWatchOutcome(input) {
|
|
250
|
+
const observedAt = () => (input.now?.() ?? new Date()).toISOString();
|
|
251
|
+
const attempts = input.actions.length;
|
|
252
|
+
let source;
|
|
253
|
+
try {
|
|
254
|
+
const metadata = await stat(input.path);
|
|
255
|
+
if (!metadata.isFile() || metadata.size > MAX_LOCAL_SANDBOX_AUDIT_BYTES) {
|
|
256
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "The local sandbox audit is not a bounded readable file.");
|
|
257
|
+
}
|
|
258
|
+
source = await readFile(input.path, "utf8");
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
const code = error.code;
|
|
262
|
+
if (code === "ENOENT" || code === "EACCES" || code === "EPERM") {
|
|
263
|
+
return deferred("CONNECTION_UNAVAILABLE", attempts, observedAt(), "The local sandbox audit is unavailable to Runtime Watch.");
|
|
264
|
+
}
|
|
265
|
+
throw error;
|
|
266
|
+
}
|
|
267
|
+
const expected = new Map(input.actions.map((action) => [action.id, action]));
|
|
268
|
+
if (expected.size !== input.actions.length || !input.resourceId || [...expected.values()].some((action) => !action.id)) {
|
|
269
|
+
return deferred("INSUFFICIENT_EVIDENCE", attempts, observedAt(), "The covered Action bindings are incomplete.");
|
|
270
|
+
}
|
|
271
|
+
const commits = new Map();
|
|
272
|
+
for (const line of source.split(/\r?\n/u)) {
|
|
273
|
+
if (!line.trim())
|
|
274
|
+
continue;
|
|
275
|
+
let raw;
|
|
276
|
+
try {
|
|
277
|
+
raw = JSON.parse(line);
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "The local sandbox audit contains invalid JSON.");
|
|
281
|
+
}
|
|
282
|
+
if (!isRecord(raw) || raw.phase !== "COMMITTED")
|
|
283
|
+
continue;
|
|
284
|
+
const record = allowlistedCommittedAuditRecord(raw);
|
|
285
|
+
if (!record) {
|
|
286
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "A committed local sandbox result is missing an allowlisted field.");
|
|
287
|
+
}
|
|
288
|
+
const expectedAction = expected.get(record.actionId);
|
|
289
|
+
if (record.resourceId !== input.resourceId || !expectedAction)
|
|
290
|
+
continue;
|
|
291
|
+
if (expectedAction.executionSessionId && expectedAction.executionSessionId !== record.executionSessionId)
|
|
292
|
+
continue;
|
|
293
|
+
const digest = createHash("sha256").update(JSON.stringify(record.observedState)).digest("hex");
|
|
294
|
+
if (digest !== record.stateSha256) {
|
|
295
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "A covered local sandbox result failed its state digest check.");
|
|
296
|
+
}
|
|
297
|
+
const existing = commits.get(record.actionId);
|
|
298
|
+
if (existing && existing.executionSessionId !== record.executionSessionId) {
|
|
299
|
+
return deferred("INSUFFICIENT_EVIDENCE", attempts, observedAt(), "A covered Action has more than one execution-session result.");
|
|
300
|
+
}
|
|
301
|
+
if (!existing || record.at > existing.at)
|
|
302
|
+
commits.set(record.actionId, { at: record.at, executionSessionId: record.executionSessionId });
|
|
303
|
+
}
|
|
304
|
+
if (commits.size !== expected.size) {
|
|
305
|
+
return deferred("INSUFFICIENT_EVIDENCE", attempts, observedAt(), "Not every covered Action has a digest-verified committed result.");
|
|
306
|
+
}
|
|
307
|
+
const latest = [...commits.values()].map((commit) => commit.at).sort().at(-1);
|
|
308
|
+
const duplicateOutcomeCount = Math.max(0, commits.size - 1);
|
|
309
|
+
return {
|
|
310
|
+
classification: duplicateOutcomeCount > 0 ? "REGRESSION" : "HEALTHY",
|
|
311
|
+
attempts,
|
|
312
|
+
providerCommitCount: commits.size,
|
|
313
|
+
duplicateOutcomeCount,
|
|
314
|
+
failureCount: duplicateOutcomeCount > 0 ? 1 : 0,
|
|
315
|
+
hardInvariantViolations: duplicateOutcomeCount > 0 ? 1 : 0,
|
|
316
|
+
observedAt: latest,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
function allowlistedCommittedAuditRecord(value) {
|
|
320
|
+
if (typeof value.resourceId !== "string"
|
|
321
|
+
|| typeof value.actionId !== "string"
|
|
322
|
+
|| typeof value.executionSessionId !== "string"
|
|
323
|
+
|| !isRecord(value.observedState)
|
|
324
|
+
|| typeof value.stateSha256 !== "string"
|
|
325
|
+
|| !/^[a-f0-9]{64}$/u.test(value.stateSha256)
|
|
326
|
+
|| typeof value.at !== "string"
|
|
327
|
+
|| Number.isNaN(Date.parse(value.at)))
|
|
328
|
+
return undefined;
|
|
329
|
+
return {
|
|
330
|
+
resourceId: value.resourceId,
|
|
331
|
+
actionId: value.actionId,
|
|
332
|
+
executionSessionId: value.executionSessionId,
|
|
333
|
+
observedState: value.observedState,
|
|
334
|
+
stateSha256: value.stateSha256,
|
|
335
|
+
at: new Date(value.at).toISOString(),
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
function deferred(classification, attempts, observedAt, reason) {
|
|
339
|
+
return { classification, attempts, observedAt, reason };
|
|
340
|
+
}
|
|
341
|
+
function isRecord(value) {
|
|
342
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
343
|
+
}
|
|
344
|
+
function coveredOutcomes(failureCase, actions, receipts) {
|
|
345
|
+
const watchStartedAt = timestamp(failureCase.runtimeWatch?.activatedAt
|
|
346
|
+
?? failureCase.candidate?.recordedAt
|
|
347
|
+
?? failureCase.updatedAt);
|
|
348
|
+
const receiptsByAction = new Map(receipts.filter((receipt) => {
|
|
349
|
+
const receiptCreatedAt = timestamp(receipt.createdAt);
|
|
350
|
+
return eligibleReceipt(receipt)
|
|
351
|
+
&& (!watchStartedAt || (receiptCreatedAt !== undefined && receiptCreatedAt >= watchStartedAt));
|
|
352
|
+
})
|
|
353
|
+
.map((receipt) => [receipt.actionId, receipt]));
|
|
354
|
+
return actions.flatMap((action) => {
|
|
355
|
+
const binding = action.businessTaskBinding;
|
|
356
|
+
const receipt = receiptsByAction.get(action.id);
|
|
357
|
+
const resourceId = typeof action.expectedState?.resourceId === "string" ? action.expectedState.resourceId : undefined;
|
|
358
|
+
return receipt && action.status === "VERIFIED" && resourceId
|
|
359
|
+
&& binding?.taskContractId === failureCase.definition.taskContractId
|
|
360
|
+
&& binding.actionPathId === failureCase.definition.actionPathId
|
|
361
|
+
&& binding.environment === failureCase.definition.environment
|
|
362
|
+
? [{ action, receipt, resourceId }]
|
|
363
|
+
: [];
|
|
364
|
+
}).sort((left, right) => left.receipt.createdAt.localeCompare(right.receipt.createdAt));
|
|
365
|
+
}
|
|
366
|
+
function coveredBindingKeys(actions, receipts) {
|
|
367
|
+
const coveredActionIds = new Set(receipts.filter(eligibleReceipt)
|
|
368
|
+
.map((receipt) => receipt.actionId));
|
|
369
|
+
return new Set(actions.flatMap((action) => {
|
|
370
|
+
const binding = action.businessTaskBinding;
|
|
371
|
+
return action.status === "VERIFIED" && coveredActionIds.has(action.id)
|
|
372
|
+
&& binding?.environment === "sandbox" && binding.taskContractId && binding.actionPathId
|
|
373
|
+
? [bindingKey({ taskContractId: binding.taskContractId, actionPathId: binding.actionPathId, environment: binding.environment })]
|
|
374
|
+
: [];
|
|
375
|
+
}));
|
|
376
|
+
}
|
|
377
|
+
function requiredAgentVersion(failureCase, actions) {
|
|
378
|
+
const versions = new Set(actions.map((action) => action.businessTaskBinding?.agentVersion ?? failureCase.candidate?.agentVersion).filter(Boolean));
|
|
379
|
+
if (versions.size !== 1)
|
|
380
|
+
throw new Error("Failure Runtime Watch requires one exact Agent version.");
|
|
381
|
+
return [...versions][0];
|
|
382
|
+
}
|
|
383
|
+
function unique(values) { return [...new Set(values)]; }
|
|
384
|
+
function bindingKey(value) {
|
|
385
|
+
return `${value.taskContractId}\0${value.actionPathId}\0${value.environment}`;
|
|
386
|
+
}
|
|
387
|
+
function eligibleReceipt(receipt) {
|
|
388
|
+
return receipt.discoveryBoundary === "SIGNED_VERIFIED_ACTION"
|
|
389
|
+
|| (receipt.currentStatus === "ACTIVE"
|
|
390
|
+
&& receipt.receipt.core.enforcementLevel === "ENFORCED"
|
|
391
|
+
&& (receipt.receipt.core.evidenceStrength === "OUTCOME_VERIFIED"
|
|
392
|
+
|| receipt.receipt.core.evidenceStrength === "INDEPENDENTLY_REVIEWED"));
|
|
393
|
+
}
|
|
394
|
+
function signedVerifiedActionReceipt(value) {
|
|
395
|
+
const receiptId = value.receiptId;
|
|
396
|
+
const actionId = value.id;
|
|
397
|
+
const signatureCount = value.receiptSignatureCount;
|
|
398
|
+
const createdAt = typeof value.updatedAt === "string" ? value.updatedAt : value.createdAt;
|
|
399
|
+
if (value.status !== "VERIFIED"
|
|
400
|
+
|| value.verificationSuccess !== true
|
|
401
|
+
|| typeof receiptId !== "string" || !receiptId
|
|
402
|
+
|| typeof actionId !== "string" || !actionId
|
|
403
|
+
|| !Number.isSafeInteger(signatureCount) || signatureCount < 1
|
|
404
|
+
|| typeof createdAt !== "string" || Number.isNaN(Date.parse(createdAt)))
|
|
405
|
+
return [];
|
|
406
|
+
return [{
|
|
407
|
+
id: receiptId,
|
|
408
|
+
actionId,
|
|
409
|
+
currentStatus: "DISCOVERED",
|
|
410
|
+
createdAt: new Date(createdAt).toISOString(),
|
|
411
|
+
receipt: { core: {} },
|
|
412
|
+
discoveryBoundary: "SIGNED_VERIFIED_ACTION",
|
|
413
|
+
}];
|
|
414
|
+
}
|
|
415
|
+
function timestamp(value) {
|
|
416
|
+
if (!value)
|
|
417
|
+
return undefined;
|
|
418
|
+
const parsed = Date.parse(value);
|
|
419
|
+
return Number.isNaN(parsed) ? undefined : parsed;
|
|
420
|
+
}
|
|
421
|
+
async function hostedJson(requestFetch, url, apiKey, init = {}) {
|
|
422
|
+
const headers = new Headers(init.headers);
|
|
423
|
+
headers.set("authorization", `Bearer ${apiKey}`);
|
|
424
|
+
if (typeof init.body === "string")
|
|
425
|
+
headers.set("content-type", "application/json");
|
|
426
|
+
const response = await requestFetch(url, { ...init, headers, redirect: "error" });
|
|
427
|
+
const value = await response.json().catch(() => ({}));
|
|
428
|
+
if (!response.ok)
|
|
429
|
+
throw new Error(typeof value.error === "string" ? value.error : `Witnora Control Plane request failed (${response.status}).`);
|
|
430
|
+
return value;
|
|
431
|
+
}
|
|
432
|
+
function list(value, field) {
|
|
433
|
+
const items = value[field];
|
|
434
|
+
if (!Array.isArray(items) || items.length > 1_000)
|
|
435
|
+
throw new Error(`Witnora returned an invalid ${field} list.`);
|
|
436
|
+
return items;
|
|
437
|
+
}
|
|
438
|
+
function required(value, name) {
|
|
439
|
+
const normalized = value.trim();
|
|
440
|
+
if (!normalized)
|
|
441
|
+
throw new Error(`Failure Runtime Watch ${name} is required.`);
|
|
442
|
+
return normalized;
|
|
443
|
+
}
|
|
444
|
+
function validateCheckpoint(value) {
|
|
445
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
446
|
+
throw new Error("Failure Runtime Watch checkpoint is invalid.");
|
|
447
|
+
const checkpoint = value;
|
|
448
|
+
if (!Array.isArray(checkpoint.processedReceiptIds)
|
|
449
|
+
|| checkpoint.processedReceiptIds.length > 1_000
|
|
450
|
+
|| checkpoint.processedReceiptIds.some((id) => typeof id !== "string" || !id || id.length > 200)
|
|
451
|
+
|| new Set(checkpoint.processedReceiptIds).size !== checkpoint.processedReceiptIds.length
|
|
452
|
+
|| typeof checkpoint.updatedAt !== "string"
|
|
453
|
+
|| Number.isNaN(Date.parse(checkpoint.updatedAt))) {
|
|
454
|
+
throw new Error("Failure Runtime Watch checkpoint is invalid.");
|
|
455
|
+
}
|
|
456
|
+
return { processedReceiptIds: [...checkpoint.processedReceiptIds], updatedAt: new Date(checkpoint.updatedAt).toISOString() };
|
|
457
|
+
}
|