witnora 0.18.11 → 0.18.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gateway-service.js +5 -0
- package/dist/gateway.js +77 -7
- package/dist/onboard.js +63 -17
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.d.ts +65 -0
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.d.ts.map +1 -0
- package/dist/vendor/onegent-runtime/failure-experiment-hosted.js +146 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts +138 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.d.ts.map +1 -0
- package/dist/vendor/onegent-runtime/failure-runtime-watch.js +387 -0
- package/package.json +1 -1
package/dist/gateway-service.js
CHANGED
|
@@ -53,6 +53,11 @@ export async function restartCurrentGatewayService(options) {
|
|
|
53
53
|
await run(plan.start.command, plan.start.args);
|
|
54
54
|
return plan;
|
|
55
55
|
}
|
|
56
|
+
export async function stopCurrentGatewayService(options) {
|
|
57
|
+
const plan = createGatewayServicePlan(currentPlanInput(options));
|
|
58
|
+
await (options.run ?? runCommand)(plan.stop.command, plan.stop.args).catch(() => undefined);
|
|
59
|
+
return plan;
|
|
60
|
+
}
|
|
56
61
|
export function createGatewayServicePlan(input) {
|
|
57
62
|
const suffix = createHash("sha256").update(`${input.repository}\n${input.gatewayDirectory}`).digest("hex").slice(0, 12);
|
|
58
63
|
const id = `witnora-gateway-${suffix}`;
|
package/dist/gateway.js
CHANGED
|
@@ -61,6 +61,7 @@ export async function initializeCustomerGateway(options) {
|
|
|
61
61
|
port: gatewayPort(process.env.WITNORA_GATEWAY_PORT, 8787),
|
|
62
62
|
storageDirectory: "data",
|
|
63
63
|
privacyMode: "metadata_only",
|
|
64
|
+
...(options.agentIdentity ? { agentIdentity: options.agentIdentity } : {}),
|
|
64
65
|
coverage: runtimeKit
|
|
65
66
|
? { recorded: "configured", enforced: "configured", outcomeVerified: "configured" }
|
|
66
67
|
: { recorded: "configured", enforced: "not_configured", outcomeVerified: "not_configured" },
|
|
@@ -136,14 +137,20 @@ export async function upgradeCustomerGatewayRuntime(options) {
|
|
|
136
137
|
if (current.projectId !== options.authorization.projectId || current.server !== options.authorization.server)
|
|
137
138
|
throw new Error("Existing Gateway project/server binding does not match this authorized Runtime upgrade.");
|
|
138
139
|
const generatedClient = clientRaw === gatewayClient(current)
|
|
139
|
-
|| (!current.runtimeWorker && clientRaw === recordedGatewayClient(current)
|
|
140
|
+
|| (!current.runtimeWorker && (clientRaw === recordedGatewayClient(current)
|
|
141
|
+
|| clientRaw === recordedGatewayClient({ ...current, agentIdentity: undefined })));
|
|
140
142
|
if (!generatedClient || readmeRaw !== gatewayReadme(current))
|
|
141
143
|
throw new Error("Existing Gateway files differ from the Witnora-generated version; refusing to overwrite them.");
|
|
142
144
|
if (current.runtimeWorker && current.runtimeWorker.adapterId !== LOCAL_SANDBOX_ADAPTER_ID) {
|
|
143
145
|
throw new Error("Existing Gateway already has a different Runtime worker; preserve it and reconfigure in Advanced mode.");
|
|
144
146
|
}
|
|
145
147
|
const runtimeKit = await prepareRuntimeSandboxKit(options.runtimeReferences, options.authorization, options.configHome, options.fetch ?? fetch, directory);
|
|
146
|
-
const next = {
|
|
148
|
+
const next = {
|
|
149
|
+
...current,
|
|
150
|
+
...(options.agentIdentity ? { agentIdentity: options.agentIdentity } : {}),
|
|
151
|
+
coverage: { recorded: "configured", enforced: "configured", outcomeVerified: "configured" },
|
|
152
|
+
runtimeWorker: runtimeKit.config,
|
|
153
|
+
};
|
|
147
154
|
next.runtimeWorker.configDigestSha256 = gatewayConfigDigest(next);
|
|
148
155
|
const replacingGeneratedRuntime = current.runtimeWorker?.adapterId === LOCAL_SANDBOX_ADAPTER_ID;
|
|
149
156
|
if (replacingGeneratedRuntime) {
|
|
@@ -155,7 +162,8 @@ export async function upgradeCustomerGatewayRuntime(options) {
|
|
|
155
162
|
|| createHash("sha256").update(probeBytes).digest("hex") !== current.runtimeWorker.probeModuleSha256) {
|
|
156
163
|
throw new Error("Existing generated Runtime modules were modified; refusing to overwrite them.");
|
|
157
164
|
}
|
|
158
|
-
if (sameRuntimeGeneration(current.runtimeWorker, next.runtimeWorker)
|
|
165
|
+
if (sameRuntimeGeneration(current.runtimeWorker, next.runtimeWorker)
|
|
166
|
+
&& JSON.stringify(current.agentIdentity) === JSON.stringify(next.agentIdentity)) {
|
|
159
167
|
return { config: current, generatedFiles: [], changed: false, rollback: async () => undefined };
|
|
160
168
|
}
|
|
161
169
|
}
|
|
@@ -601,8 +609,12 @@ export async function runCustomerGateway(options) {
|
|
|
601
609
|
throw error;
|
|
602
610
|
}
|
|
603
611
|
}
|
|
604
|
-
function configManagedWorkflowHarnessImport() {
|
|
605
|
-
|
|
612
|
+
async function configManagedWorkflowHarnessImport() {
|
|
613
|
+
const [managed, runtimeWatch] = await Promise.all([
|
|
614
|
+
import(new URL("./vendor/onegent-runtime/managed-workflow-harness.js", import.meta.url).href),
|
|
615
|
+
import(new URL("./vendor/onegent-runtime/failure-runtime-watch.js", import.meta.url).href),
|
|
616
|
+
]);
|
|
617
|
+
return { ...managed, ...runtimeWatch };
|
|
606
618
|
}
|
|
607
619
|
function configBusinessTaskEvaluatorImport() {
|
|
608
620
|
return import(new URL("./vendor/onegent-runtime/business-task-evaluator.js", import.meta.url).href);
|
|
@@ -663,6 +675,43 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
663
675
|
return body.report;
|
|
664
676
|
},
|
|
665
677
|
});
|
|
678
|
+
const runtimeWatchConfiguration = input.config.schemaVersion === MANAGED_WORKFLOW_HARNESS_SCHEMA
|
|
679
|
+
&& input.runtimeWorker?.adapterId === LOCAL_SANDBOX_ADAPTER_ID
|
|
680
|
+
&& (!input.config.realPathActivations || (input.config.realPathActivations.length === 1
|
|
681
|
+
&& input.config.realPathActivations[0]?.environment === "sandbox"
|
|
682
|
+
&& input.config.realPathActivations[0].actionPathIds.length === 1));
|
|
683
|
+
const activation = runtimeWatchConfiguration
|
|
684
|
+
&& input.config.schemaVersion === MANAGED_WORKFLOW_HARNESS_SCHEMA
|
|
685
|
+
&& input.config.realPathActivations?.length === 1
|
|
686
|
+
&& input.config.realPathActivations[0]?.environment === "sandbox"
|
|
687
|
+
&& input.config.realPathActivations[0].actionPathIds.length === 1
|
|
688
|
+
? input.config.realPathActivations[0]
|
|
689
|
+
: undefined;
|
|
690
|
+
const runtimeWatch = runtimeWatchConfiguration
|
|
691
|
+
&& input.managed.ManagedFailureRuntimeWatch
|
|
692
|
+
&& input.managed.FileFailureRuntimeWatchCheckpointStore
|
|
693
|
+
&& input.managed.createFailureRuntimeWatchHostedClient
|
|
694
|
+
&& input.managed.readLocalSandboxRuntimeWatchOutcome
|
|
695
|
+
? new input.managed.ManagedFailureRuntimeWatch({
|
|
696
|
+
client: input.managed.createFailureRuntimeWatchHostedClient({
|
|
697
|
+
baseUrl: input.server,
|
|
698
|
+
projectId: input.projectId,
|
|
699
|
+
apiKey: input.apiKey,
|
|
700
|
+
fetch: requestFetch,
|
|
701
|
+
}),
|
|
702
|
+
checkpoints: new input.managed.FileFailureRuntimeWatchCheckpointStore(join(input.directory, "data", "failure-runtime-watch")),
|
|
703
|
+
intervalMs: input.config.pollIntervalMs ?? 5_000,
|
|
704
|
+
...(activation ? { bindings: [{ taskContractId: activation.taskContractId, actionPathId: activation.actionPathIds[0], environment: "sandbox" }] } : {}),
|
|
705
|
+
readOutcome: async ({ resourceId, actions, receipts }) => input.managed.readLocalSandboxRuntimeWatchOutcome({
|
|
706
|
+
path: join(input.directory, "data", "runtime-sandbox", "fixture", "audit.jsonl"),
|
|
707
|
+
resourceId,
|
|
708
|
+
actions: actions.map((action) => ({
|
|
709
|
+
id: action.id,
|
|
710
|
+
executionSessionId: receipts.find((receipt) => receipt.actionId === action.id)?.receipt?.core?.executionSessionId ?? "",
|
|
711
|
+
})),
|
|
712
|
+
}),
|
|
713
|
+
})
|
|
714
|
+
: undefined;
|
|
666
715
|
let timer;
|
|
667
716
|
let closing = false;
|
|
668
717
|
let lastTickAt;
|
|
@@ -675,12 +724,19 @@ export async function createConfiguredWorkflowHarness(input) {
|
|
|
675
724
|
await managedEvaluator.healthCheck();
|
|
676
725
|
}
|
|
677
726
|
const result = await worker.tick();
|
|
727
|
+
const watchResult = runtimeWatch ? await runtimeWatch.tick() : undefined;
|
|
678
728
|
lastTickAt = new Date().toISOString();
|
|
679
729
|
lastError = (result.evaluationsFailed ?? 0) > 0 ? result.limitations?.[0] ?? "One or more evaluations failed closed." : undefined;
|
|
680
730
|
if ((result.evaluationsFailed ?? 0) > 0) {
|
|
681
731
|
process.stderr.write(`Managed Workflow Harness stopped ${result.evaluationsFailed} evaluation(s) fail-closed. ${lastError}\n`);
|
|
682
732
|
}
|
|
683
|
-
return
|
|
733
|
+
return watchResult ? {
|
|
734
|
+
...result,
|
|
735
|
+
runtimeWatchCasesInspected: watchResult.casesInspected ?? 0,
|
|
736
|
+
runtimeWatchObservationsUploaded: watchResult.observationsUploaded ?? 0,
|
|
737
|
+
runtimeWatchOutcomesDeferred: watchResult.outcomesDeferred ?? 0,
|
|
738
|
+
runtimeWatchLimitations: watchResult.limitations ?? [],
|
|
739
|
+
} : result;
|
|
684
740
|
}
|
|
685
741
|
catch (error) {
|
|
686
742
|
lastTickAt = new Date().toISOString();
|
|
@@ -743,6 +799,7 @@ export async function createContinuousAssuranceController(input) {
|
|
|
743
799
|
server: input.server,
|
|
744
800
|
apiKey: input.apiKey,
|
|
745
801
|
config,
|
|
802
|
+
runtimeWorker: input.config.runtimeWorker,
|
|
746
803
|
managed,
|
|
747
804
|
evaluatorKit: input.evaluatorKit,
|
|
748
805
|
fetch: request,
|
|
@@ -1358,6 +1415,8 @@ function parseConfig(raw) {
|
|
|
1358
1415
|
}
|
|
1359
1416
|
if (value.privacyMode !== "metadata_only")
|
|
1360
1417
|
throw new Error("Gateway privacyMode must remain metadata_only.");
|
|
1418
|
+
if (value.agentIdentity)
|
|
1419
|
+
validateGatewayAgentIdentity(value.agentIdentity);
|
|
1361
1420
|
if (!value.host || !Number.isSafeInteger(value.port) || Number(value.port) < 1 || Number(value.port) > 65_535)
|
|
1362
1421
|
throw new Error("Gateway host and port are invalid.");
|
|
1363
1422
|
if (value.runtimeWorker)
|
|
@@ -1368,6 +1427,13 @@ function parseConfig(raw) {
|
|
|
1368
1427
|
}
|
|
1369
1428
|
return config;
|
|
1370
1429
|
}
|
|
1430
|
+
function validateGatewayAgentIdentity(value) {
|
|
1431
|
+
for (const [key, item] of Object.entries(value)) {
|
|
1432
|
+
if (typeof item !== "string" || item.trim().length === 0 || item.length > 200) {
|
|
1433
|
+
throw new Error(`Gateway agentIdentity.${key} must be a non-empty string no longer than 200 characters.`);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1371
1437
|
export function parseManagedWorkflowHarnessConfig(raw) {
|
|
1372
1438
|
const value = JSON.parse(raw);
|
|
1373
1439
|
const workflowIds = value.workflowIds;
|
|
@@ -1652,7 +1718,11 @@ async function ed25519PublicKeyPem(handle) {
|
|
|
1652
1718
|
return createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
1653
1719
|
}
|
|
1654
1720
|
function recordedGatewayClient(config) {
|
|
1655
|
-
|
|
1721
|
+
const defaultAgent = config.agentIdentity ? JSON.stringify(config.agentIdentity) : undefined;
|
|
1722
|
+
const startPayload = defaultAgent
|
|
1723
|
+
? `const payload = { ...metadata, agent: metadata.agent ?? ${defaultAgent} };\n return post(runId, "start", { payload, idempotencyKey: "run-start" });`
|
|
1724
|
+
: `return post(runId, "start", { payload: metadata, idempotencyKey: "run-start" });`;
|
|
1725
|
+
return `import { readFile } from "node:fs/promises";\n\nconst baseUrl = "http://${config.host}:${config.port}";\nlet gatewayToken;\n\nasync function token() {\n if (gatewayToken) return gatewayToken;\n const secrets = JSON.parse(await readFile(new URL("./secrets.json", import.meta.url), "utf8"));\n if (typeof secrets.gatewayToken !== "string" || secrets.gatewayToken.length < 32) {\n throw new Error("Witnora local Gateway token is missing or invalid.");\n }\n gatewayToken = secrets.gatewayToken;\n return gatewayToken;\n}\n\nasync function post(runId, operation, body) {\n if (!/^[A-Za-z0-9._:-]+$/.test(runId)) throw new Error("Witnora runId contains unsupported characters.");\n const response = await fetch(\`\${baseUrl}/v1/runs/\${encodeURIComponent(runId)}/\${operation}\`, {\n method: "POST",\n headers: { authorization: \`Bearer \${await token()}\`, "content-type": "application/json" },\n body: JSON.stringify(body),\n });\n const result = await response.json().catch(() => ({}));\n if (!response.ok) throw new Error(result.error ?? \`Witnora Gateway returned HTTP \${response.status}.\`);\n return result;\n}\n\nexport const witnoraGateway = {\n start(runId, metadata = {}) {\n ${startPayload}\n },\n event(runId, type, metadata = {}, idempotencyKey = \`\${type}-\${crypto.randomUUID()}\`) {\n return post(runId, "events", { type, payload: metadata, idempotencyKey });\n },\n complete(runId, metadata = {}) {\n return post(runId, "complete", {\n payload: metadata,\n evidenceStrength: {\n schemaVersion: "agentcert.evidence_strength.v0.1",\n level: "recorded",\n claims: [],\n limitations: ["No write-credential mediation or independent outcome probe is configured."],\n },\n idempotencyKey: "run-complete",\n });\n },\n};\n`;
|
|
1656
1726
|
}
|
|
1657
1727
|
async function writeExclusive(path, content, force, mode) {
|
|
1658
1728
|
if (!force && await exists(path))
|
package/dist/onboard.js
CHANGED
|
@@ -12,13 +12,19 @@ import { doctorCustomerGateway, activateManagedWorkflowHarness, ensureCustomerGa
|
|
|
12
12
|
import { discoverRuntimeReferences, planRuntimeSandboxModules } from "./runtime-sandbox-kit.js";
|
|
13
13
|
import { automaticRuntimeReferencesUsable, bootstrapLocalRuntime, verifyAutomaticRuntimeProbe } from "./runtime-bootstrap.js";
|
|
14
14
|
import { activateRealPathIntegrations } from "./real-path-activation.js";
|
|
15
|
-
import { installCurrentGatewayService } from "./gateway-service.js";
|
|
15
|
+
import { installCurrentGatewayService, stopCurrentGatewayService } from "./gateway-service.js";
|
|
16
16
|
export async function runOnboard(options) {
|
|
17
17
|
const requestFetch = options.fetch ?? fetch;
|
|
18
18
|
const output = options.output ?? ((message) => process.stdout.write(message));
|
|
19
19
|
const server = normalizeServer(options.server ?? DEFAULT_WITNORA_SERVER);
|
|
20
20
|
const repositoryPath = resolve(options.repository ?? process.cwd());
|
|
21
21
|
const repository = await inspectRepository(repositoryPath, options.template);
|
|
22
|
+
const agentIdentity = {
|
|
23
|
+
externalId: repository.name,
|
|
24
|
+
name: humanizeAgentName(repository.name),
|
|
25
|
+
version: repository.version,
|
|
26
|
+
framework: frameworkForTemplate(repository.template),
|
|
27
|
+
};
|
|
22
28
|
const connectionName = options.name ?? repository.slug;
|
|
23
29
|
const token = await authorizeProjectConnection({
|
|
24
30
|
projectId: options.projectId,
|
|
@@ -85,6 +91,19 @@ export async function runOnboard(options) {
|
|
|
85
91
|
start: startManagedCustomerGateway,
|
|
86
92
|
stop: stopManagedCustomerGateway,
|
|
87
93
|
};
|
|
94
|
+
const gatewayServiceLifecycle = options.gatewayServiceLifecycle ?? (!options.gatewayLifecycle ? {
|
|
95
|
+
stop: async ({ repository, configHome }) => {
|
|
96
|
+
await stopCurrentGatewayService({
|
|
97
|
+
repository,
|
|
98
|
+
configHome,
|
|
99
|
+
cliEntry: fileURLToPath(new URL("./cli.js", import.meta.url)),
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
} : undefined);
|
|
103
|
+
const stopGatewayOwners = async () => {
|
|
104
|
+
await gatewayServiceLifecycle?.stop({ repository: repositoryPath, configHome: options.configHome });
|
|
105
|
+
return gatewayLifecycle.stop({ repository: repositoryPath, configHome: options.configHome, fetch: requestFetch });
|
|
106
|
+
};
|
|
88
107
|
let gatewayMigration;
|
|
89
108
|
let managedGateway;
|
|
90
109
|
let runtimeUpgrade;
|
|
@@ -95,6 +114,24 @@ export async function runOnboard(options) {
|
|
|
95
114
|
};
|
|
96
115
|
let assuranceHarnessChanged = false;
|
|
97
116
|
let continuousService;
|
|
117
|
+
const prepareRuntime = async (operation) => {
|
|
118
|
+
const delays = [250, 500, 1_000, 2_000, 4_000, 8_000];
|
|
119
|
+
let lastError;
|
|
120
|
+
for (let attempt = 0; attempt <= delays.length; attempt += 1) {
|
|
121
|
+
try {
|
|
122
|
+
return await operation();
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
if (!(error instanceof RuntimeSetupNotReadyError))
|
|
126
|
+
throw error;
|
|
127
|
+
lastError = error;
|
|
128
|
+
if (attempt === delays.length)
|
|
129
|
+
break;
|
|
130
|
+
await (options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))))(delays[attempt]);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
throw lastError;
|
|
134
|
+
};
|
|
98
135
|
await reportInstall(requestFetch, server, token.projectId, token.apiKey, setupPlan.id, { status: "installing", attemptId });
|
|
99
136
|
try {
|
|
100
137
|
generatedFiles.push(...await generateRepositoryConfig(repositoryPath, repository.template, repository.name));
|
|
@@ -137,11 +174,12 @@ export async function runOnboard(options) {
|
|
|
137
174
|
if (gatewayState.status === "absent") {
|
|
138
175
|
let gateway;
|
|
139
176
|
try {
|
|
140
|
-
gateway = await initializeCustomerGateway({
|
|
177
|
+
gateway = await prepareRuntime(() => initializeCustomerGateway({
|
|
141
178
|
projectId: token.projectId, server, repository: repositoryPath, authorization: token,
|
|
142
179
|
runtimeReferences: localRuntime.references,
|
|
180
|
+
agentIdentity,
|
|
143
181
|
fetch: requestFetch, configHome: options.configHome, output,
|
|
144
|
-
});
|
|
182
|
+
}));
|
|
145
183
|
}
|
|
146
184
|
catch (error) {
|
|
147
185
|
if (!(error instanceof RuntimeSetupNotReadyError) || !localRuntime.references)
|
|
@@ -157,9 +195,9 @@ export async function runOnboard(options) {
|
|
|
157
195
|
await saveConnection(binding.connectionName, { server, projectId: token.projectId, apiKey: token.apiKey }, { configHome: options.configHome });
|
|
158
196
|
if (localRuntime.references) {
|
|
159
197
|
try {
|
|
160
|
-
runtimeUpgrade = await upgradeCustomerGatewayRuntime({ repository: repositoryPath, authorization: token, runtimeReferences: localRuntime.references, configHome: options.configHome, fetch: requestFetch });
|
|
198
|
+
runtimeUpgrade = await prepareRuntime(() => upgradeCustomerGatewayRuntime({ repository: repositoryPath, authorization: token, runtimeReferences: localRuntime.references, agentIdentity, configHome: options.configHome, fetch: requestFetch }));
|
|
161
199
|
if (runtimeUpgrade.changed) {
|
|
162
|
-
const stopped = await
|
|
200
|
+
const stopped = await stopGatewayOwners();
|
|
163
201
|
if (stopped.state !== "STOPPED" || stopped.healthy)
|
|
164
202
|
throw new Error("The prior managed Gateway process did not stop before Runtime generation replacement.");
|
|
165
203
|
}
|
|
@@ -180,18 +218,19 @@ export async function runOnboard(options) {
|
|
|
180
218
|
}
|
|
181
219
|
}
|
|
182
220
|
else {
|
|
183
|
-
const stopped = await
|
|
221
|
+
const stopped = await stopGatewayOwners();
|
|
184
222
|
if (stopped.state === "STOPPED" || stopped.state === "STALE") {
|
|
185
223
|
output(`\nStopped the managed Gateway for project ${binding.projectId} before rebinding this repository.\n`);
|
|
186
224
|
}
|
|
187
|
-
await assertGatewayStopped(requestFetch, binding.host, binding.port, binding.projectId, token.projectId);
|
|
225
|
+
await assertGatewayStopped(requestFetch, binding.host, binding.port, binding.collectorId, binding.projectId, token.projectId);
|
|
188
226
|
gatewayMigration = await archiveGateway(gatewayState.directory, repositoryPath, binding.projectId);
|
|
189
227
|
output(`\nExisting Gateway belongs to project ${binding.projectId}; archived it at ${gatewayMigration.archiveDirectory}.\n`);
|
|
190
|
-
const gateway = await initializeCustomerGateway({
|
|
228
|
+
const gateway = await prepareRuntime(() => initializeCustomerGateway({
|
|
191
229
|
projectId: token.projectId, server, repository: repositoryPath, authorization: token,
|
|
192
230
|
runtimeReferences: localRuntime.references,
|
|
231
|
+
agentIdentity,
|
|
193
232
|
fetch: requestFetch, configHome: options.configHome, output,
|
|
194
|
-
});
|
|
233
|
+
}));
|
|
195
234
|
generatedFiles.push(...gateway.generatedFiles);
|
|
196
235
|
}
|
|
197
236
|
}
|
|
@@ -213,14 +252,14 @@ export async function runOnboard(options) {
|
|
|
213
252
|
assuranceHarnessReadiness = { state: "WAITING_FOR_CUSTOMER_HARNESS", limitation: detail };
|
|
214
253
|
}
|
|
215
254
|
if (assuranceHarnessChanged && gatewayState.status !== "absent") {
|
|
216
|
-
const stopped = await
|
|
255
|
+
const stopped = await stopGatewayOwners();
|
|
217
256
|
if (stopped.state !== "STOPPED" && stopped.state !== "STALE")
|
|
218
257
|
throw new Error("The prior managed Gateway process did not stop before Assurance Harness activation.");
|
|
219
258
|
}
|
|
220
259
|
if (!options.gatewayLifecycle) {
|
|
221
260
|
// Stop any detached Gateway left by an earlier CLI before the OS-owned
|
|
222
261
|
// supervisor is installed. The service becomes the single process owner.
|
|
223
|
-
await
|
|
262
|
+
await stopGatewayOwners().catch(() => undefined);
|
|
224
263
|
const port = await ensureCustomerGatewayPortAvailable({ repository: repositoryPath });
|
|
225
264
|
if (port.changed)
|
|
226
265
|
output(`Rebound this repository's generated Gateway to available localhost port ${port.port}; another project remains untouched.\n`);
|
|
@@ -252,7 +291,7 @@ export async function runOnboard(options) {
|
|
|
252
291
|
: { state: "RECORDED_ONLY", checkedAt: new Date().toISOString(), limitations: [runtimeLimitation ?? "The local sandbox Runtime worker has not established exact adapter, probe, source-key, identity, mandate, and readiness bindings."] };
|
|
253
292
|
await reportInstall(requestFetch, server, token.projectId, token.apiKey, setupPlan.id, {
|
|
254
293
|
status: "verified", attemptId, generatedFiles: relativeGeneratedFiles(repositoryPath, generatedFiles), runtimeReadiness,
|
|
255
|
-
connectedAgent:
|
|
294
|
+
connectedAgent: agentIdentity,
|
|
256
295
|
...(runtimeBinding ? { runtimeBinding } : {}),
|
|
257
296
|
});
|
|
258
297
|
output(`\nWitnora Setup Autopilot completed for ${repository.name}.\n`);
|
|
@@ -278,7 +317,7 @@ export async function runOnboard(options) {
|
|
|
278
317
|
catch (error) {
|
|
279
318
|
const diagnosis = error instanceof Error ? error.message : String(error);
|
|
280
319
|
if (managedGateway?.started) {
|
|
281
|
-
await
|
|
320
|
+
await stopGatewayOwners().catch(() => undefined);
|
|
282
321
|
}
|
|
283
322
|
await realPathActivation?.rollback().catch(() => undefined);
|
|
284
323
|
await rollbackGeneratedFiles(generatedFiles);
|
|
@@ -323,18 +362,25 @@ async function inspectGatewayBinding(directory, projectId, server) {
|
|
|
323
362
|
const existingProjectId = typeof value.projectId === "string" ? value.projectId.trim() : "";
|
|
324
363
|
const existingServer = typeof value.server === "string" ? normalizeServer(value.server) : "";
|
|
325
364
|
const connectionName = typeof value.connectionName === "string" ? value.connectionName.trim() : "";
|
|
365
|
+
const collectorId = typeof value.collectorId === "string" ? value.collectorId.trim() : "";
|
|
326
366
|
const host = typeof value.host === "string" && value.host.trim() ? value.host.trim() : "127.0.0.1";
|
|
327
367
|
const port = Number(value.port);
|
|
328
|
-
if (!existingProjectId || !existingServer || !connectionName || !Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
329
|
-
throw new Error("Existing Gateway configuration is missing a valid project, server, host, or port. Preserve it and repair the setup in Advanced mode.");
|
|
368
|
+
if (!existingProjectId || !existingServer || !connectionName || !collectorId || !Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
369
|
+
throw new Error("Existing Gateway configuration is missing a valid project, server, connection, collector, host, or port. Preserve it and repair the setup in Advanced mode.");
|
|
330
370
|
}
|
|
331
|
-
return { matches: existingProjectId === projectId && existingServer === server, projectId: existingProjectId, server: existingServer, connectionName, host, port };
|
|
371
|
+
return { matches: existingProjectId === projectId && existingServer === server, projectId: existingProjectId, server: existingServer, connectionName, collectorId, host, port };
|
|
332
372
|
}
|
|
333
|
-
async function assertGatewayStopped(requestFetch, host, port, oldProjectId, newProjectId) {
|
|
373
|
+
async function assertGatewayStopped(requestFetch, host, port, oldCollectorId, oldProjectId, newProjectId) {
|
|
334
374
|
try {
|
|
335
375
|
const response = await requestFetch(`http://${host}:${port}/healthz`, { signal: AbortSignal.timeout(800) });
|
|
336
376
|
if (!response.ok)
|
|
337
377
|
return;
|
|
378
|
+
const health = await response.json().catch(() => ({}));
|
|
379
|
+
// A different repository may legitimately own the old localhost port.
|
|
380
|
+
// It must remain untouched; this repository will select a free port after
|
|
381
|
+
// its prior Gateway directory is archived and regenerated.
|
|
382
|
+
if (health.collectorId !== oldCollectorId)
|
|
383
|
+
return;
|
|
338
384
|
}
|
|
339
385
|
catch {
|
|
340
386
|
return;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export declare const FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION: "witnora.failure_experiment_submission.v0.1";
|
|
2
|
+
export interface HostedFailureDefinition {
|
|
3
|
+
title: string;
|
|
4
|
+
taskContractId: string;
|
|
5
|
+
actionPathId: string;
|
|
6
|
+
provider: string;
|
|
7
|
+
environment: "sandbox" | "production";
|
|
8
|
+
failureSignal: string;
|
|
9
|
+
successSignal: string;
|
|
10
|
+
hardInvariants: string[];
|
|
11
|
+
outcomeCheck: {
|
|
12
|
+
kind: "READ_ONLY_PROVIDER_QUERY" | "READ_ONLY_DATABASE_QUERY" | "SIGNED_WEBHOOK_OBSERVATION" | "CUSTOMER_PROBE";
|
|
13
|
+
providerState: string;
|
|
14
|
+
expectedValue: string;
|
|
15
|
+
};
|
|
16
|
+
reproduction: {
|
|
17
|
+
kind: "DETERMINISTIC";
|
|
18
|
+
requiredRuns: number;
|
|
19
|
+
} | {
|
|
20
|
+
kind: "STATISTICAL";
|
|
21
|
+
runs: number;
|
|
22
|
+
minimumFailureRate: number;
|
|
23
|
+
allowedError: number;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export interface HostedFailureExperimentInput {
|
|
27
|
+
discoveryKey: string;
|
|
28
|
+
phase: "BASELINE" | "CANDIDATE" | "RUNTIME";
|
|
29
|
+
/** Stable source observation time; callers should reuse it on retry. */
|
|
30
|
+
observedAt?: string;
|
|
31
|
+
definition: HostedFailureDefinition;
|
|
32
|
+
result: {
|
|
33
|
+
agentVersion: string;
|
|
34
|
+
attempts: number;
|
|
35
|
+
providerCommitCount: number;
|
|
36
|
+
duplicateOutcomeCount: number;
|
|
37
|
+
failureCount: number;
|
|
38
|
+
hardInvariantViolations?: number;
|
|
39
|
+
observation: "REPORTED" | "RECORDED";
|
|
40
|
+
executionControl: "ENFORCED" | "NOT_ENFORCED" | "UNKNOWN";
|
|
41
|
+
outcome: "VERIFIED" | "NOT_VERIFIED" | "INCONCLUSIVE";
|
|
42
|
+
review: "NOT_REVIEWED" | "INTERNALLY_REVIEWED" | "INDEPENDENTLY_REVIEWED";
|
|
43
|
+
receiptIds?: string[];
|
|
44
|
+
};
|
|
45
|
+
evidence: {
|
|
46
|
+
summary: string;
|
|
47
|
+
measurements?: Record<string, string | number | boolean | null>;
|
|
48
|
+
limitations?: string[];
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export interface HostedFailureExperimentOptions {
|
|
52
|
+
baseUrl: string;
|
|
53
|
+
projectId: string;
|
|
54
|
+
apiKey: string;
|
|
55
|
+
fetch?: typeof fetch;
|
|
56
|
+
}
|
|
57
|
+
export interface HostedFailureExperimentResult {
|
|
58
|
+
run: Record<string, unknown>;
|
|
59
|
+
evidence: Record<string, unknown>;
|
|
60
|
+
completion: Record<string, unknown>;
|
|
61
|
+
failureCase: Record<string, unknown>;
|
|
62
|
+
verifiedFixReport: Record<string, unknown>;
|
|
63
|
+
}
|
|
64
|
+
export declare function uploadFailureExperiment(input: HostedFailureExperimentInput, options: HostedFailureExperimentOptions): Promise<HostedFailureExperimentResult>;
|
|
65
|
+
//# sourceMappingURL=failure-experiment-hosted.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-experiment-hosted.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-experiment-hosted.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,4CAA4C,EAAG,4CAAqD,CAAC;AAElH,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,SAAS,GAAG,YAAY,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE;QACZ,IAAI,EAAE,0BAA0B,GAAG,0BAA0B,GAAG,4BAA4B,GAAG,gBAAgB,CAAC;QAChH,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,YAAY,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GACzD;QAAE,IAAI,EAAE,aAAa,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7F;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,uBAAuB,CAAC;IACpC,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,YAAY,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,WAAW,EAAE,UAAU,GAAG,UAAU,CAAC;QACrC,gBAAgB,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC;QAC1D,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,cAAc,CAAC;QACtD,MAAM,EAAE,cAAc,GAAG,qBAAqB,GAAG,wBAAwB,CAAC;QAC1E,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QAChE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;CACH;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,MAAM,WAAW,6BAA6B;IAC5C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,4BAA4B,EACnC,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,6BAA6B,CAAC,CAsFxC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION = "witnora.failure_experiment_submission.v0.1";
|
|
3
|
+
export async function uploadFailureExperiment(input, options) {
|
|
4
|
+
if (input.definition.environment !== "sandbox") {
|
|
5
|
+
throw new Error("Automatic Failure Case progression is limited to the customer sandbox.");
|
|
6
|
+
}
|
|
7
|
+
validateExperimentCounts(input.result);
|
|
8
|
+
const baseUrl = required(options.baseUrl, "baseUrl").replace(/\/$/, "");
|
|
9
|
+
const projectId = required(options.projectId, "projectId");
|
|
10
|
+
const apiKey = required(options.apiKey, "apiKey");
|
|
11
|
+
const requestFetch = options.fetch ?? fetch;
|
|
12
|
+
const projectUrl = `${baseUrl}/v1/projects/${encodeURIComponent(projectId)}`;
|
|
13
|
+
const submittedAt = input.observedAt === undefined ? new Date().toISOString() : normalizedTimestamp(input.observedAt, "observedAt");
|
|
14
|
+
const boundedEvidence = {
|
|
15
|
+
schemaVersion: "witnora.failure_experiment_evidence.v0.1",
|
|
16
|
+
discoveryKey: input.discoveryKey,
|
|
17
|
+
phase: input.phase,
|
|
18
|
+
taskContractId: input.definition.taskContractId,
|
|
19
|
+
actionPathId: input.definition.actionPathId,
|
|
20
|
+
environment: input.definition.environment,
|
|
21
|
+
provider: input.definition.provider,
|
|
22
|
+
summary: bounded(input.evidence.summary, "evidence.summary", 1, 2_000),
|
|
23
|
+
measurements: boundedMeasurements(input.evidence.measurements),
|
|
24
|
+
limitations: (input.evidence.limitations ?? []).slice(0, 20).map((item) => bounded(item, "evidence.limitation", 1, 500)),
|
|
25
|
+
result: structuredClone(input.result),
|
|
26
|
+
observedAt: submittedAt,
|
|
27
|
+
rawPayloadRetainedByCustomer: true,
|
|
28
|
+
};
|
|
29
|
+
const bytes = Buffer.from(`${JSON.stringify(boundedEvidence, null, 2)}\n`, "utf8");
|
|
30
|
+
if (bytes.byteLength > 64 * 1024)
|
|
31
|
+
throw new Error("Failure experiment evidence exceeds the 64 KiB hosted boundary.");
|
|
32
|
+
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
33
|
+
const operation = `failure-experiment-${digest.slice(0, 32)}`;
|
|
34
|
+
const run = await jsonRequest(requestFetch, `${projectUrl}/runs`, apiKey, {
|
|
35
|
+
method: "POST",
|
|
36
|
+
headers: { "idempotency-key": `${operation}:run` },
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
externalId: `${input.discoveryKey}:${input.phase.toLowerCase()}:${digest.slice(0, 16)}`,
|
|
39
|
+
kind: "custom",
|
|
40
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
41
|
+
startedAt: submittedAt,
|
|
42
|
+
metadata: { failureExperiment: {
|
|
43
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
44
|
+
discoveryKey: input.discoveryKey,
|
|
45
|
+
phase: input.phase,
|
|
46
|
+
taskContractId: input.definition.taskContractId,
|
|
47
|
+
actionPathId: input.definition.actionPathId,
|
|
48
|
+
evidenceDigestSha256: digest,
|
|
49
|
+
rawPayloadRetainedByCustomer: true,
|
|
50
|
+
} },
|
|
51
|
+
}),
|
|
52
|
+
});
|
|
53
|
+
const runId = id(run, "Run");
|
|
54
|
+
const query = new URLSearchParams({
|
|
55
|
+
fileName: `failure-experiment-${input.phase.toLowerCase()}-evidence.json`,
|
|
56
|
+
kind: "failure_experiment",
|
|
57
|
+
schemaVersion: "witnora.failure_experiment_evidence.v0.1",
|
|
58
|
+
runId,
|
|
59
|
+
});
|
|
60
|
+
const evidence = await jsonRequest(requestFetch, `${projectUrl}/evidence?${query}`, apiKey, {
|
|
61
|
+
method: "POST",
|
|
62
|
+
headers: { "content-type": "application/json", "idempotency-key": `${operation}:evidence` },
|
|
63
|
+
body: new Uint8Array(bytes).buffer,
|
|
64
|
+
});
|
|
65
|
+
const evidenceId = id(evidence, "Evidence");
|
|
66
|
+
const completion = await jsonRequest(requestFetch, `${projectUrl}/runs/${encodeURIComponent(runId)}/complete`, apiKey, {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { "idempotency-key": `${operation}:complete` },
|
|
69
|
+
body: JSON.stringify({
|
|
70
|
+
status: input.phase === "BASELINE" ? "manual_review" : input.result.failureCount ? "failed" : "passed",
|
|
71
|
+
score: input.result.failureCount ? 0 : 1,
|
|
72
|
+
summary: boundedEvidence.summary,
|
|
73
|
+
metadata: { evidenceId, evidenceSha256: evidence.sha256, failureExperimentDigestSha256: digest },
|
|
74
|
+
}),
|
|
75
|
+
});
|
|
76
|
+
const failureCase = await jsonRequest(requestFetch, `${projectUrl}/failure-cases/ingest`, apiKey, {
|
|
77
|
+
method: "POST",
|
|
78
|
+
headers: { "idempotency-key": `${operation}:ingest` },
|
|
79
|
+
body: JSON.stringify({
|
|
80
|
+
schemaVersion: FAILURE_EXPERIMENT_SUBMISSION_SCHEMA_VERSION,
|
|
81
|
+
discoveryKey: input.discoveryKey,
|
|
82
|
+
phase: input.phase,
|
|
83
|
+
definition: input.definition,
|
|
84
|
+
result: { ...input.result, runIds: [runId], evidenceIds: [evidenceId] },
|
|
85
|
+
}),
|
|
86
|
+
});
|
|
87
|
+
const failureCaseId = id(failureCase, "Failure Case");
|
|
88
|
+
const verifiedFixReport = await jsonRequest(requestFetch, `${projectUrl}/failure-cases/${encodeURIComponent(failureCaseId)}/report`, apiKey, { method: "GET" });
|
|
89
|
+
return { run, evidence, completion, failureCase, verifiedFixReport };
|
|
90
|
+
}
|
|
91
|
+
function validateExperimentCounts(result) {
|
|
92
|
+
const counts = [
|
|
93
|
+
result.attempts,
|
|
94
|
+
result.providerCommitCount,
|
|
95
|
+
result.duplicateOutcomeCount,
|
|
96
|
+
result.failureCount,
|
|
97
|
+
result.hardInvariantViolations ?? 0,
|
|
98
|
+
];
|
|
99
|
+
if (counts.some((value) => !Number.isInteger(value) || value < 0)) {
|
|
100
|
+
throw new Error("Failure experiment counts must be non-negative integers.");
|
|
101
|
+
}
|
|
102
|
+
if (result.attempts < 1
|
|
103
|
+
|| result.providerCommitCount > result.attempts
|
|
104
|
+
|| result.failureCount > result.attempts
|
|
105
|
+
|| result.duplicateOutcomeCount > result.providerCommitCount) {
|
|
106
|
+
throw new Error("Failure experiment counts are internally inconsistent.");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function jsonRequest(requestFetch, url, apiKey, init) {
|
|
110
|
+
const headers = new Headers(init.headers);
|
|
111
|
+
headers.set("authorization", `Bearer ${apiKey}`);
|
|
112
|
+
if (typeof init.body === "string")
|
|
113
|
+
headers.set("content-type", "application/json");
|
|
114
|
+
const response = await requestFetch(url, { ...init, headers, redirect: "error" });
|
|
115
|
+
const value = await response.json().catch(() => ({}));
|
|
116
|
+
if (!response.ok)
|
|
117
|
+
throw new Error(typeof value.error === "string" ? value.error : `Witnora Control Plane request failed (${response.status}).`);
|
|
118
|
+
return value;
|
|
119
|
+
}
|
|
120
|
+
function required(value, name) {
|
|
121
|
+
return bounded(value.trim(), name, 1, 2_000);
|
|
122
|
+
}
|
|
123
|
+
function normalizedTimestamp(value, name) {
|
|
124
|
+
if (!value.trim() || Number.isNaN(Date.parse(value)))
|
|
125
|
+
throw new Error(`Failure experiment ${name} must be an ISO timestamp.`);
|
|
126
|
+
return new Date(value).toISOString();
|
|
127
|
+
}
|
|
128
|
+
function bounded(value, name, minimum, maximum) {
|
|
129
|
+
if (value.length < minimum || value.length > maximum)
|
|
130
|
+
throw new Error(`Failure experiment ${name} must contain ${minimum}-${maximum} characters.`);
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
function boundedMeasurements(value) {
|
|
134
|
+
const entries = Object.entries(value ?? {}).slice(0, 50);
|
|
135
|
+
for (const [key, item] of entries) {
|
|
136
|
+
bounded(key, "evidence.measurement key", 1, 100);
|
|
137
|
+
if (typeof item === "string")
|
|
138
|
+
bounded(item, "evidence.measurement value", 0, 500);
|
|
139
|
+
}
|
|
140
|
+
return Object.fromEntries(entries);
|
|
141
|
+
}
|
|
142
|
+
function id(value, kind) {
|
|
143
|
+
if (typeof value.id !== "string" || !value.id)
|
|
144
|
+
throw new Error(`Witnora Control Plane returned ${kind} without an id.`);
|
|
145
|
+
return value.id;
|
|
146
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { type HostedFailureDefinition, type HostedFailureExperimentInput } from "./failure-experiment-hosted.js";
|
|
2
|
+
export type FailureRuntimeWatchHealth = "HEALTHY" | "REGRESSION" | "INSUFFICIENT_EVIDENCE" | "CONNECTION_UNAVAILABLE";
|
|
3
|
+
export interface FailureRuntimeWatchCase {
|
|
4
|
+
id: string;
|
|
5
|
+
projectId: string;
|
|
6
|
+
status: "FIX_VERIFIED" | "RUNTIME_PROTECTED" | "REGRESSION_DETECTED" | string;
|
|
7
|
+
definition: HostedFailureDefinition;
|
|
8
|
+
discovery?: {
|
|
9
|
+
discoveryKey?: string;
|
|
10
|
+
};
|
|
11
|
+
candidate?: {
|
|
12
|
+
agentVersion: string;
|
|
13
|
+
};
|
|
14
|
+
runtimeWatch?: {
|
|
15
|
+
receiptIds?: string[];
|
|
16
|
+
latest?: {
|
|
17
|
+
receiptIds?: string[];
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export interface FailureRuntimeWatchAction {
|
|
22
|
+
id: string;
|
|
23
|
+
status: string;
|
|
24
|
+
expectedState?: Record<string, unknown>;
|
|
25
|
+
businessTaskBinding?: {
|
|
26
|
+
taskContractId?: string;
|
|
27
|
+
actionPathId?: string;
|
|
28
|
+
environment?: string;
|
|
29
|
+
agentVersion?: string;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export interface FailureRuntimeWatchReceipt {
|
|
33
|
+
id: string;
|
|
34
|
+
actionId: string;
|
|
35
|
+
currentStatus: string;
|
|
36
|
+
createdAt: string;
|
|
37
|
+
receipt: {
|
|
38
|
+
core: {
|
|
39
|
+
evidenceStrength?: string;
|
|
40
|
+
enforcementLevel?: string;
|
|
41
|
+
executionSessionId?: string;
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export interface FailureRuntimeWatchOutcome {
|
|
46
|
+
classification: "HEALTHY" | "REGRESSION" | "INSUFFICIENT_EVIDENCE" | "CONNECTION_UNAVAILABLE" | "SCHEMA_DIGEST_DRIFT";
|
|
47
|
+
attempts?: number;
|
|
48
|
+
providerCommitCount?: number;
|
|
49
|
+
duplicateOutcomeCount?: number;
|
|
50
|
+
failureCount?: number;
|
|
51
|
+
hardInvariantViolations?: number;
|
|
52
|
+
observedAt: string;
|
|
53
|
+
reason?: string;
|
|
54
|
+
}
|
|
55
|
+
export type FailureRuntimeWatchOutcomeReader = (input: {
|
|
56
|
+
failureCase: FailureRuntimeWatchCase;
|
|
57
|
+
resourceId: string;
|
|
58
|
+
actions: FailureRuntimeWatchAction[];
|
|
59
|
+
receipts: FailureRuntimeWatchReceipt[];
|
|
60
|
+
}) => Promise<FailureRuntimeWatchOutcome>;
|
|
61
|
+
export interface FailureRuntimeWatchStatusInput {
|
|
62
|
+
health: FailureRuntimeWatchHealth;
|
|
63
|
+
lastCheckedAt: string;
|
|
64
|
+
nextCheckAt: string;
|
|
65
|
+
detail?: string;
|
|
66
|
+
}
|
|
67
|
+
export interface FailureRuntimeWatchHostedClient {
|
|
68
|
+
listFailureCases(): Promise<FailureRuntimeWatchCase[]>;
|
|
69
|
+
listActions(): Promise<FailureRuntimeWatchAction[]>;
|
|
70
|
+
listReceipts(): Promise<FailureRuntimeWatchReceipt[]>;
|
|
71
|
+
uploadObservation(input: HostedFailureExperimentInput & {
|
|
72
|
+
observedAt: string;
|
|
73
|
+
}): Promise<unknown>;
|
|
74
|
+
updateStatus(failureCaseId: string, input: FailureRuntimeWatchStatusInput): Promise<unknown>;
|
|
75
|
+
}
|
|
76
|
+
export declare function createFailureRuntimeWatchHostedClient(options: {
|
|
77
|
+
baseUrl: string;
|
|
78
|
+
projectId: string;
|
|
79
|
+
apiKey: string;
|
|
80
|
+
fetch?: typeof fetch;
|
|
81
|
+
}): FailureRuntimeWatchHostedClient;
|
|
82
|
+
export interface FailureRuntimeWatchCheckpoint {
|
|
83
|
+
processedReceiptIds: string[];
|
|
84
|
+
updatedAt: string;
|
|
85
|
+
}
|
|
86
|
+
export interface FailureRuntimeWatchCheckpointStore {
|
|
87
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
88
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
89
|
+
}
|
|
90
|
+
export declare class MemoryFailureRuntimeWatchCheckpointStore implements FailureRuntimeWatchCheckpointStore {
|
|
91
|
+
#private;
|
|
92
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
93
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
94
|
+
}
|
|
95
|
+
export declare class FileFailureRuntimeWatchCheckpointStore implements FailureRuntimeWatchCheckpointStore {
|
|
96
|
+
private readonly directory;
|
|
97
|
+
constructor(directory: string);
|
|
98
|
+
load(key: string): Promise<FailureRuntimeWatchCheckpoint | undefined>;
|
|
99
|
+
save(key: string, checkpoint: FailureRuntimeWatchCheckpoint): Promise<void>;
|
|
100
|
+
private path;
|
|
101
|
+
}
|
|
102
|
+
export interface ManagedFailureRuntimeWatchResult {
|
|
103
|
+
casesInspected: number;
|
|
104
|
+
observationsUploaded: number;
|
|
105
|
+
outcomesDeferred: number;
|
|
106
|
+
limitations: string[];
|
|
107
|
+
}
|
|
108
|
+
export declare class ManagedFailureRuntimeWatch {
|
|
109
|
+
#private;
|
|
110
|
+
constructor(options: {
|
|
111
|
+
client: FailureRuntimeWatchHostedClient;
|
|
112
|
+
checkpoints: FailureRuntimeWatchCheckpointStore;
|
|
113
|
+
intervalMs?: number;
|
|
114
|
+
now?: () => Date;
|
|
115
|
+
readOutcome: FailureRuntimeWatchOutcomeReader;
|
|
116
|
+
bindings?: Array<{
|
|
117
|
+
taskContractId: string;
|
|
118
|
+
actionPathId: string;
|
|
119
|
+
environment: "sandbox";
|
|
120
|
+
}>;
|
|
121
|
+
});
|
|
122
|
+
tick(): Promise<ManagedFailureRuntimeWatchResult>;
|
|
123
|
+
private runTick;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Reads only the local sandbox result fields required by Runtime Watch. Raw
|
|
127
|
+
* provider payloads and unbound audit records never leave the customer Harness.
|
|
128
|
+
*/
|
|
129
|
+
export declare function readLocalSandboxRuntimeWatchOutcome(input: {
|
|
130
|
+
path: string;
|
|
131
|
+
resourceId: string;
|
|
132
|
+
actions: Array<{
|
|
133
|
+
id: string;
|
|
134
|
+
executionSessionId: string;
|
|
135
|
+
}>;
|
|
136
|
+
now?: () => Date;
|
|
137
|
+
}): Promise<FailureRuntimeWatchOutcome>;
|
|
138
|
+
//# sourceMappingURL=failure-runtime-watch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-runtime-watch.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/failure-runtime-watch.ts"],"names":[],"mappings":"AAIA,OAAO,EAA2B,KAAK,uBAAuB,EAAE,KAAK,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAE1I,MAAM,MAAM,yBAAyB,GACjC,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,wBAAwB,CAAC;AAE7B,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,GAAG,MAAM,CAAC;IAC9E,UAAU,EAAE,uBAAuB,CAAC;IACpC,SAAS,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,SAAS,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,YAAY,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;SAAE,CAAA;KAAE,CAAC;CAC9E;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,mBAAmB,CAAC,EAAE;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE;YAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CAC1G;AAED,MAAM,WAAW,0BAA0B;IACzC,cAAc,EAAE,SAAS,GAAG,YAAY,GAAG,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAC;IACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE;IACrD,WAAW,EAAE,uBAAuB,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,yBAAyB,EAAE,CAAC;IACrC,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACxC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,yBAAyB,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,+BAA+B;IAC9C,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACvD,WAAW,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAAC;IACpD,YAAY,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IACtD,iBAAiB,CAAC,KAAK,EAAE,4BAA4B,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClG,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,8BAA8B,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9F;AAED,wBAAgB,qCAAqC,CAAC,OAAO,EAAE;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,GAAG,+BAA+B,CAkBlC;AAED,MAAM,WAAW,6BAA6B;IAC5C,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E;AAED,qBAAa,wCAAyC,YAAW,kCAAkC;;IAG3F,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IAKrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;CAGlF;AAED,qBAAa,sCAAuC,YAAW,kCAAkC;IACnF,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAIxC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,6BAA6B,GAAG,SAAS,CAAC;IASrE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IASjF,OAAO,CAAC,IAAI;CAIb;AAED,MAAM,WAAW,gCAAgC;IAC/C,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,0BAA0B;;gBASzB,OAAO,EAAE;QACnB,MAAM,EAAE,+BAA+B,CAAC;QACxC,WAAW,EAAE,kCAAkC,CAAC;QAChD,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;QACjB,WAAW,EAAE,gCAAgC,CAAC;QAC9C,QAAQ,CAAC,EAAE,KAAK,CAAC;YAAE,cAAc,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,SAAS,CAAA;SAAE,CAAC,CAAC;KAC5F;IAYD,IAAI,IAAI,OAAO,CAAC,gCAAgC,CAAC;YAMnC,OAAO;CAgItB;AAID;;;GAGG;AACH,wBAAsB,mCAAmC,CAAC,KAAK,EAAE;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3D,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,0BAA0B,CAAC,CA6DtC"}
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { uploadFailureExperiment } from "./failure-experiment-hosted.js";
|
|
5
|
+
export function createFailureRuntimeWatchHostedClient(options) {
|
|
6
|
+
const baseUrl = required(options.baseUrl, "baseUrl").replace(/\/$/u, "");
|
|
7
|
+
const projectId = required(options.projectId, "projectId");
|
|
8
|
+
const apiKey = required(options.apiKey, "apiKey");
|
|
9
|
+
const requestFetch = options.fetch ?? fetch;
|
|
10
|
+
const projectUrl = `${baseUrl}/v1/projects/${encodeURIComponent(projectId)}`;
|
|
11
|
+
return {
|
|
12
|
+
listFailureCases: async () => list(await hostedJson(requestFetch, `${projectUrl}/failure-cases`, apiKey), "failureCases"),
|
|
13
|
+
listActions: async () => list(await hostedJson(requestFetch, `${projectUrl}/actions`, apiKey), "actions"),
|
|
14
|
+
listReceipts: async () => list(await hostedJson(requestFetch, `${projectUrl}/receipts`, apiKey), "receipts"),
|
|
15
|
+
uploadObservation: async (input) => uploadFailureExperiment(input, { baseUrl, projectId, apiKey, fetch: requestFetch }),
|
|
16
|
+
updateStatus: async (failureCaseId, input) => hostedJson(requestFetch, `${projectUrl}/failure-cases/${encodeURIComponent(required(failureCaseId, "failureCaseId"))}/runtime-watch-status`, apiKey, { method: "POST", body: JSON.stringify(input) }),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export class MemoryFailureRuntimeWatchCheckpointStore {
|
|
20
|
+
#checkpoints = new Map();
|
|
21
|
+
async load(key) {
|
|
22
|
+
const checkpoint = this.#checkpoints.get(key);
|
|
23
|
+
return checkpoint ? structuredClone(checkpoint) : undefined;
|
|
24
|
+
}
|
|
25
|
+
async save(key, checkpoint) {
|
|
26
|
+
this.#checkpoints.set(key, structuredClone(checkpoint));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export class FileFailureRuntimeWatchCheckpointStore {
|
|
30
|
+
directory;
|
|
31
|
+
constructor(directory) {
|
|
32
|
+
this.directory = directory;
|
|
33
|
+
if (!directory.trim())
|
|
34
|
+
throw new Error("Failure Runtime Watch checkpoint directory is required.");
|
|
35
|
+
}
|
|
36
|
+
async load(key) {
|
|
37
|
+
try {
|
|
38
|
+
return validateCheckpoint(JSON.parse(await readFile(this.path(key), "utf8")));
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error.code === "ENOENT")
|
|
42
|
+
return undefined;
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async save(key, checkpoint) {
|
|
47
|
+
const value = validateCheckpoint(checkpoint);
|
|
48
|
+
await mkdir(this.directory, { recursive: true });
|
|
49
|
+
const target = this.path(key);
|
|
50
|
+
const temporary = `${target}.${randomUUID()}.tmp`;
|
|
51
|
+
await writeFile(temporary, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
52
|
+
await rename(temporary, target);
|
|
53
|
+
}
|
|
54
|
+
path(key) {
|
|
55
|
+
if (!key.trim())
|
|
56
|
+
throw new Error("Failure Runtime Watch checkpoint key is required.");
|
|
57
|
+
return join(this.directory, `${createHash("sha256").update(key).digest("hex")}.json`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export class ManagedFailureRuntimeWatch {
|
|
61
|
+
#client;
|
|
62
|
+
#checkpoints;
|
|
63
|
+
#intervalMs;
|
|
64
|
+
#now;
|
|
65
|
+
#readOutcome;
|
|
66
|
+
#bindings;
|
|
67
|
+
#activeTick;
|
|
68
|
+
constructor(options) {
|
|
69
|
+
this.#client = options.client;
|
|
70
|
+
this.#checkpoints = options.checkpoints;
|
|
71
|
+
this.#intervalMs = options.intervalMs ?? 5_000;
|
|
72
|
+
this.#now = options.now ?? (() => new Date());
|
|
73
|
+
this.#readOutcome = options.readOutcome;
|
|
74
|
+
this.#bindings = options.bindings ? new Set(options.bindings.map(bindingKey)) : undefined;
|
|
75
|
+
if (!Number.isSafeInteger(this.#intervalMs) || this.#intervalMs < 1_000 || this.#intervalMs > 60_000) {
|
|
76
|
+
throw new Error("Failure Runtime Watch intervalMs must be between 1000 and 60000.");
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
tick() {
|
|
80
|
+
if (this.#activeTick)
|
|
81
|
+
return this.#activeTick;
|
|
82
|
+
this.#activeTick = this.runTick().finally(() => { this.#activeTick = undefined; });
|
|
83
|
+
return this.#activeTick;
|
|
84
|
+
}
|
|
85
|
+
async runTick() {
|
|
86
|
+
const failureCases = await this.#client.listFailureCases();
|
|
87
|
+
const candidates = failureCases.filter((failureCase) => failureCase.definition.environment === "sandbox"
|
|
88
|
+
&& (!this.#bindings || this.#bindings.has(bindingKey(failureCase.definition)))
|
|
89
|
+
&& ["FIX_VERIFIED", "RUNTIME_PROTECTED", "REGRESSION_DETECTED"].includes(failureCase.status));
|
|
90
|
+
if (!candidates.length)
|
|
91
|
+
return { casesInspected: 0, observationsUploaded: 0, outcomesDeferred: 0, limitations: [] };
|
|
92
|
+
let actions;
|
|
93
|
+
let receipts;
|
|
94
|
+
try {
|
|
95
|
+
[actions, receipts] = await Promise.all([this.#client.listActions(), this.#client.listReceipts()]);
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
const detail = "Runtime Watch could not load new Action and Receipt metadata from Witnora.";
|
|
99
|
+
await Promise.all(candidates.map((failureCase) => {
|
|
100
|
+
const checkedAt = this.#now().toISOString();
|
|
101
|
+
return this.#client.updateStatus(failureCase.id, {
|
|
102
|
+
health: "CONNECTION_UNAVAILABLE",
|
|
103
|
+
lastCheckedAt: checkedAt,
|
|
104
|
+
nextCheckAt: new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString(),
|
|
105
|
+
detail,
|
|
106
|
+
});
|
|
107
|
+
}));
|
|
108
|
+
return { casesInspected: candidates.length, observationsUploaded: 0, outcomesDeferred: candidates.length, limitations: [detail] };
|
|
109
|
+
}
|
|
110
|
+
let eligible = candidates;
|
|
111
|
+
if (!this.#bindings) {
|
|
112
|
+
const discoveredBindings = coveredBindingKeys(actions, receipts);
|
|
113
|
+
if (discoveredBindings.size !== 1) {
|
|
114
|
+
const detail = "Runtime Watch requires one exact sandbox Task and action path with an active enforced Receipt.";
|
|
115
|
+
await Promise.all(candidates.map((failureCase) => {
|
|
116
|
+
const checkedAt = this.#now().toISOString();
|
|
117
|
+
return this.#client.updateStatus(failureCase.id, {
|
|
118
|
+
health: "INSUFFICIENT_EVIDENCE",
|
|
119
|
+
lastCheckedAt: checkedAt,
|
|
120
|
+
nextCheckAt: new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString(),
|
|
121
|
+
detail,
|
|
122
|
+
});
|
|
123
|
+
}));
|
|
124
|
+
return { casesInspected: candidates.length, observationsUploaded: 0, outcomesDeferred: candidates.length, limitations: [detail] };
|
|
125
|
+
}
|
|
126
|
+
const discoveredBinding = [...discoveredBindings][0];
|
|
127
|
+
eligible = candidates.filter((failureCase) => bindingKey(failureCase.definition) === discoveredBinding);
|
|
128
|
+
}
|
|
129
|
+
let observationsUploaded = 0;
|
|
130
|
+
let outcomesDeferred = 0;
|
|
131
|
+
const limitations = [];
|
|
132
|
+
for (const failureCase of eligible) {
|
|
133
|
+
const checkedAt = this.#now().toISOString();
|
|
134
|
+
const nextCheckAt = new Date(Date.parse(checkedAt) + this.#intervalMs).toISOString();
|
|
135
|
+
const checkpoint = await this.#checkpoints.load(failureCase.id) ?? {
|
|
136
|
+
processedReceiptIds: failureCase.runtimeWatch?.receiptIds ?? failureCase.runtimeWatch?.latest?.receiptIds ?? [],
|
|
137
|
+
updatedAt: checkedAt,
|
|
138
|
+
};
|
|
139
|
+
const covered = coveredOutcomes(failureCase, actions, receipts);
|
|
140
|
+
const pending = covered.filter(({ receipt }) => !checkpoint.processedReceiptIds.includes(receipt.id));
|
|
141
|
+
if (!pending.length) {
|
|
142
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
143
|
+
health: failureCase.status === "REGRESSION_DETECTED" ? "REGRESSION" : "HEALTHY",
|
|
144
|
+
lastCheckedAt: checkedAt,
|
|
145
|
+
nextCheckAt,
|
|
146
|
+
});
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const resourceId = pending[0].resourceId;
|
|
150
|
+
const group = covered.filter((item) => item.resourceId === resourceId);
|
|
151
|
+
const outcome = await this.#readOutcome({
|
|
152
|
+
failureCase,
|
|
153
|
+
resourceId,
|
|
154
|
+
actions: group.map((item) => item.action),
|
|
155
|
+
receipts: group.map((item) => item.receipt),
|
|
156
|
+
});
|
|
157
|
+
if (outcome.classification !== "HEALTHY" && outcome.classification !== "REGRESSION") {
|
|
158
|
+
outcomesDeferred += 1;
|
|
159
|
+
limitations.push(outcome.reason ?? `Failure Runtime Watch deferred ${failureCase.id}: ${outcome.classification}.`);
|
|
160
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
161
|
+
health: outcome.classification === "SCHEMA_DIGEST_DRIFT" ? "INSUFFICIENT_EVIDENCE" : outcome.classification,
|
|
162
|
+
lastCheckedAt: checkedAt,
|
|
163
|
+
nextCheckAt,
|
|
164
|
+
...(outcome.reason ? { detail: outcome.reason } : {}),
|
|
165
|
+
});
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const receiptIds = group.map((item) => item.receipt.id);
|
|
169
|
+
const agentVersion = requiredAgentVersion(failureCase, group.map((item) => item.action));
|
|
170
|
+
await this.#client.uploadObservation({
|
|
171
|
+
discoveryKey: failureCase.discovery?.discoveryKey ?? failureCase.id,
|
|
172
|
+
phase: "RUNTIME",
|
|
173
|
+
observedAt: outcome.observedAt,
|
|
174
|
+
definition: structuredClone(failureCase.definition),
|
|
175
|
+
result: {
|
|
176
|
+
agentVersion,
|
|
177
|
+
attempts: outcome.attempts ?? group.length,
|
|
178
|
+
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
179
|
+
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
180
|
+
failureCount: outcome.failureCount ?? 0,
|
|
181
|
+
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
182
|
+
observation: "RECORDED",
|
|
183
|
+
executionControl: "ENFORCED",
|
|
184
|
+
outcome: "VERIFIED",
|
|
185
|
+
review: "NOT_REVIEWED",
|
|
186
|
+
receiptIds,
|
|
187
|
+
},
|
|
188
|
+
evidence: {
|
|
189
|
+
summary: outcome.classification === "REGRESSION"
|
|
190
|
+
? "A covered sandbox result reproduced the defined failure after the candidate was verified."
|
|
191
|
+
: "A covered sandbox result remained healthy after the candidate was verified.",
|
|
192
|
+
measurements: {
|
|
193
|
+
providerCommitCount: outcome.providerCommitCount ?? 0,
|
|
194
|
+
duplicateOutcomeCount: outcome.duplicateOutcomeCount ?? 0,
|
|
195
|
+
failureCount: outcome.failureCount ?? 0,
|
|
196
|
+
hardInvariantViolations: outcome.hardInvariantViolations ?? 0,
|
|
197
|
+
receiptCount: receiptIds.length,
|
|
198
|
+
},
|
|
199
|
+
limitations: ["This automated observation covers the exact customer-owned sandbox task and action path only; it does not establish production reliability."],
|
|
200
|
+
},
|
|
201
|
+
});
|
|
202
|
+
observationsUploaded += 1;
|
|
203
|
+
await this.#client.updateStatus(failureCase.id, {
|
|
204
|
+
health: outcome.classification === "REGRESSION" ? "REGRESSION" : "HEALTHY",
|
|
205
|
+
lastCheckedAt: checkedAt,
|
|
206
|
+
nextCheckAt,
|
|
207
|
+
});
|
|
208
|
+
await this.#checkpoints.save(failureCase.id, {
|
|
209
|
+
processedReceiptIds: unique([...checkpoint.processedReceiptIds, ...receiptIds]),
|
|
210
|
+
updatedAt: checkedAt,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
return { casesInspected: eligible.length, observationsUploaded, outcomesDeferred, limitations };
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const MAX_LOCAL_SANDBOX_AUDIT_BYTES = 10 * 1024 * 1024;
|
|
217
|
+
/**
|
|
218
|
+
* Reads only the local sandbox result fields required by Runtime Watch. Raw
|
|
219
|
+
* provider payloads and unbound audit records never leave the customer Harness.
|
|
220
|
+
*/
|
|
221
|
+
export async function readLocalSandboxRuntimeWatchOutcome(input) {
|
|
222
|
+
const observedAt = () => (input.now?.() ?? new Date()).toISOString();
|
|
223
|
+
const attempts = input.actions.length;
|
|
224
|
+
let source;
|
|
225
|
+
try {
|
|
226
|
+
const metadata = await stat(input.path);
|
|
227
|
+
if (!metadata.isFile() || metadata.size > MAX_LOCAL_SANDBOX_AUDIT_BYTES) {
|
|
228
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "The local sandbox audit is not a bounded readable file.");
|
|
229
|
+
}
|
|
230
|
+
source = await readFile(input.path, "utf8");
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
const code = error.code;
|
|
234
|
+
if (code === "ENOENT" || code === "EACCES" || code === "EPERM") {
|
|
235
|
+
return deferred("CONNECTION_UNAVAILABLE", attempts, observedAt(), "The local sandbox audit is unavailable to Runtime Watch.");
|
|
236
|
+
}
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
const expected = new Map(input.actions.map((action) => [`${action.id}\0${action.executionSessionId}`, action]));
|
|
240
|
+
if (expected.size !== input.actions.length || !input.resourceId || [...expected.values()].some((action) => !action.id || !action.executionSessionId)) {
|
|
241
|
+
return deferred("INSUFFICIENT_EVIDENCE", attempts, observedAt(), "The covered Action and execution-session bindings are incomplete.");
|
|
242
|
+
}
|
|
243
|
+
const commits = new Map();
|
|
244
|
+
for (const line of source.split(/\r?\n/u)) {
|
|
245
|
+
if (!line.trim())
|
|
246
|
+
continue;
|
|
247
|
+
let raw;
|
|
248
|
+
try {
|
|
249
|
+
raw = JSON.parse(line);
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "The local sandbox audit contains invalid JSON.");
|
|
253
|
+
}
|
|
254
|
+
if (!isRecord(raw) || raw.phase !== "COMMITTED")
|
|
255
|
+
continue;
|
|
256
|
+
const record = allowlistedCommittedAuditRecord(raw);
|
|
257
|
+
if (!record) {
|
|
258
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "A committed local sandbox result is missing an allowlisted field.");
|
|
259
|
+
}
|
|
260
|
+
const key = `${record.actionId}\0${record.executionSessionId}`;
|
|
261
|
+
if (record.resourceId !== input.resourceId || !expected.has(key))
|
|
262
|
+
continue;
|
|
263
|
+
const digest = createHash("sha256").update(JSON.stringify(record.observedState)).digest("hex");
|
|
264
|
+
if (digest !== record.stateSha256) {
|
|
265
|
+
return deferred("SCHEMA_DIGEST_DRIFT", attempts, observedAt(), "A covered local sandbox result failed its state digest check.");
|
|
266
|
+
}
|
|
267
|
+
const existing = commits.get(key);
|
|
268
|
+
if (!existing || record.at > existing.at)
|
|
269
|
+
commits.set(key, { at: record.at });
|
|
270
|
+
}
|
|
271
|
+
if (commits.size !== expected.size) {
|
|
272
|
+
return deferred("INSUFFICIENT_EVIDENCE", attempts, observedAt(), "Not every covered Action has a digest-verified committed result.");
|
|
273
|
+
}
|
|
274
|
+
const latest = [...commits.values()].map((commit) => commit.at).sort().at(-1);
|
|
275
|
+
const duplicateOutcomeCount = Math.max(0, commits.size - 1);
|
|
276
|
+
return {
|
|
277
|
+
classification: duplicateOutcomeCount > 0 ? "REGRESSION" : "HEALTHY",
|
|
278
|
+
attempts,
|
|
279
|
+
providerCommitCount: commits.size,
|
|
280
|
+
duplicateOutcomeCount,
|
|
281
|
+
failureCount: duplicateOutcomeCount > 0 ? 1 : 0,
|
|
282
|
+
hardInvariantViolations: duplicateOutcomeCount > 0 ? 1 : 0,
|
|
283
|
+
observedAt: latest,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
function allowlistedCommittedAuditRecord(value) {
|
|
287
|
+
if (typeof value.resourceId !== "string"
|
|
288
|
+
|| typeof value.actionId !== "string"
|
|
289
|
+
|| typeof value.executionSessionId !== "string"
|
|
290
|
+
|| !isRecord(value.observedState)
|
|
291
|
+
|| typeof value.stateSha256 !== "string"
|
|
292
|
+
|| !/^[a-f0-9]{64}$/u.test(value.stateSha256)
|
|
293
|
+
|| typeof value.at !== "string"
|
|
294
|
+
|| Number.isNaN(Date.parse(value.at)))
|
|
295
|
+
return undefined;
|
|
296
|
+
return {
|
|
297
|
+
resourceId: value.resourceId,
|
|
298
|
+
actionId: value.actionId,
|
|
299
|
+
executionSessionId: value.executionSessionId,
|
|
300
|
+
observedState: value.observedState,
|
|
301
|
+
stateSha256: value.stateSha256,
|
|
302
|
+
at: new Date(value.at).toISOString(),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
function deferred(classification, attempts, observedAt, reason) {
|
|
306
|
+
return { classification, attempts, observedAt, reason };
|
|
307
|
+
}
|
|
308
|
+
function isRecord(value) {
|
|
309
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
310
|
+
}
|
|
311
|
+
function coveredOutcomes(failureCase, actions, receipts) {
|
|
312
|
+
const receiptsByAction = new Map(receipts.filter((receipt) => receipt.currentStatus === "ACTIVE"
|
|
313
|
+
&& receipt.receipt.core.enforcementLevel === "ENFORCED"
|
|
314
|
+
&& (receipt.receipt.core.evidenceStrength === "OUTCOME_VERIFIED" || receipt.receipt.core.evidenceStrength === "INDEPENDENTLY_REVIEWED"))
|
|
315
|
+
.map((receipt) => [receipt.actionId, receipt]));
|
|
316
|
+
return actions.flatMap((action) => {
|
|
317
|
+
const binding = action.businessTaskBinding;
|
|
318
|
+
const receipt = receiptsByAction.get(action.id);
|
|
319
|
+
const resourceId = typeof action.expectedState?.resourceId === "string" ? action.expectedState.resourceId : undefined;
|
|
320
|
+
return receipt && action.status === "VERIFIED" && resourceId
|
|
321
|
+
&& binding?.taskContractId === failureCase.definition.taskContractId
|
|
322
|
+
&& binding.actionPathId === failureCase.definition.actionPathId
|
|
323
|
+
&& binding.environment === failureCase.definition.environment
|
|
324
|
+
? [{ action, receipt, resourceId }]
|
|
325
|
+
: [];
|
|
326
|
+
}).sort((left, right) => left.receipt.createdAt.localeCompare(right.receipt.createdAt));
|
|
327
|
+
}
|
|
328
|
+
function coveredBindingKeys(actions, receipts) {
|
|
329
|
+
const coveredActionIds = new Set(receipts.filter((receipt) => receipt.currentStatus === "ACTIVE"
|
|
330
|
+
&& receipt.receipt.core.enforcementLevel === "ENFORCED"
|
|
331
|
+
&& (receipt.receipt.core.evidenceStrength === "OUTCOME_VERIFIED" || receipt.receipt.core.evidenceStrength === "INDEPENDENTLY_REVIEWED"))
|
|
332
|
+
.map((receipt) => receipt.actionId));
|
|
333
|
+
return new Set(actions.flatMap((action) => {
|
|
334
|
+
const binding = action.businessTaskBinding;
|
|
335
|
+
return action.status === "VERIFIED" && coveredActionIds.has(action.id)
|
|
336
|
+
&& binding?.environment === "sandbox" && binding.taskContractId && binding.actionPathId
|
|
337
|
+
? [bindingKey({ taskContractId: binding.taskContractId, actionPathId: binding.actionPathId, environment: binding.environment })]
|
|
338
|
+
: [];
|
|
339
|
+
}));
|
|
340
|
+
}
|
|
341
|
+
function requiredAgentVersion(failureCase, actions) {
|
|
342
|
+
const versions = new Set(actions.map((action) => action.businessTaskBinding?.agentVersion ?? failureCase.candidate?.agentVersion).filter(Boolean));
|
|
343
|
+
if (versions.size !== 1)
|
|
344
|
+
throw new Error("Failure Runtime Watch requires one exact Agent version.");
|
|
345
|
+
return [...versions][0];
|
|
346
|
+
}
|
|
347
|
+
function unique(values) { return [...new Set(values)]; }
|
|
348
|
+
function bindingKey(value) {
|
|
349
|
+
return `${value.taskContractId}\0${value.actionPathId}\0${value.environment}`;
|
|
350
|
+
}
|
|
351
|
+
async function hostedJson(requestFetch, url, apiKey, init = {}) {
|
|
352
|
+
const headers = new Headers(init.headers);
|
|
353
|
+
headers.set("authorization", `Bearer ${apiKey}`);
|
|
354
|
+
if (typeof init.body === "string")
|
|
355
|
+
headers.set("content-type", "application/json");
|
|
356
|
+
const response = await requestFetch(url, { ...init, headers, redirect: "error" });
|
|
357
|
+
const value = await response.json().catch(() => ({}));
|
|
358
|
+
if (!response.ok)
|
|
359
|
+
throw new Error(typeof value.error === "string" ? value.error : `Witnora Control Plane request failed (${response.status}).`);
|
|
360
|
+
return value;
|
|
361
|
+
}
|
|
362
|
+
function list(value, field) {
|
|
363
|
+
const items = value[field];
|
|
364
|
+
if (!Array.isArray(items) || items.length > 1_000)
|
|
365
|
+
throw new Error(`Witnora returned an invalid ${field} list.`);
|
|
366
|
+
return items;
|
|
367
|
+
}
|
|
368
|
+
function required(value, name) {
|
|
369
|
+
const normalized = value.trim();
|
|
370
|
+
if (!normalized)
|
|
371
|
+
throw new Error(`Failure Runtime Watch ${name} is required.`);
|
|
372
|
+
return normalized;
|
|
373
|
+
}
|
|
374
|
+
function validateCheckpoint(value) {
|
|
375
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
376
|
+
throw new Error("Failure Runtime Watch checkpoint is invalid.");
|
|
377
|
+
const checkpoint = value;
|
|
378
|
+
if (!Array.isArray(checkpoint.processedReceiptIds)
|
|
379
|
+
|| checkpoint.processedReceiptIds.length > 1_000
|
|
380
|
+
|| checkpoint.processedReceiptIds.some((id) => typeof id !== "string" || !id || id.length > 200)
|
|
381
|
+
|| new Set(checkpoint.processedReceiptIds).size !== checkpoint.processedReceiptIds.length
|
|
382
|
+
|| typeof checkpoint.updatedAt !== "string"
|
|
383
|
+
|| Number.isNaN(Date.parse(checkpoint.updatedAt))) {
|
|
384
|
+
throw new Error("Failure Runtime Watch checkpoint is invalid.");
|
|
385
|
+
}
|
|
386
|
+
return { processedReceiptIds: [...checkpoint.processedReceiptIds], updatedAt: new Date(checkpoint.updatedAt).toISOString() };
|
|
387
|
+
}
|