witnora 0.13.9 → 0.13.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/vendor/onegent-runtime/business-task-evaluator.d.ts +9 -0
- package/dist/vendor/onegent-runtime/business-task-evaluator.d.ts.map +1 -1
- package/dist/vendor/onegent-runtime/business-task-evaluator.js +39 -5
- package/dist/vendor/onegent-runtime/managed-workflow-harness.d.ts +41 -0
- package/dist/vendor/onegent-runtime/managed-workflow-harness.d.ts.map +1 -1
- package/dist/vendor/onegent-runtime/managed-workflow-harness.js +84 -2
- package/dist/vendor/onegent-runtime/task-evaluation-hosted.d.ts +8 -0
- package/dist/vendor/onegent-runtime/task-evaluation-hosted.d.ts.map +1 -1
- package/dist/vendor/onegent-runtime/task-evaluation-hosted.js +15 -1
- package/package.json +1 -1
|
@@ -9,7 +9,16 @@ export interface BusinessTaskEvaluationBinding {
|
|
|
9
9
|
taskKey: string;
|
|
10
10
|
agentId: string;
|
|
11
11
|
agentVersion: string;
|
|
12
|
+
baselineAgentVersion?: string;
|
|
12
13
|
mode: "REPLAY" | "SHADOW";
|
|
14
|
+
scenarioPriority?: {
|
|
15
|
+
source: "PROJECT_HISTORY" | "APPROVED_SUITE";
|
|
16
|
+
prioritizedScenarioIds: string[];
|
|
17
|
+
priorEvaluations: number;
|
|
18
|
+
priorFailures: number;
|
|
19
|
+
priorRecoveries: number;
|
|
20
|
+
reason: string;
|
|
21
|
+
};
|
|
13
22
|
}
|
|
14
23
|
export interface BusinessTaskEvaluatorServerOptions {
|
|
15
24
|
credential: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"business-task-evaluator.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/business-task-evaluator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAGL,KAAK,yBAAyB,EAC9B,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAE3B,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;AAE9B,eAAO,MAAM,sCAAsC,EAAG,sCAA+C,CAAC;AACtG,eAAO,MAAM,+CAA+C,EAAG,+CAAwD,CAAC;AAExH,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,wBAAwB,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"business-task-evaluator.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/business-task-evaluator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAGL,KAAK,yBAAyB,EAC9B,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAE3B,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;AAE9B,eAAO,MAAM,sCAAsC,EAAG,sCAA+C,CAAC;AACtG,eAAO,MAAM,+CAA+C,EAAG,+CAAwD,CAAC;AAExH,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,wBAAwB,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,gBAAgB,CAAC,EAAE;QACjB,MAAM,EAAE,iBAAiB,GAAG,gBAAgB,CAAC;QAC7C,sBAAsB,EAAE,MAAM,EAAE,CAAC;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,kCAAkC;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAC;IACnJ,mBAAmB,CAAC,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,2BAA2B,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrL,uBAAuB,CAAC,CAAC,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC1O,sBAAsB,CAAC,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAC;IACtJ,uBAAuB,CAAC,CAAC,OAAO,EAAE;QAChC,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,aAAa,CAAC;QACtB,KAAK,EAAE,OAAO,CAAC;QACf,OAAO,EAAE,CAAC,MAAM,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,sBAAsB,EAAE,MAAM,CAAA;SAAE,KAAK,IAAI,CAAC;KAC/E,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;CACnI;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,kCAAkC;aA+C1E,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;aAQ3B,OAAO,CAAC,IAAI,CAAC;EAK/B"}
|
|
@@ -83,7 +83,7 @@ async function evaluate(options, task, binding, signal) {
|
|
|
83
83
|
options.createReplayHarness(task, binding, signal), options.loadReplayScenarios(task, binding, signal),
|
|
84
84
|
]);
|
|
85
85
|
return runReplayEvaluation({
|
|
86
|
-
task, harness, tenantId, scenarios,
|
|
86
|
+
task, harness, tenantId, scenarios: prioritizeScenarios(scenarios, binding.scenarioPriority?.prioritizedScenarioIds),
|
|
87
87
|
evaluateCandidate: (scenario) => options.evaluateReplayCandidate(scenario, task, binding, signal),
|
|
88
88
|
});
|
|
89
89
|
}
|
|
@@ -91,7 +91,7 @@ async function evaluate(options, task, binding, signal) {
|
|
|
91
91
|
throw new EvaluatorError(409, "shadow_not_configured");
|
|
92
92
|
return runShadowEvaluation({
|
|
93
93
|
task,
|
|
94
|
-
observations: await options.loadShadowObservations(task, binding, signal),
|
|
94
|
+
observations: prioritizeScenarios(await options.loadShadowObservations(task, binding, signal), binding.scenarioPriority?.prioritizedScenarioIds),
|
|
95
95
|
evaluateCandidate: (context) => options.evaluateShadowCandidate(context, task, binding, signal),
|
|
96
96
|
});
|
|
97
97
|
}
|
|
@@ -101,21 +101,55 @@ function parseRequest(input) {
|
|
|
101
101
|
if (body.schemaVersion !== BUSINESS_TASK_EVALUATION_REQUEST_SCHEMA_VERSION)
|
|
102
102
|
throw new EvaluatorError(422, "invalid_schema");
|
|
103
103
|
const evaluation = object(body.evaluation, "evaluation");
|
|
104
|
-
exactKeys(evaluation, ["nodeId", "taskContractId", "taskContractDigestSha256", "taskKey", "agentId", "agentVersion", "mode"], "evaluation");
|
|
104
|
+
exactKeys(evaluation, ["nodeId", "taskContractId", "taskContractDigestSha256", "taskKey", "agentId", "agentVersion", "baselineAgentVersion", "mode", "scenarioPriority"], "evaluation");
|
|
105
105
|
if (evaluation.mode !== "REPLAY" && evaluation.mode !== "SHADOW")
|
|
106
106
|
throw new EvaluatorError(422, "invalid_mode");
|
|
107
107
|
const binding = {
|
|
108
108
|
nodeId: requiredString(evaluation.nodeId), taskContractId: requiredString(evaluation.taskContractId),
|
|
109
109
|
taskContractDigestSha256: digest(String(evaluation.taskContractDigestSha256 ?? ""), "taskContractDigestSha256"),
|
|
110
110
|
taskKey: requiredString(evaluation.taskKey), agentId: requiredString(evaluation.agentId),
|
|
111
|
-
agentVersion: requiredString(evaluation.agentVersion),
|
|
111
|
+
agentVersion: requiredString(evaluation.agentVersion),
|
|
112
|
+
...(evaluation.baselineAgentVersion === undefined ? {} : { baselineAgentVersion: requiredString(evaluation.baselineAgentVersion) }),
|
|
113
|
+
...(evaluation.scenarioPriority === undefined ? {} : { scenarioPriority: parseScenarioPriority(evaluation.scenarioPriority) }),
|
|
114
|
+
mode: evaluation.mode,
|
|
112
115
|
};
|
|
113
116
|
return { task: object(body.task, "task"), evaluation: binding };
|
|
114
117
|
}
|
|
118
|
+
function parseScenarioPriority(value) {
|
|
119
|
+
const priority = object(value, "scenario_priority");
|
|
120
|
+
exactKeys(priority, ["source", "prioritizedScenarioIds", "priorEvaluations", "priorFailures", "priorRecoveries", "reason"], "scenario_priority");
|
|
121
|
+
if (priority.source !== "PROJECT_HISTORY" && priority.source !== "APPROVED_SUITE")
|
|
122
|
+
throw new EvaluatorError(422, "invalid_scenario_priority");
|
|
123
|
+
const ids = Array.isArray(priority.prioritizedScenarioIds) ? priority.prioritizedScenarioIds : [];
|
|
124
|
+
if (ids.length > 100 || new Set(ids).size !== ids.length || ids.some((id) => typeof id !== "string" || !id || id.length > 200))
|
|
125
|
+
throw new EvaluatorError(422, "invalid_scenario_priority");
|
|
126
|
+
const counts = [priority.priorEvaluations, priority.priorFailures, priority.priorRecoveries];
|
|
127
|
+
if (counts.some((count) => !Number.isSafeInteger(count) || Number(count) < 0))
|
|
128
|
+
throw new EvaluatorError(422, "invalid_scenario_priority");
|
|
129
|
+
const reason = requiredString(priority.reason);
|
|
130
|
+
if (reason.length > 500)
|
|
131
|
+
throw new EvaluatorError(422, "invalid_scenario_priority");
|
|
132
|
+
return {
|
|
133
|
+
source: priority.source,
|
|
134
|
+
prioritizedScenarioIds: ids,
|
|
135
|
+
priorEvaluations: Number(priority.priorEvaluations),
|
|
136
|
+
priorFailures: Number(priority.priorFailures),
|
|
137
|
+
priorRecoveries: Number(priority.priorRecoveries),
|
|
138
|
+
reason,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function prioritizeScenarios(scenarios, ids) {
|
|
142
|
+
if (!ids?.length)
|
|
143
|
+
return scenarios;
|
|
144
|
+
const order = new Map(ids.map((id, index) => [id, index]));
|
|
145
|
+
return scenarios.map((scenario, index) => ({ scenario, index }))
|
|
146
|
+
.sort((left, right) => (order.get(left.scenario.id) ?? Number.MAX_SAFE_INTEGER) - (order.get(right.scenario.id) ?? Number.MAX_SAFE_INTEGER) || left.index - right.index)
|
|
147
|
+
.map(({ scenario }) => scenario);
|
|
148
|
+
}
|
|
115
149
|
function validateBinding(task, binding) {
|
|
116
150
|
if (task.id !== binding.taskContractId || task.digestSha256 !== binding.taskContractDigestSha256
|
|
117
151
|
|| task.taskKey !== binding.taskKey || task.subject?.agentId !== binding.agentId
|
|
118
|
-
|| task.subject?.agentVersion !== binding.agentVersion)
|
|
152
|
+
|| task.subject?.agentVersion !== (binding.baselineAgentVersion ?? binding.agentVersion))
|
|
119
153
|
throw new EvaluatorError(409, "task_binding_mismatch");
|
|
120
154
|
}
|
|
121
155
|
function authorized(request, credential) {
|
|
@@ -29,7 +29,45 @@ export interface ManagedWorkflowReadyEvaluation {
|
|
|
29
29
|
taskKey: string;
|
|
30
30
|
agentId: string;
|
|
31
31
|
agentVersion: string;
|
|
32
|
+
baselineAgentVersion?: string;
|
|
32
33
|
mode: ManagedWorkflowEvaluationMode;
|
|
34
|
+
scenarioPriority?: {
|
|
35
|
+
source: "PROJECT_HISTORY" | "APPROVED_SUITE";
|
|
36
|
+
prioritizedScenarioIds: string[];
|
|
37
|
+
priorEvaluations: number;
|
|
38
|
+
priorFailures: number;
|
|
39
|
+
priorRecoveries: number;
|
|
40
|
+
reason: string;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export interface ManagedAutopilotDispatchPlan {
|
|
44
|
+
schemaVersion: "witnora.nora_assurance_autopilot.v0.1";
|
|
45
|
+
projectId: string;
|
|
46
|
+
generatedAt: string;
|
|
47
|
+
dispatches: Array<{
|
|
48
|
+
id: string;
|
|
49
|
+
projectId: string;
|
|
50
|
+
taskContractId: string;
|
|
51
|
+
taskContractDigestSha256: string;
|
|
52
|
+
taskKey: string;
|
|
53
|
+
taskVersion: number;
|
|
54
|
+
agentId: string;
|
|
55
|
+
baselineAgentVersion: string;
|
|
56
|
+
candidateAgentVersion: string;
|
|
57
|
+
mode: ManagedWorkflowEvaluationMode;
|
|
58
|
+
changeIds: string[];
|
|
59
|
+
changeSetDigestSha256: string;
|
|
60
|
+
affectedActionPathIds: string[];
|
|
61
|
+
scenarioPriority: NonNullable<ManagedWorkflowReadyEvaluation["scenarioPriority"]>;
|
|
62
|
+
status: "READY" | "RUNNING" | "COMPLETED" | "NEEDS_DECISION" | "BLOCKED";
|
|
63
|
+
}>;
|
|
64
|
+
safety: {
|
|
65
|
+
productionWrites: 0;
|
|
66
|
+
rawInputsRetainedByCustomer: true;
|
|
67
|
+
customerHarnessRequired: true;
|
|
68
|
+
policyAuthorityGranted: false;
|
|
69
|
+
};
|
|
70
|
+
limitations: string[];
|
|
33
71
|
}
|
|
34
72
|
export interface ManagedWorkflowHostedClient {
|
|
35
73
|
listActiveWorkflows(): Promise<Array<{
|
|
@@ -39,8 +77,10 @@ export interface ManagedWorkflowHostedClient {
|
|
|
39
77
|
}>>;
|
|
40
78
|
getExecutionPlan(workflowId: string): Promise<ManagedWorkflowExecutionPlan>;
|
|
41
79
|
getTask(taskContractId: string): Promise<TaskEvaluationContract>;
|
|
80
|
+
getAutopilotPlan?(): Promise<ManagedAutopilotDispatchPlan>;
|
|
42
81
|
upload(report: TaskEvaluationReport, input: {
|
|
43
82
|
externalId: string;
|
|
83
|
+
autopilot?: ManagedAutopilotDispatchPlan["dispatches"][number];
|
|
44
84
|
}): Promise<unknown>;
|
|
45
85
|
}
|
|
46
86
|
export declare function createManagedWorkflowHostedClient(options: {
|
|
@@ -71,6 +111,7 @@ export declare class FileWorkflowEvaluationCheckpointStore implements WorkflowEv
|
|
|
71
111
|
}
|
|
72
112
|
export interface ManagedBusinessWorkflowHarnessResult {
|
|
73
113
|
workflowsInspected: number;
|
|
114
|
+
autopilotDispatchesInspected: number;
|
|
74
115
|
evaluationsCompleted: number;
|
|
75
116
|
evaluationsFailed: number;
|
|
76
117
|
limitations: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"managed-workflow-harness.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/managed-workflow-harness.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGzF,MAAM,MAAM,6BAA6B,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEhE,MAAM,WAAW,4BAA4B;IAC3C,aAAa,EAAE,+CAA+C,CAAC;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAA;KAAE,CAAC;IAC7E,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,GAAG,gBAAgB,GAAG,WAAW,CAAA;KAAE,CAAC,CAAC;IACvG,gBAAgB,EAAE,8BAA8B,EAAE,CAAC;IACnD,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE;QAAE,gBAAgB,EAAE,CAAC,CAAC;QAAC,uBAAuB,EAAE,KAAK,CAAC;QAAC,uBAAuB,EAAE,IAAI,CAAA;KAAE,CAAC;IAC/F,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,wBAAwB,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"managed-workflow-harness.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/managed-workflow-harness.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGzF,MAAM,MAAM,6BAA6B,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEhE,MAAM,WAAW,4BAA4B;IAC3C,aAAa,EAAE,+CAA+C,CAAC;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAA;KAAE,CAAC;IAC7E,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,GAAG,gBAAgB,GAAG,WAAW,CAAA;KAAE,CAAC,CAAC;IACvG,gBAAgB,EAAE,8BAA8B,EAAE,CAAC;IACnD,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE;QAAE,gBAAgB,EAAE,CAAC,CAAC;QAAC,uBAAuB,EAAE,KAAK,CAAC;QAAC,uBAAuB,EAAE,IAAI,CAAA;KAAE,CAAC;IAC/F,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,wBAAwB,EAAE,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,IAAI,EAAE,6BAA6B,CAAC;IACpC,gBAAgB,CAAC,EAAE;QACjB,MAAM,EAAE,iBAAiB,GAAG,gBAAgB,CAAC;QAC7C,sBAAsB,EAAE,MAAM,EAAE,CAAC;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,4BAA4B;IAC3C,aAAa,EAAE,uCAAuC,CAAC;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,wBAAwB,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAC9H,OAAO,EAAE,MAAM,CAAC;QAAC,oBAAoB,EAAE,MAAM,CAAC;QAAC,qBAAqB,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,6BAA6B,CAAC;QAClH,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,qBAAqB,EAAE,MAAM,CAAC;QAAC,qBAAqB,EAAE,MAAM,EAAE,CAAC;QACpF,gBAAgB,EAAE,WAAW,CAAC,8BAA8B,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAClF,MAAM,EAAE,OAAO,GAAG,SAAS,GAAG,WAAW,GAAG,gBAAgB,GAAG,SAAS,CAAC;KAC1E,CAAC,CAAC;IACH,MAAM,EAAE;QAAE,gBAAgB,EAAE,CAAC,CAAC;QAAC,2BAA2B,EAAE,IAAI,CAAC;QAAC,uBAAuB,EAAE,IAAI,CAAC;QAAC,sBAAsB,EAAE,KAAK,CAAA;KAAE,CAAC;IACjI,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,2BAA2B;IAC1C,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAA;KAAE,CAAC,CAAC,CAAC;IAC1G,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAC5E,OAAO,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACjE,gBAAgB,CAAC,IAAI,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAC3D,MAAM,CAAC,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,4BAA4B,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACvJ;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE;IACzD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,GAAG,2BAA2B,CAqD9B;AAED,MAAM,WAAW,iCAAiC;IAChD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC,CAAC;IAC7D,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,qBAAa,uCAAwC,YAAW,iCAAiC;;IAGzF,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC;IAK5D,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAG3C;AAED,qBAAa,qCAAsC,YAAW,iCAAiC;IACjF,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAIxC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC;IAS5D,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9D,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C,OAAO,CAAC,IAAI;CAGb;AAED,MAAM,WAAW,oCAAoC;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,4BAA4B,EAAE,MAAM,CAAC;IACrC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,8BAA8B;;gBAO7B,OAAO,EAAE;QACnB,MAAM,EAAE,2BAA2B,CAAC;QACpC,WAAW,EAAE,iCAAiC,CAAC;QAC/C,QAAQ,EAAE,CAAC,KAAK,EAAE;YAAE,IAAI,EAAE,sBAAsB,CAAC;YAAC,UAAU,EAAE,8BAA8B,CAAA;SAAE,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACjI,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB;IAUD,IAAI,IAAI,OAAO,CAAC,oCAAoC,CAAC;YAMvC,OAAO;CAuFtB"}
|
|
@@ -32,12 +32,21 @@ export function createManagedWorkflowHostedClient(options) {
|
|
|
32
32
|
},
|
|
33
33
|
getExecutionPlan: async (workflowId) => await hostedJson(requestFetch, `${projectUrl}/business-workflows/${encodeURIComponent(required(workflowId, "workflowId"))}/execution-plan`, apiKey),
|
|
34
34
|
getTask: async (taskContractId) => await hostedJson(requestFetch, `${projectUrl}/business-tasks/${encodeURIComponent(required(taskContractId, "taskContractId"))}`, apiKey),
|
|
35
|
+
getAutopilotPlan: async () => await hostedJson(requestFetch, `${projectUrl}/nora/autopilot`, apiKey),
|
|
35
36
|
upload: async (report, input) => await uploadTaskEvaluationReport(report, {
|
|
36
37
|
baseUrl,
|
|
37
38
|
projectId,
|
|
38
39
|
apiKey,
|
|
39
40
|
externalId: input.externalId,
|
|
40
41
|
fetch: requestFetch,
|
|
42
|
+
...(input.autopilot ? { autopilot: {
|
|
43
|
+
dispatchId: input.autopilot.id,
|
|
44
|
+
changeSetDigestSha256: input.autopilot.changeSetDigestSha256,
|
|
45
|
+
candidateAgentVersion: input.autopilot.candidateAgentVersion,
|
|
46
|
+
baselineAgentVersion: input.autopilot.baselineAgentVersion,
|
|
47
|
+
changeIds: input.autopilot.changeIds,
|
|
48
|
+
affectedActionPathIds: input.autopilot.affectedActionPathIds,
|
|
49
|
+
} } : {}),
|
|
41
50
|
}),
|
|
42
51
|
};
|
|
43
52
|
}
|
|
@@ -153,9 +162,82 @@ export class ManagedBusinessWorkflowHarness {
|
|
|
153
162
|
break;
|
|
154
163
|
}
|
|
155
164
|
}
|
|
156
|
-
|
|
165
|
+
const autopilot = this.#client.getAutopilotPlan ? await this.#client.getAutopilotPlan() : {
|
|
166
|
+
schemaVersion: "witnora.nora_assurance_autopilot.v0.1",
|
|
167
|
+
projectId: "legacy-client",
|
|
168
|
+
generatedAt: new Date(0).toISOString(),
|
|
169
|
+
dispatches: [],
|
|
170
|
+
safety: { productionWrites: 0, rawInputsRetainedByCustomer: true, customerHarnessRequired: true, policyAuthorityGranted: false },
|
|
171
|
+
limitations: ["This client does not expose Nora Autopilot dispatches."],
|
|
172
|
+
};
|
|
173
|
+
validateAutopilotPlan(autopilot);
|
|
174
|
+
const readyDispatches = autopilot.dispatches.filter((dispatch) => dispatch.status === "READY");
|
|
175
|
+
const autopilotResults = await mapLimit(readyDispatches, this.#maxConcurrency, async (dispatch) => {
|
|
176
|
+
const evaluation = {
|
|
177
|
+
nodeId: `autopilot:${dispatch.id}`,
|
|
178
|
+
taskContractId: dispatch.taskContractId,
|
|
179
|
+
taskContractDigestSha256: dispatch.taskContractDigestSha256,
|
|
180
|
+
taskKey: dispatch.taskKey,
|
|
181
|
+
agentId: dispatch.agentId,
|
|
182
|
+
agentVersion: dispatch.candidateAgentVersion,
|
|
183
|
+
baselineAgentVersion: dispatch.baselineAgentVersion,
|
|
184
|
+
mode: dispatch.mode,
|
|
185
|
+
scenarioPriority: structuredClone(dispatch.scenarioPriority),
|
|
186
|
+
};
|
|
187
|
+
const key = `autopilot:${autopilot.projectId}:${dispatch.id}`;
|
|
188
|
+
try {
|
|
189
|
+
const task = await this.#client.getTask(dispatch.taskContractId);
|
|
190
|
+
validateTask(task, evaluation);
|
|
191
|
+
let report = await this.#checkpoints.load(key);
|
|
192
|
+
if (!report) {
|
|
193
|
+
report = await this.#evaluate({ task: structuredClone(task), evaluation: structuredClone(evaluation) });
|
|
194
|
+
validateReport(report, task, evaluation);
|
|
195
|
+
await this.#checkpoints.save(key, report);
|
|
196
|
+
}
|
|
197
|
+
else
|
|
198
|
+
validateReport(report, task, evaluation);
|
|
199
|
+
await this.#client.upload(report, { externalId: `autopilot:${dispatch.id}`, autopilot: dispatch });
|
|
200
|
+
await this.#checkpoints.complete(key);
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
limitations.push(`Autopilot evaluation ${dispatch.taskKey}:${dispatch.mode} failed closed; no policy or release state changed.`);
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
evaluationsCompleted += autopilotResults.filter(Boolean).length;
|
|
209
|
+
evaluationsFailed += autopilotResults.filter((value) => !value).length;
|
|
210
|
+
return { workflowsInspected: workflows.length, autopilotDispatchesInspected: readyDispatches.length, evaluationsCompleted, evaluationsFailed, limitations };
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function validateAutopilotPlan(plan) {
|
|
214
|
+
if (plan.schemaVersion !== "witnora.nora_assurance_autopilot.v0.1"
|
|
215
|
+
|| plan.safety.productionWrites !== 0 || plan.safety.rawInputsRetainedByCustomer !== true
|
|
216
|
+
|| plan.safety.customerHarnessRequired !== true || plan.safety.policyAuthorityGranted !== false
|
|
217
|
+
|| plan.dispatches.length > 200)
|
|
218
|
+
throw new Error("Managed Workflow Harness rejected an unsafe Nora Autopilot plan.");
|
|
219
|
+
const seen = new Set();
|
|
220
|
+
for (const dispatch of plan.dispatches) {
|
|
221
|
+
if (dispatch.projectId !== plan.projectId || seen.has(dispatch.id) || !/^[a-f0-9]{64}$/.test(dispatch.id)
|
|
222
|
+
|| !/^[a-f0-9]{64}$/.test(dispatch.taskContractDigestSha256) || !/^[a-f0-9]{64}$/.test(dispatch.changeSetDigestSha256)
|
|
223
|
+
|| (dispatch.mode !== "REPLAY" && dispatch.mode !== "SHADOW") || !dispatch.agentId || !dispatch.baselineAgentVersion || !dispatch.candidateAgentVersion
|
|
224
|
+
|| !Array.isArray(dispatch.changeIds) || !dispatch.changeIds.length || !Array.isArray(dispatch.affectedActionPathIds)
|
|
225
|
+
|| !validScenarioPriority(dispatch.scenarioPriority)) {
|
|
226
|
+
throw new Error("Managed Workflow Harness rejected an invalid Nora Autopilot dispatch.");
|
|
227
|
+
}
|
|
228
|
+
seen.add(dispatch.id);
|
|
157
229
|
}
|
|
158
230
|
}
|
|
231
|
+
function validScenarioPriority(value) {
|
|
232
|
+
return Boolean(value
|
|
233
|
+
&& (value.source === "PROJECT_HISTORY" || value.source === "APPROVED_SUITE")
|
|
234
|
+
&& Array.isArray(value.prioritizedScenarioIds)
|
|
235
|
+
&& value.prioritizedScenarioIds.length <= 100
|
|
236
|
+
&& new Set(value.prioritizedScenarioIds).size === value.prioritizedScenarioIds.length
|
|
237
|
+
&& value.prioritizedScenarioIds.every((id) => typeof id === "string" && id.length > 0 && id.length <= 200)
|
|
238
|
+
&& [value.priorEvaluations, value.priorFailures, value.priorRecoveries].every((count) => Number.isSafeInteger(count) && count >= 0)
|
|
239
|
+
&& typeof value.reason === "string" && value.reason.length > 0 && value.reason.length <= 500);
|
|
240
|
+
}
|
|
159
241
|
function validatePlan(plan, workflow) {
|
|
160
242
|
if (plan.schemaVersion !== "witnora.business_workflow_execution_plan.v0.1"
|
|
161
243
|
|| plan.workflow.id !== workflow.id
|
|
@@ -190,7 +272,7 @@ function validateTask(task, evaluation) {
|
|
|
190
272
|
|| task.digestSha256 !== evaluation.taskContractDigestSha256
|
|
191
273
|
|| task.taskKey !== evaluation.taskKey
|
|
192
274
|
|| task.subject.agentId !== evaluation.agentId
|
|
193
|
-
|| task.subject.agentVersion !== evaluation.agentVersion) {
|
|
275
|
+
|| task.subject.agentVersion !== (evaluation.baselineAgentVersion ?? evaluation.agentVersion)) {
|
|
194
276
|
throw new Error("Managed Workflow Harness rejected a Business Task that did not match the execution plan.");
|
|
195
277
|
}
|
|
196
278
|
}
|
|
@@ -5,6 +5,14 @@ export interface TaskEvaluationHostedOptions {
|
|
|
5
5
|
apiKey: string;
|
|
6
6
|
externalId?: string;
|
|
7
7
|
fetch?: typeof fetch;
|
|
8
|
+
autopilot?: {
|
|
9
|
+
dispatchId: string;
|
|
10
|
+
changeSetDigestSha256: string;
|
|
11
|
+
candidateAgentVersion: string;
|
|
12
|
+
baselineAgentVersion: string;
|
|
13
|
+
changeIds: string[];
|
|
14
|
+
affectedActionPathIds: string[];
|
|
15
|
+
};
|
|
8
16
|
}
|
|
9
17
|
export interface TaskEvaluationHostedResult {
|
|
10
18
|
run: Record<string, unknown>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task-evaluation-hosted.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/task-evaluation-hosted.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"task-evaluation-hosted.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/task-evaluation-hosted.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,SAAS,CAAC,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,qBAAqB,EAAE,MAAM,EAAE,CAAC;KACjC,CAAC;CACH;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,0BAA0B,CAAC,CAuErC;AAED,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA+B1H"}
|
|
@@ -26,7 +26,21 @@ export async function uploadTaskEvaluationReport(report, options) {
|
|
|
26
26
|
mode: report.mode,
|
|
27
27
|
writesBlocked: true,
|
|
28
28
|
rawInputsRetainedByCustomer: true,
|
|
29
|
-
|
|
29
|
+
...(options.autopilot ? {
|
|
30
|
+
schemaVersion: "witnora.task_evaluation_binding.v0.2",
|
|
31
|
+
baselineAgentVersion: options.autopilot.baselineAgentVersion,
|
|
32
|
+
candidateAgentVersion: options.autopilot.candidateAgentVersion,
|
|
33
|
+
} : {}),
|
|
34
|
+
}, ...(options.autopilot ? { noraAutopilot: {
|
|
35
|
+
schemaVersion: "witnora.nora_autopilot_evaluation_binding.v0.1",
|
|
36
|
+
dispatchId: options.autopilot.dispatchId,
|
|
37
|
+
changeSetDigestSha256: options.autopilot.changeSetDigestSha256,
|
|
38
|
+
taskContractId: report.task.id,
|
|
39
|
+
candidateAgentVersion: options.autopilot.candidateAgentVersion,
|
|
40
|
+
mode: report.mode,
|
|
41
|
+
changeIds: [...options.autopilot.changeIds],
|
|
42
|
+
affectedActionPathIds: [...options.autopilot.affectedActionPathIds],
|
|
43
|
+
} } : {}) },
|
|
30
44
|
}),
|
|
31
45
|
});
|
|
32
46
|
if (typeof run.id !== "string" || !run.id)
|