witnora 0.13.6 → 0.13.8

package/dist/vendor/onegent-runtime/task-evaluation.d.ts

@@ -0,0 +1,123 @@
+import type { SandboxCertificationHarness } from "./sandbox-harness.js";
+import type { CreateActionIntentInput } from "./types.js";
+export declare const TASK_EVALUATION_SCHEMA_VERSION: "witnora.task_evaluation.v0.1";
+export interface TaskEvaluationContract {
+    id: string;
+    digestSha256: string;
+    taskKey: string;
+    version: number;
+    subject: {
+        agentId: string;
+        agentVersion: string;
+    };
+    scenarioSuite: {
+        id: string;
+        version: string;
+        sha256: string;
+    };
+    actionPaths: Array<{
+        id: string;
+        actionType: "SUBMIT" | "PAY" | "SEND" | "UPDATE";
+        targetSystem: string;
+        requiredPermissions: string[];
+        customerDecision?: "ALLOW" | "REQUIRE_APPROVAL" | "DENY";
+    }>;
+    successCriteria: Array<{
+        id: string;
+        evaluator: "DETERMINISTIC" | "READ_ONLY_PROBE";
+        predicateDigestSha256: string;
+    }>;
+}
+export interface TaskEvaluationIntent {
+    pathId: string;
+    parametersDigestSha256: string;
+    action?: CreateActionIntentInput;
+}
+export interface TaskEvaluationScenario {
+    id: string;
+    source: "HISTORICAL" | "FIXTURE" | "LIVE_SHADOW";
+    sanitized: boolean;
+    input: unknown;
+    inputDigestSha256: string;
+    baseline: {
+        resultDigestSha256: string;
+        actionIntents: Array<Omit<TaskEvaluationIntent, "action">>;
+    };
+}
+export interface TaskCandidateEvaluation {
+    resultDigestSha256: string;
+    actionIntents: TaskEvaluationIntent[];
+    criteria: Array<{
+        id: string;
+        passed: boolean;
+    }>;
+}
+export interface ShadowCandidateEvaluation {
+    resultDigestSha256: string;
+    criteria: Array<{
+        id: string;
+        passed: boolean;
+    }>;
+}
+export interface TaskEvaluationReport {
+    schemaVersion: typeof TASK_EVALUATION_SCHEMA_VERSION;
+    mode: "REPLAY" | "SHADOW";
+    classification: "REPLAY_SANDBOX_ONLY" | "SHADOW_NO_WRITE";
+    generatedAt: string;
+    task: {
+        id: string;
+        digestSha256: string;
+        taskKey: string;
+        version: number;
+        subject: TaskEvaluationContract["subject"];
+        scenarioSuite: TaskEvaluationContract["scenarioSuite"];
+    };
+    verdict: "PASSED" | "MATCHED" | "CHANGED" | "BLOCKED";
+    summary: {
+        scenarios: number;
+        matched: number;
+        changed: number;
+        blocked: number;
+        sandboxActions: number;
+        productionWrites: 0;
+    };
+    scenarios: Array<{
+        id: string;
+        source: TaskEvaluationScenario["source"];
+        inputDigestSha256: string;
+        matched: boolean;
+        blocked: boolean;
+        sandboxActionCount: number;
+        criteria: Array<{
+            id: string;
+            status: "PASSED" | "FAILED" | "NOT_OBSERVED";
+            evaluator: "DETERMINISTIC" | "READ_ONLY_PROBE";
+        }>;
+        differences: string[];
+    }>;
+    limitations: string[];
+}
+export declare function runReplayEvaluation(options: {
+    task: TaskEvaluationContract;
+    harness: SandboxCertificationHarness;
+    tenantId: string;
+    scenarios: TaskEvaluationScenario[];
+    evaluateCandidate: (scenario: {
+        id: string;
+        source: TaskEvaluationScenario["source"];
+        input: unknown;
+    }) => Promise<TaskCandidateEvaluation>;
+    now?: () => Date;
+}): Promise<TaskEvaluationReport>;
+export declare function runShadowEvaluation(options: {
+    task: TaskEvaluationContract;
+    observations: TaskEvaluationScenario[];
+    evaluateCandidate: (context: {
+        id: string;
+        source: "LIVE_SHADOW";
+        input: unknown;
+        propose: (intent: Omit<TaskEvaluationIntent, "action">) => void;
+    }) => Promise<ShadowCandidateEvaluation>;
+    now?: () => Date;
+}): Promise<TaskEvaluationReport>;
+//# sourceMappingURL=task-evaluation.d.ts.map

package/dist/vendor/onegent-runtime/task-evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-evaluation.d.ts","sourceRoot":"","sources":["../../../../onegent-runtime/src/task-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,eAAO,MAAM,8BAA8B,EAAG,8BAAuC,CAAC;AAEtF,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,aAAa,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,WAAW,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,UAAU,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;QACjD,YAAY,EAAE,MAAM,CAAC;QACrB,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAC9B,gBAAgB,CAAC,EAAE,OAAO,GAAG,kBAAkB,GAAG,MAAM,CAAC;KAC1D,CAAC,CAAC;IACH,eAAe,EAAE,KAAK,CAAC;QACrB,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,eAAe,GAAG,iBAAiB,CAAC;QAC/C,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB,EAAE,MAAM,CAAC;IAC/B,MAAM,CAAC,EAAE,uBAAuB,CAAC;CAClC;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,aAAa,CAAC;IACjD,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE;QACR,kBAAkB,EAAE,MAAM,CAAC;QAC3B,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC;KAC5D,CAAC;CACH;AAED,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,yBAAyB;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,OAAO,8BAA8B,CAAC;IACrD,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,cAAc,EAAE,qBAAqB,GAAG,iBAAiB,CAAC;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE;QACJ,EAAE,EAAE,MAAM,CAAC;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3C,aAAa,EAAE,sBAAsB,CAAC,eAAe,CAAC,CAAC;KACxD,CAAC;IACF,OAAO,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IACtD,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,CAAC,CAAC;KACrB,CAAC;IACF,SAAS,EAAE,KAAK,CAAC;QACf,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACzC,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;QACjB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,QAAQ,EAAE,KAAK,CAAC;YACd,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,cAAc,CAAC;YAC7C,SAAS,EAAE,eAAe,GAAG,iBAAiB,CAAC;SAChD,CAAC,CAAC;QACH,WAAW,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC,CAAC;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,2BAA2B,CAAC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,sBAAsB,EAAE,CAAC;IACpC,iBAAiB,EAAE,CAAC,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC5I,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAoDhC;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,IAAI,EAAE,sBAAsB,CAAC;IAC7B,YAAY,EAAE,sBAAsB,EAAE,CAAC;IACvC,iBAAiB,EAAE,CAAC,OAAO,EAAE;QAC3B,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,aAAa,CAAC;QACtB,KAAK,EAAE,OAAO,CAAC;QACf,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,oBAAoB,EAAE,QAAQ,CAAC,KAAK,IAAI,CAAC;KACjE,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACzC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAgDhC"}

package/dist/vendor/onegent-runtime/task-evaluation.js

@@ -0,0 +1,206 @@
+export const TASK_EVALUATION_SCHEMA_VERSION = "witnora.task_evaluation.v0.1";
+export async function runReplayEvaluation(options) {
+    validateTask(options.task);
+    validateScenarios(options.scenarios, false);
+    const now = options.now ?? (() => new Date());
+    const run = await options.harness.startRun({ tenantId: options.tenantId });
+    const results = [];
+    for (const scenario of options.scenarios) {
+        await options.harness.resetTenant(options.tenantId);
+        const candidate = await options.evaluateCandidate({ id: scenario.id, source: scenario.source, input: scenario.input });
+        validateCandidate(candidate, options.task);
+        const differences = compareCandidate(scenario, candidate);
+        let blocked = false;
+        let sandboxActionCount = 0;
+        for (const intent of candidate.actionIntents) {
+            const path = options.task.actionPaths.find((item) => item.id === intent.pathId);
+            if (!path || !intent.action || !actionMatchesPath(intent.action, path)) {
+                blocked = true;
+                differences.push(`Action path ${intent.pathId} was not bound to the approved sandbox contract.`);
+                continue;
+            }
+            const execution = await run.executeAction(intent.action, {
+                approval: { approved: true, reviewerId: `business-task:${options.task.id}`, comment: "Approved synthetic Replay path from the customer Business Task Contract." },
+                rollbackAfterVerification: true,
+                rollbackReason: "Replay restores the synthetic tenant after each observed action.",
+            });
+            if (execution.status !== "verified" && execution.status !== "rolled_back") {
+                blocked = true;
+                differences.push(`Sandbox action ${intent.pathId} was ${execution.status}${execution.rejectionCode ? ` (${execution.rejectionCode})` : ""}.`);
+            }
+            else {
+                sandboxActionCount += 1;
+            }
+        }
+        const criteria = options.task.successCriteria.map((criterion) => ({
+            id: criterion.id,
+            evaluator: criterion.evaluator,
+            status: candidate.criteria.find((item) => item.id === criterion.id)?.passed === true ? "PASSED" : "FAILED",
+        }));
+        if (criteria.some((item) => item.status !== "PASSED"))
+            differences.push("One or more declared success criteria did not pass.");
+        const matched = !blocked && differences.length === 0;
+        results.push({
+            id: scenario.id,
+            source: scenario.source,
+            inputDigestSha256: scenario.inputDigestSha256,
+            matched,
+            blocked,
+            sandboxActionCount,
+            criteria,
+            differences: [...new Set(differences)],
+        });
+    }
+    run.complete();
+    return report("REPLAY", "REPLAY_SANDBOX_ONLY", options.task, results, now());
+}
+export async function runShadowEvaluation(options) {
+    validateTask(options.task);
+    validateScenarios(options.observations, true);
+    const now = options.now ?? (() => new Date());
+    const results = [];
+    for (const observation of options.observations) {
+        const proposals = [];
+        const candidate = await options.evaluateCandidate({
+            id: observation.id,
+            source: "LIVE_SHADOW",
+            input: observation.input,
+            propose: (intent) => {
+                digest(intent.parametersDigestSha256, `candidate action ${intent.pathId} digest`);
+                if (!options.task.actionPaths.some((path) => path.id === intent.pathId)) {
+                    throw new Error(`Candidate proposed undeclared action path ${intent.pathId}.`);
+                }
+                proposals.push(structuredClone(intent));
+            },
+        });
+        validateShadowCandidate(candidate, options.task);
+        const candidateForComparison = {
+            resultDigestSha256: candidate.resultDigestSha256,
+            actionIntents: proposals,
+            criteria: candidate.criteria,
+        };
+        const differences = compareCandidate(observation, candidateForComparison);
+        const criteria = options.task.successCriteria.map((criterion) => ({
+            id: criterion.id,
+            evaluator: criterion.evaluator,
+            status: criterion.evaluator === "READ_ONLY_PROBE"
+                ? "NOT_OBSERVED"
+                : candidate.criteria.find((item) => item.id === criterion.id)?.passed === true
+                    ? "PASSED"
+                    : "FAILED",
+        }));
+        if (criteria.some((item) => item.status === "FAILED"))
+            differences.push("One or more deterministic success criteria did not pass.");
+        results.push({
+            id: observation.id,
+            source: observation.source,
+            inputDigestSha256: observation.inputDigestSha256,
+            matched: differences.length === 0,
+            blocked: false,
+            sandboxActionCount: 0,
+            criteria,
+            differences: [...new Set(differences)],
+        });
+    }
+    return report("SHADOW", "SHADOW_NO_WRITE", options.task, results, now());
+}
+function report(mode, classification, task, scenarios, generatedAt) {
+    const blocked = scenarios.filter((item) => item.blocked).length;
+    const matched = scenarios.filter((item) => item.matched).length;
+    const changed = scenarios.length - matched - blocked;
+    return {
+        schemaVersion: TASK_EVALUATION_SCHEMA_VERSION,
+        mode,
+        classification,
+        generatedAt: generatedAt.toISOString(),
+        task: {
+            id: task.id,
+            digestSha256: task.digestSha256,
+            taskKey: task.taskKey,
+            version: task.version,
+            subject: structuredClone(task.subject),
+            scenarioSuite: structuredClone(task.scenarioSuite),
+        },
+        verdict: blocked ? "BLOCKED" : changed ? "CHANGED" : mode === "SHADOW" ? "MATCHED" : "PASSED",
+        summary: {
+            scenarios: scenarios.length,
+            matched,
+            changed,
+            blocked,
+            sandboxActions: scenarios.reduce((sum, item) => sum + item.sandboxActionCount, 0),
+            productionWrites: 0,
+        },
+        scenarios,
+        limitations: mode === "REPLAY"
+            ? [
+                "Replay covers only the declared scenarios and network-denied synthetic or sanitized sandbox paths.",
+                "Replay results do not establish production enforcement, production outcome verification, or future Agent reliability.",
+            ]
+            : [
+                "Shadow compares candidate decisions against observed inputs while candidate writes remain blocked.",
+                "Shadow results do not establish that the candidate produced a real production outcome.",
+            ],
+    };
+}
+function compareCandidate(scenario, candidate) {
+    const differences = [];
+    if (candidate.resultDigestSha256 !== scenario.baseline.resultDigestSha256)
+        differences.push("Candidate result digest differs from the baseline.");
+    const expected = scenario.baseline.actionIntents.map(intentFingerprint).sort();
+    const observed = candidate.actionIntents.map(intentFingerprint).sort();
+    if (JSON.stringify(expected) !== JSON.stringify(observed))
+        differences.push("Candidate action intents differ from the baseline.");
+    return differences;
+}
+function intentFingerprint(intent) {
+    return `${intent.pathId}:${intent.parametersDigestSha256}`;
+}
+function actionMatchesPath(action, path) {
+    const permissions = [...new Set(action.requestedPermissions ?? [])].sort();
+    return path.customerDecision !== "DENY"
+        && action.environment !== "production"
+        && !action.targetUrl
+        && action.actionType === path.actionType
+        && action.targetSystem === path.targetSystem
+        && JSON.stringify(permissions) === JSON.stringify([...new Set(path.requiredPermissions)].sort());
+}
+function validateTask(task) {
+    digest(task.digestSha256, "task.digestSha256");
+    digest(task.scenarioSuite.sha256, "task.scenarioSuite.sha256");
+    if (!task.actionPaths.length || !task.successCriteria.length)
+        throw new Error("Task evaluation requires action paths and success criteria.");
+    for (const criterion of task.successCriteria)
+        digest(criterion.predicateDigestSha256, `criterion ${criterion.id}`);
+}
+function validateScenarios(scenarios, shadow) {
+    if (!scenarios.length || scenarios.length > 500)
+        throw new Error("Task evaluation requires 1 to 500 scenarios.");
+    for (const scenario of scenarios) {
+        if (!shadow && scenario.source === "LIVE_SHADOW")
+            throw new Error("Replay cannot consume a LIVE_SHADOW scenario.");
+        if (shadow && scenario.source !== "LIVE_SHADOW")
+            throw new Error("Shadow requires LIVE_SHADOW observations.");
+        if (scenario.sanitized !== true)
+            throw new Error(`Scenario ${scenario.id} must be explicitly sanitized.`);
+        digest(scenario.inputDigestSha256, `scenario ${scenario.id} input digest`);
+        digest(scenario.baseline.resultDigestSha256, `scenario ${scenario.id} baseline result digest`);
+        scenario.baseline.actionIntents.forEach((intent) => digest(intent.parametersDigestSha256, `scenario ${scenario.id} baseline action digest`));
+    }
+}
+function validateShadowCandidate(candidate, task) {
+    digest(candidate.resultDigestSha256, "candidate result digest");
+    const criterionIds = new Set(task.successCriteria.map((item) => item.id));
+    if (candidate.criteria.some((item) => !criterionIds.has(item.id)))
+        throw new Error("Candidate returned an undeclared success criterion.");
+}
+function validateCandidate(candidate, task) {
+    digest(candidate.resultDigestSha256, "candidate result digest");
+    candidate.actionIntents.forEach((intent) => digest(intent.parametersDigestSha256, `candidate action ${intent.pathId} digest`));
+    const criterionIds = new Set(task.successCriteria.map((item) => item.id));
+    if (candidate.criteria.some((item) => !criterionIds.has(item.id)))
+        throw new Error("Candidate returned an undeclared success criterion.");
+}
+function digest(value, name) {
+    if (!/^[a-f0-9]{64}$/.test(value))
+        throw new Error(`${name} must be a lowercase SHA-256 digest.`);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "witnora",
-  "version": "0.13.6",
+  "version": "0.13.8",
   "description": "Independent assurance for covered agent action paths across models and frameworks.",
   "type": "module",
   "license": "Apache-2.0",