witnora 0.8.1 → 0.9.0

package/dist/runner.js

@@ -0,0 +1,349 @@
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { renderAgentCertBadge } from "./badge.js";
+import { dirname, join, resolve } from "node:path";
+import { buildEvidenceBundle } from "./bundle.js";
+import { recordsFromAgentCertResult, renderCorpusSummary, summarizeCorpus, writeReviewedFailureDataset, } from "./corpus.js";
+import { openCorpusStore, parseCorpusStoreKind } from "./corpus-store.js";
+import { applyFailureReviews, readFailureReviews } from "./failure-review.js";
+import { governCorpusRecords } from "./corpus-governance.js";
+import { buildMonitorSnapshot, writeMonitorSnapshot } from "./monitor.js";
+import { normalizeMcpBenchResult, normalizeOnegentAuditPacket, normalizeTripwireResult } from "./normalizers.js";
+import { renderHtmlReport, renderMarkdownReport } from "./report.js";
+const JOB_KEYS = ["mcpbench", "tripwire", "onegent"];
+export function publicDemoRunProfile() {
+    return {
+        schemaVersion: "1",
+        subject: {
+            name: "agentcert-public-demo",
+            type: "agent",
+        },
+        artifacts: {
+            mcpbench: "public-demo/lifecycle-evidence/mcpbench-passing/results.json",
+            tripwire: "public-demo/browser-agent-robustness/evidence/tripwire-public-demo/tripwire-result.json",
+            onegent: "public-demo/lifecycle-evidence/onegent-procurement/audit-packet.json",
+        },
+        outputDir: "public-demo/browser-agent-robustness/evidence/agentcert-public-demo",
+        run: {
+            corpus: {
+                path: "public-demo/browser-agent-robustness/evidence/agentcert-corpus.jsonl",
+                reviewsPath: "public-demo/browser-agent-robustness/evidence/failure-reviews.jsonl",
+                replace: true,
+            },
+            monitor: {
+                outputs: [
+                    "packages/agentcert-dashboard/public/data/monitor.json",
+                    "public-demo/agentcert-monitor/data/monitor.json",
+                ],
+                detailUrl: "../browser-agent-robustness/",
+            },
+            dataset: {
+                reviewedOutputs: ["public-demo/browser-agent-robustness/evidence/reviewed-failure-dataset.jsonl"],
+            },
+            gate: {
+                failOnVerdict: false,
+            },
+        },
+    };
+}
+export async function loadRunProfile(profileName, configPath) {
+    if (profileName === "public-demo") {
+        return publicDemoRunProfile();
+    }
+    if (profileName && profileName !== "public-demo") {
+        throw new Error(`Unknown Witnora run profile "${profileName}". Available profiles: public-demo.`);
+    }
+    if (configPath) {
+        return (await readJson(configPath));
+    }
+    const defaultConfigPath = "witnora.config.json";
+    try {
+        await access(resolve(defaultConfigPath));
+        return (await readJson(defaultConfigPath));
+    }
+    catch {
+        const legacyConfigPath = "agentcert.config.json";
+        try {
+            await access(resolve(legacyConfigPath));
+            return (await readJson(legacyConfigPath));
+        }
+        catch {
+            throw new Error("witnora run requires --profile public-demo, --config <path>, explicit artifact flags, witnora.config.json, or the legacy agentcert.config.json in the current directory.");
+        }
+    }
+}
+export function profileFromArtifactFlags(overrides) {
+    return {
+        schemaVersion: "1",
+        subject: {
+            name: overrides.subject ?? "witnora-subject",
+            type: "agent",
+        },
+        artifacts: {
+            mcpbench: overrides.mcpbench,
+            tripwire: overrides.tripwire,
+            onegent: overrides.onegent,
+        },
+        outputDir: overrides.outDir ?? ".witnora/latest",
+        run: {
+            corpus: {
+                path: overrides.corpusPath ?? ".witnora/corpus/corpus.jsonl",
+                replace: overrides.replaceCorpus ?? false,
+            },
+            monitor: {
+                outputs: [overrides.monitorOut ?? ".witnora/latest/monitor.json"],
+            },
+            dataset: {
+                reviewedOutputs: [overrides.reviewedDatasetOut ?? ".witnora/latest/reviewed-failure-dataset.jsonl"],
+            },
+            gate: {
+                failOnVerdict: overrides.failOnVerdict ?? false,
+            },
+        },
+    };
+}
+export function applyRunOverrides(profile, overrides = {}) {
+    const next = {
+        ...profile,
+        subject: {
+            ...profile.subject,
+            name: overrides.subject ?? profile.subject.name,
+        },
+        artifacts: {
+            ...profile.artifacts,
+            ...(overrides.mcpbench ? { mcpbench: overrides.mcpbench } : {}),
+            ...(overrides.tripwire ? { tripwire: overrides.tripwire } : {}),
+            ...(overrides.onegent ? { onegent: overrides.onegent } : {}),
+        },
+        outputDir: overrides.outDir ?? profile.outputDir,
+        run: {
+            ...profile.run,
+            corpus: {
+                ...profile.run?.corpus,
+                ...(overrides.corpusPath ? { path: overrides.corpusPath } : {}),
+                ...(overrides.reviewsPath ? { reviewsPath: overrides.reviewsPath } : {}),
+                ...(overrides.replaceCorpus === undefined ? {} : { replace: overrides.replaceCorpus }),
+            },
+            monitor: {
+                ...profile.run?.monitor,
+                ...(overrides.monitorOut ? { outputs: [overrides.monitorOut] } : {}),
+            },
+            dataset: {
+                ...profile.run?.dataset,
+                ...(overrides.reviewedDatasetOut ? { reviewedOutputs: [overrides.reviewedDatasetOut] } : {}),
+            },
+            gate: {
+                ...profile.run?.gate,
+                ...(overrides.failOnVerdict === undefined ? {} : { failOnVerdict: overrides.failOnVerdict }),
+            },
+        },
+    };
+    return next;
+}
+export async function runAgentCertProfile(profileInput, options = {}) {
+    const cwd = resolve(options.cwd ?? process.cwd());
+    const profile = applyRunOverrides(profileInput, options.overrides);
+    const steps = [];
+    const loaded = [];
+    for (const key of JOB_KEYS) {
+        const job = jobFor(profile, key);
+        if (!job.artifact && !job.command) {
+            steps.push({ id: key, status: "skipped", message: "No command or artifact configured." });
+            continue;
+        }
+        if (job.command && options.runCommands !== false) {
+            const step = await runJobCommand(key, job.command, cwd, options.commandStdio ?? "inherit");
+            steps.push(step);
+            if (step.status === "failed" && !job.allowCommandFailure) {
+                throw new Error(`${key} command failed with exit code ${step.exitCode ?? "unknown"}.`);
+            }
+        }
+        if (!job.artifact) {
+            if (job.required !== false) {
+                throw new Error(`${key} did not define an artifact path.`);
+            }
+            steps.push({ id: key, status: "skipped", message: "No artifact configured." });
+            continue;
+        }
+        const raw = await readJson(job.artifact, cwd);
+        loaded.push({ key, path: job.artifact, raw, result: normalizeArtifact(key, raw, job.artifact) });
+        steps.push({ id: key, status: "passed", artifactPath: job.artifact });
+    }
+    if (loaded.length === 0) {
+        throw new Error("No Witnora artifacts were loaded. Configure at least one MCPBench, Tripwire, or Onegent artifact.");
+    }
+    const bundle = buildEvidenceBundle(loaded.map((artifact) => artifact.result), profile.subject.name, profile.subject.type);
+    const reviewsPath = profile.run?.corpus?.reviewsPath;
+    const reviews = await readFailureReviews(reviewsPath ? resolve(cwd, reviewsPath) : undefined);
+    const records = governCorpusRecords(applyFailureReviews(loaded.flatMap(({ result, path, raw }) => recordsFromAgentCertResult(result, path, profile.subject.name, raw)), reviews), { consent: "private", consentSource: "agentcert-runner-default" });
+    const outputs = { monitor: [], reviewedDataset: [] };
+    const reportEnabled = profile.run?.report?.enabled ?? true;
+    const reportDir = profile.run?.report?.outDir ?? profile.outputDir;
+    if (reportEnabled && reportDir) {
+        const resolvedReportDir = resolve(cwd, reportDir);
+        await mkdir(resolvedReportDir, { recursive: true });
+        await writeFile(`${resolvedReportDir}/agentcert-evidence.json`, `${JSON.stringify(bundle, null, 2)}\n`);
+        await writeFile(`${resolvedReportDir}/agentcert-report.md`, renderMarkdownReport(bundle));
+        await writeFile(`${resolvedReportDir}/agentcert-report.html`, renderHtmlReport(bundle));
+        await writeFile(`${resolvedReportDir}/badge.svg`, renderAgentCertBadge(bundle));
+        steps.push({ id: "report", status: "passed", artifactPath: reportDir });
+        outputs.reportDir = reportDir;
+        outputs.evidenceBundle = artifactPath(reportDir, "agentcert-evidence.json");
+        outputs.markdownReport = artifactPath(reportDir, "agentcert-report.md");
+        outputs.htmlReport = artifactPath(reportDir, "agentcert-report.html");
+        outputs.badge = artifactPath(reportDir, "badge.svg");
+    }
+    else {
+        steps.push({ id: "report", status: "skipped", message: "Report output disabled." });
+    }
+    const corpusOptions = corpusStoreOptions(profile, cwd);
+    const store = await openCorpusStore(corpusOptions);
+    try {
+        await store.append(records, { replace: profile.run?.corpus?.replace ?? false });
+        const allRecords = await store.readAll();
+        steps.push({ id: "corpus", status: "passed", artifactPath: profile.run?.corpus?.path ?? corpusOptions.jsonlPath });
+        outputs.corpus = profile.run?.corpus?.path ?? corpusOptions.jsonlPath ?? store.description;
+        const monitorOutputs = monitorOutputsFor(profile);
+        for (const outPath of monitorOutputs) {
+            const snapshot = buildMonitorSnapshot(allRecords, {
+                subject: profile.run?.monitor?.subject ?? profile.subject.name,
+                detailUrl: profile.run?.monitor?.detailUrl,
+            });
+            await writeMonitorSnapshot(resolve(cwd, outPath), snapshot);
+            outputs.monitor.push(outPath);
+        }
+        const reviewedDatasetOutputs = reviewedDatasetOutputsFor(profile);
+        for (const outPath of reviewedDatasetOutputs) {
+            await writeReviewedFailureDataset(resolve(cwd, outPath), allRecords);
+            outputs.reviewedDataset.push(outPath);
+        }
+        steps.push({
+            id: "dataset",
+            status: reviewedDatasetOutputs.length > 0 ? "passed" : "skipped",
+            artifactPath: reviewedDatasetOutputs.join(", "),
+            message: reviewedDatasetOutputs.length > 0 ? undefined : "No reviewed dataset output configured.",
+        });
+        steps.push({
+            id: "monitor",
+            status: monitorOutputs.length > 0 ? "passed" : "skipped",
+            artifactPath: monitorOutputs.join(", "),
+            message: monitorOutputs.length > 0 ? undefined : "No monitor output configured.",
+        });
+    }
+    finally {
+        await store.close();
+    }
+    const manifestPath = profile.run?.manifest?.out ?? (profile.outputDir ? join(profile.outputDir, "agentcert-run-manifest.json") : undefined);
+    const manifest = {
+        schemaVersion: "1",
+        kind: "agentcert.run_manifest",
+        runId: stableRunId(profile, bundle),
+        generatedAt: new Date().toISOString(),
+        subject: profile.subject,
+        steps,
+        outputs,
+        verdict: bundle.verdict,
+        summary: bundle.summary,
+    };
+    if (manifestPath) {
+        const resolvedManifestPath = resolve(cwd, manifestPath);
+        await mkdir(dirname(resolvedManifestPath), { recursive: true });
+        manifest.outputs.manifest = normalizeArtifactPath(manifestPath);
+        await writeFile(resolvedManifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+    }
+    const failOnVerdict = profile.run?.gate?.failOnVerdict ?? false;
+    return {
+        exitCode: failOnVerdict && !bundle.verdict.passed ? 1 : 0,
+        bundle,
+        records,
+        manifest,
+    };
+}
+export function renderRunSummary(outcome) {
+    const lines = [
+        "# Witnora Run",
+        "",
+        `Subject: ${outcome.manifest.subject.name}`,
+        `Verdict: ${outcome.bundle.verdict.passed ? "PASS" : "FAIL"} (${outcome.bundle.verdict.score}/100)`,
+        `Products: ${outcome.bundle.summary.products.join(", ")}`,
+        `Records: ${outcome.records.length}`,
+        "",
+        "## Outputs",
+        ...Object.entries(outcome.manifest.outputs)
+            .filter(([, value]) => (Array.isArray(value) ? value.length > 0 : Boolean(value)))
+            .map(([key, value]) => `- ${key}: ${Array.isArray(value) ? value.join(", ") : value}`),
+        "",
+        "## Corpus Summary",
+        renderCorpusSummary(summarizeCorpus(outcome.records)).trim(),
+    ];
+    return `${lines.join("\n")}\n`;
+}
+async function runJobCommand(id, command, cwd, stdio) {
+    const startedAt = Date.now();
+    const exitCode = await new Promise((resolveExit, reject) => {
+        const child = spawn(command, { cwd, shell: true, stdio });
+        child.on("error", reject);
+        child.on("exit", resolveExit);
+    });
+    const durationMs = Date.now() - startedAt;
+    return {
+        id,
+        command,
+        durationMs,
+        exitCode: exitCode ?? undefined,
+        status: exitCode === 0 ? "passed" : "failed",
+    };
+}
+function jobFor(profile, key) {
+    const explicit = profile.run?.jobs?.[key] ?? {};
+    return {
+        required: true,
+        ...explicit,
+        artifact: explicit.artifact ?? profile.artifacts[key],
+    };
+}
+function normalizeArtifact(key, raw, artifactPath) {
+    if (key === "mcpbench")
+        return normalizeMcpBenchResult(raw, artifactPath);
+    if (key === "tripwire")
+        return normalizeTripwireResult(raw, artifactPath);
+    return normalizeOnegentAuditPacket(raw, artifactPath);
+}
+function corpusStoreOptions(profile, cwd) {
+    const corpus = profile.run?.corpus ?? {};
+    return {
+        kind: parseCorpusStoreKind(corpus.store),
+        jsonlPath: corpus.path ? resolve(cwd, corpus.path) : resolve(cwd, ".witnora/corpus/corpus.jsonl"),
+        sqlitePath: corpus.sqlitePath ? resolve(cwd, corpus.sqlitePath) : resolve(cwd, ".witnora/corpus/witnora.sqlite"),
+        databaseUrl: corpus.databaseUrl ?? process.env.WITNORA_DATABASE_URL ?? process.env.AGENTCERT_DATABASE_URL ?? process.env.DATABASE_URL,
+        tableName: corpus.tableName,
+    };
+}
+function monitorOutputsFor(profile) {
+    const monitor = profile.run?.monitor;
+    return [...(monitor?.outputs ?? []), ...(monitor?.out ? [monitor.out] : [])];
+}
+function reviewedDatasetOutputsFor(profile) {
+    const dataset = profile.run?.dataset;
+    return [...(dataset?.reviewedOutputs ?? []), ...(dataset?.reviewedOut ? [dataset.reviewedOut] : [])];
+}
+function artifactPath(base, file) {
+    return normalizeArtifactPath(`${base.replace(/[\\/]+$/, "")}/${file}`);
+}
+function normalizeArtifactPath(path) {
+    return path.replace(/\\/g, "/");
+}
+async function readJson(path, cwd = process.cwd()) {
+    const fullPath = resolve(cwd, path);
+    await access(fullPath);
+    const raw = await readFile(fullPath, "utf8");
+    return JSON.parse(raw);
+}
+function stableRunId(profile, bundle) {
+    const hash = createHash("sha256")
+        .update(`${profile.subject.name}:${bundle.runId}:${JSON.stringify(bundle.summary)}`)
+        .digest("hex")
+        .slice(0, 12);
+    return `agentcert_run_${hash}`;
+}