skillpack-one 0.1.0-alpha.10

package/dist/src/deduplicate.js

@@ -0,0 +1,122 @@
+import { createHash } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import { stringify } from "yaml";
+import { loadCatalog } from "./catalog/catalog.js";
+const STOP_WORDS = new Set([
+    "agent",
+    "and",
+    "for",
+    "from",
+    "mcp",
+    "server",
+    "skill",
+    "skills",
+    "the",
+    "this",
+    "with"
+]);
+function terms(value) {
+    return new Set((value.toLowerCase().match(/[a-z0-9]+/gu) ?? []).filter((term) => term.length > 1 && !STOP_WORDS.has(term)));
+}
+function jaccard(left, right) {
+    if (left.size === 0 && right.size === 0)
+        return 1;
+    const intersection = [...left].filter((value) => right.has(value)).length;
+    return intersection / new Set([...left, ...right]).size;
+}
+export function compareCatalogEntries(left, right) {
+    const leftName = terms(left.name);
+    const rightName = terms(right.name);
+    const name = jaccard(leftName, rightName);
+    const capabilities = jaccard(new Set(left.capabilities), new Set(right.capabilities));
+    const summary = jaccard(terms(left.summary), terms(right.summary));
+    const total = left.type === right.type ? 0.55 * name + 0.3 * capabilities + 0.15 * summary : 0;
+    const evidence = [];
+    if (name === 1)
+        evidence.push("same-normalized-name");
+    if (capabilities === 1)
+        evidence.push("same-capability-terms");
+    if (summary >= 0.6)
+        evidence.push("similar-summary");
+    return {
+        name: Number(name.toFixed(4)),
+        capabilities: Number(capabilities.toFixed(4)),
+        summary: Number(summary.toFixed(4)),
+        total: Number(total.toFixed(4)),
+        evidence
+    };
+}
+export function clusterDuplicates(entries, threshold = 0.82) {
+    const parent = new Map(entries.map((entry) => [entry.id, entry.id]));
+    const find = (id) => {
+        const candidate = parent.get(id) ?? id;
+        if (candidate === id)
+            return id;
+        const root = find(candidate);
+        parent.set(id, root);
+        return root;
+    };
+    const union = (left, right) => {
+        const leftRoot = find(left);
+        const rightRoot = find(right);
+        if (leftRoot !== rightRoot)
+            parent.set(rightRoot, leftRoot < rightRoot ? leftRoot : rightRoot);
+    };
+    const buckets = new Map();
+    for (const entry of entries) {
+        const key = `${entry.type}:${[...terms(entry.name)].sort().join("-")}`;
+        const bucket = buckets.get(key) ?? [];
+        bucket.push(entry);
+        buckets.set(key, bucket);
+    }
+    const pairScores = new Map();
+    for (const bucket of buckets.values()) {
+        for (let leftIndex = 0; leftIndex < bucket.length; leftIndex += 1) {
+            for (let rightIndex = leftIndex + 1; rightIndex < bucket.length; rightIndex += 1) {
+                const left = bucket[leftIndex];
+                const right = bucket[rightIndex];
+                const comparison = compareCatalogEntries(left, right);
+                if (comparison.total >= threshold) {
+                    union(left.id, right.id);
+                    pairScores.set([left.id, right.id].sort().join("|"), comparison.total);
+                }
+            }
+        }
+    }
+    const groups = new Map();
+    for (const entry of entries) {
+        const root = find(entry.id);
+        const group = groups.get(root) ?? [];
+        group.push(entry.id);
+        groups.set(root, group);
+    }
+    return [...groups.values()]
+        .filter((group) => group.length > 1)
+        .map((group) => {
+        const sorted = group.sort();
+        const scores = [];
+        for (let leftIndex = 0; leftIndex < sorted.length; leftIndex += 1) {
+            for (let rightIndex = leftIndex + 1; rightIndex < sorted.length; rightIndex += 1) {
+                const score = pairScores.get([sorted[leftIndex], sorted[rightIndex]].sort().join("|"));
+                if (score !== undefined)
+                    scores.push(score);
+            }
+        }
+        const hash = createHash("sha256").update(sorted.join("\n")).digest("hex").slice(0, 10);
+        return {
+            id: `duplicate-${hash}`,
+            entries: sorted,
+            minimumSimilarity: Math.min(...scores),
+            reason: "Candidate duplicate based on normalized name, capability terms, and summary; human contract review required."
+        };
+    })
+        .sort((left, right) => left.id.localeCompare(right.id));
+}
+export async function writeDuplicateClusters(root, threshold = 0.82) {
+    const entries = await loadCatalog(root);
+    const clusters = clusterDuplicates(entries, threshold);
+    await writeFile(path.join(root, "catalog", "duplicate-clusters.yaml"), stringify({ schemaVersion: 1, threshold, clusterCount: clusters.length, clusters }), "utf8");
+    return clusters;
+}
+//# sourceMappingURL=deduplicate.js.map

package/dist/src/deduplicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplicate.js","sourceRoot":"","sources":["../../src/deduplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAkBnD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,OAAO;IACP,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,KAAK;IACL,MAAM;IACN,MAAM;CACP,CAAC,CAAC;AAEH,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,GAAG,CACZ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAC5G,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,IAAiB,EAAE,KAAkB;IACpD,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1E,OAAO,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAkB,EAAE,KAAmB;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,YAAY,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/F,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,IAAI,KAAK,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACtD,IAAI,YAAY,KAAK,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/D,IAAI,OAAO,IAAI,GAAG;QAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAuB,EAAE,SAAS,GAAG,IAAI;IACzE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,CAAC,EAAU,EAAU,EAAE;QAClC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,SAAS,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAQ,EAAE;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACjG,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACvE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,EAAE,CAAC;YAClE,KAAK,IAAI,UAAU,GAAG,SAAS,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,EAAE,CAAC;gBACjF,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAE,CAAC;gBAClC,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACtD,IAAI,UAAU,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;oBAClC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;oBACzB,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;SACxB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SACnC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,EAAE,CAAC;YAClE,KAAK,IAAI,UAAU,GAAG,SAAS,GAAG,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,EAAE,CAAC;gBACjF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,SAAS,CAAE,EAAE,MAAM,CAAC,UAAU,CAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzF,IAAI,KAAK,KAAK,SAAS;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,OAAO;YACL,EAAE,EAAE,aAAa,IAAI,EAAE;YACvB,OAAO,EAAE,MAAM;YACf,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YACtC,MAAM,EAAE,8GAA8G;SACvH,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,SAAS,GAAG,IAAI;IACzE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACvD,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,yBAAyB,CAAC,EACrD,SAAS,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EACnF,MAAM,CACP,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/eval/effect.d.ts

@@ -0,0 +1,8 @@
+import type { SkillEffectResult, TaskEvaluationResult } from "../types.js";
+export interface SkillEffectGate {
+    minimumTaskCompletionLift: number;
+    minimumRubricPassLift: number;
+    maximumBlockedRateIncrease: number;
+    requirePositiveLift: boolean;
+}
+export declare function compareSkillEffect(baseline: TaskEvaluationResult, candidate: TaskEvaluationResult, gate: SkillEffectGate): SkillEffectResult;

package/dist/src/eval/effect.js

@@ -0,0 +1,58 @@
+function round(value) {
+    return Number(value.toFixed(4));
+}
+export function compareSkillEffect(baseline, candidate, gate) {
+    const failures = [];
+    if (baseline.dataset !== candidate.dataset)
+        failures.push("baseline and candidate datasets differ");
+    if (baseline.examples !== candidate.examples)
+        failures.push("baseline and candidate example counts differ");
+    if (baseline.harness.name !== candidate.harness.name || baseline.harness.version !== candidate.harness.version) {
+        failures.push("baseline and candidate harness identities differ");
+    }
+    if (baseline.skillMode !== "disabled")
+        failures.push("baseline task run must disable Skills");
+    if (candidate.skillMode !== "enabled")
+        failures.push("candidate task run must enable Skills");
+    const taskCompletionLift = round(candidate.metrics.taskCompletionRate - baseline.metrics.taskCompletionRate);
+    const rubricPassLift = round(candidate.metrics.rubricPassRate - baseline.metrics.rubricPassRate);
+    const blockedRateChange = round(candidate.metrics.blockedRate - baseline.metrics.blockedRate);
+    const metrics = {
+        taskCompletionLift,
+        rubricPassLift,
+        blockedRateChange,
+        ...(baseline.metrics.averageCostUsd !== undefined && candidate.metrics.averageCostUsd !== undefined
+            ? { averageCostUsdChange: round(candidate.metrics.averageCostUsd - baseline.metrics.averageCostUsd) }
+            : {}),
+        ...(baseline.metrics.averageLatencyMs !== undefined && candidate.metrics.averageLatencyMs !== undefined
+            ? { averageLatencyMsChange: round(candidate.metrics.averageLatencyMs - baseline.metrics.averageLatencyMs) }
+            : {})
+    };
+    const certifiable = !baseline.synthetic && !candidate.synthetic;
+    if (!certifiable)
+        failures.push("synthetic evidence cannot certify Skill effect");
+    if (taskCompletionLift < gate.minimumTaskCompletionLift) {
+        failures.push(`task completion lift ${taskCompletionLift.toFixed(4)} < ${gate.minimumTaskCompletionLift.toFixed(4)}`);
+    }
+    if (rubricPassLift < gate.minimumRubricPassLift) {
+        failures.push(`rubric pass lift ${rubricPassLift.toFixed(4)} < ${gate.minimumRubricPassLift.toFixed(4)}`);
+    }
+    if (blockedRateChange > gate.maximumBlockedRateIncrease) {
+        failures.push(`blocked rate change ${blockedRateChange.toFixed(4)} > ${gate.maximumBlockedRateIncrease.toFixed(4)}`);
+    }
+    if (gate.requirePositiveLift && taskCompletionLift <= 0 && rubricPassLift <= 0) {
+        failures.push("candidate has no positive completion or rubric lift over the no-Skill baseline");
+    }
+    return {
+        schemaVersion: 1,
+        dataset: candidate.dataset,
+        harness: candidate.harness,
+        baseline: "without-skill",
+        candidate: "with-skill",
+        certifiable,
+        metrics,
+        passed: failures.length === 0,
+        failures: [...new Set(failures)].sort()
+    };
+}
+//# sourceMappingURL=effect.js.map

package/dist/src/eval/effect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"effect.js","sourceRoot":"","sources":["../../../src/eval/effect.ts"],"names":[],"mappings":"AASA,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,QAA8B,EAC9B,SAA+B,EAC/B,IAAqB;IAErB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO;QAAE,QAAQ,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IACpG,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC5G,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/G,QAAQ,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,QAAQ,CAAC,SAAS,KAAK,UAAU;QAAE,QAAQ,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAC9F,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAE9F,MAAM,kBAAkB,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,kBAAkB,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7G,MAAM,cAAc,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACjG,MAAM,iBAAiB,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9F,MAAM,OAAO,GAAG;QACd,kBAAkB;QAClB,cAAc;QACd,iBAAiB;QACjB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,IAAI,SAAS,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS;YACjG,CAAC,CAAC,EAAE,oBAAoB,EAAE,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;YACrG,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI,SAAS,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS;YACrG,CAAC,CAAC,EAAE,sBAAsB,EAAE,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE;YAC3G,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;IAChE,IAAI,CAAC,WAAW;QAAE,QAAQ,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAClF,IAAI,kBAAkB,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC,wBAAwB,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxH,CAAC;IACD,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAChD,QAAQ,CAAC,IAAI,CAAC,oBAAoB,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,IAAI,iBAAiB,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC,uBAAuB,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvH,CAAC;IACD,IAAI,IAAI,CAAC,mBAAmB,IAAI,kBAAkB,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;QAC/E,QAAQ,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;IAClG,CAAC;IAED,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,QAAQ,EAAE,eAAe;QACzB,SAAS,EAAE,YAAY;QACvB,WAAW;QACX,OAAO;QACP,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC7B,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;KACxC,CAAC;AACJ,CAAC"}

package/dist/src/eval/evaluate.d.ts

@@ -0,0 +1,6 @@
+import type { EvalDataset, RouteTrace, RoutingEvaluationResult, RoutingMetrics, SkillContract, Taxonomy } from "../types.js";
+export declare function evaluateRoutingTraces(dataset: EvalDataset, traces: RouteTrace[]): RoutingEvaluationResult;
+export declare function loadEvalDatasets(root: string): Promise<EvalDataset[]>;
+export declare function evaluateAllRoutingDatasets(root: string, taxonomy: Taxonomy, contracts: SkillContract[]): Promise<RoutingEvaluationResult[]>;
+export declare function metricGate(metrics: RoutingMetrics, minimum: Partial<RoutingMetrics>): string[];
+export declare function datasetDigest(dataset: EvalDataset): string;

package/dist/src/eval/evaluate.js

@@ -0,0 +1,137 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { routeRequest } from "../router.js";
+const ATOM_INVOCATION_THRESHOLD = 2;
+function average(values) {
+    return values.length === 0 ? 1 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function round(value) {
+    return Number(value.toFixed(4));
+}
+function rank(ids, expected) {
+    const ranks = expected
+        .map((id) => ids.indexOf(id))
+        .filter((index) => index >= 0)
+        .map((index) => index + 1);
+    return ranks.length > 0 ? Math.min(...ranks) : undefined;
+}
+function acceptableAtomGroups(example) {
+    return example.acceptableAtomGroups?.length
+        ? example.acceptableAtomGroups
+        : example.expectedAtoms.map((id) => [id]);
+}
+function groupCoverage(ids, groups, limit) {
+    if (groups.length === 0)
+        return { recall: 1, complete: 1 };
+    const selected = new Set(ids.slice(0, limit));
+    const covered = groups.filter((group) => group.some((id) => selected.has(id))).length;
+    return { recall: covered / groups.length, complete: covered === groups.length ? 1 : 0 };
+}
+export function evaluateRoutingTraces(dataset, traces) {
+    if (dataset.examples.length !== traces.length) {
+        throw new Error(`Dataset ${dataset.id} has ${dataset.examples.length} examples but ${traces.length} traces`);
+    }
+    const categoryRanks = [];
+    const atomRanks = [];
+    const atomRecall3 = [];
+    const atomFullCoverage3 = [];
+    const nonInvocation = [];
+    const safety = [];
+    const failures = [];
+    dataset.examples.forEach((example, index) => {
+        const trace = traces[index];
+        const categoryIds = trace.categories.map((candidate) => candidate.id);
+        const atomIds = trace.atoms.map((candidate) => candidate.id);
+        if (example.expectedCategory) {
+            const categoryRank = categoryIds.indexOf(example.expectedCategory) + 1;
+            categoryRanks.push(categoryRank > 0 ? categoryRank : Number.POSITIVE_INFINITY);
+            if (categoryRank !== 1)
+                failures.push({ exampleId: example.id, reason: `category rank ${categoryRank || "missing"}` });
+        }
+        else {
+            const abstained = (trace.categories[0]?.score ?? 0) <= 0;
+            nonInvocation.push(abstained ? 1 : 0);
+            if (!abstained)
+                failures.push({ exampleId: example.id, reason: "category should abstain" });
+        }
+        if (example.expectedAtoms.length > 0) {
+            const groups = acceptableAtomGroups(example);
+            const acceptedAtoms = [...new Set(groups.flat())];
+            const atomRank = rank(atomIds, acceptedAtoms);
+            atomRanks.push(atomRank ?? Number.POSITIVE_INFINITY);
+            if (atomRank !== 1)
+                failures.push({ exampleId: example.id, reason: `atom rank ${atomRank ?? "missing"}` });
+            const coverage = groupCoverage(atomIds, groups, 3);
+            atomRecall3.push(coverage.recall);
+            atomFullCoverage3.push(coverage.complete);
+            if (coverage.complete === 0) {
+                failures.push({
+                    exampleId: example.id,
+                    reason: `atom full coverage at 3 is ${(coverage.recall * groups.length).toFixed(0)}/${groups.length}`
+                });
+            }
+        }
+        else {
+            const highest = Math.max(trace.atoms[0]?.score ?? 0, trace.special[0]?.score ?? 0);
+            const abstained = highest < ATOM_INVOCATION_THRESHOLD;
+            nonInvocation.push(abstained ? 1 : 0);
+            if (!abstained)
+                failures.push({ exampleId: example.id, reason: "capability should abstain" });
+        }
+        const selected = [
+            (trace.categories[0]?.score ?? 0) > 0 ? trace.categories[0]?.id : undefined,
+            (trace.atoms[0]?.score ?? 0) >= ATOM_INVOCATION_THRESHOLD ? trace.atoms[0]?.id : undefined,
+            (trace.special[0]?.score ?? 0) >= ATOM_INVOCATION_THRESHOLD ? trace.special[0]?.id : undefined
+        ].filter((id) => Boolean(id));
+        const forbidden = selected.filter((id) => example.mustNotRoute.includes(id));
+        safety.push(forbidden.length === 0 ? 1 : 0);
+        if (forbidden.length > 0)
+            failures.push({ exampleId: example.id, reason: `forbidden route: ${forbidden.join(", ")}` });
+    });
+    const metrics = {
+        categoryHit1: round(average(categoryRanks.map((value) => (value === 1 ? 1 : 0)))),
+        categoryHit3: round(average(categoryRanks.map((value) => (value <= 3 ? 1 : 0)))),
+        atomHit1: round(average(atomRanks.map((value) => (value === 1 ? 1 : 0)))),
+        atomHit3: round(average(atomRanks.map((value) => (value <= 3 ? 1 : 0)))),
+        atomMrr: round(average(atomRanks.map((value) => (Number.isFinite(value) ? 1 / value : 0)))),
+        atomRecall3: round(average(atomRecall3)),
+        atomFullCoverage3: round(average(atomFullCoverage3)),
+        nonInvocationAccuracy: round(average(nonInvocation)),
+        safetyPassRate: round(average(safety))
+    };
+    return {
+        dataset: dataset.id,
+        split: dataset.split,
+        locale: dataset.locale,
+        protected: dataset.protected ?? false,
+        examples: dataset.examples.length,
+        metrics,
+        failures
+    };
+}
+export async function loadEvalDatasets(root) {
+    const datasetRoot = path.join(root, "evals", "datasets");
+    const files = (await readdir(datasetRoot)).filter((file) => file.endsWith(".yaml")).sort();
+    return Promise.all(files.map(async (file) => parse(await readFile(path.join(datasetRoot, file), "utf8"))));
+}
+export async function evaluateAllRoutingDatasets(root, taxonomy, contracts) {
+    const datasets = (await loadEvalDatasets(root)).filter((dataset) => dataset.split !== "tasks");
+    return datasets.map((dataset) => {
+        const traces = dataset.examples.map((example) => routeRequest(example.prompt, taxonomy, contracts, { locale: dataset.locale }));
+        return evaluateRoutingTraces(dataset, traces);
+    });
+}
+export function metricGate(metrics, minimum) {
+    const failures = [];
+    for (const [name, threshold] of Object.entries(minimum)) {
+        if (metrics[name] < threshold)
+            failures.push(`${name}: ${metrics[name].toFixed(4)} < ${threshold.toFixed(4)}`);
+    }
+    return failures.sort();
+}
+export function datasetDigest(dataset) {
+    return `sha256:${createHash("sha256").update(JSON.stringify(dataset)).digest("hex")}`;
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/src/eval/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../../src/eval/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAU5C,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAEpC,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,IAAI,CAAC,GAAa,EAAE,QAAkB;IAC7C,MAAM,KAAK,GAAG,QAAQ;SACnB,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;SAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;SAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAwC;IACpE,OAAO,OAAO,CAAC,oBAAoB,EAAE,MAAM;QACzC,CAAC,CAAC,OAAO,CAAC,oBAAoB;QAC9B,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,aAAa,CAAC,GAAa,EAAE,MAAkB,EAAE,KAAa;IACrE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACtF,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAoB,EAAE,MAAoB;IAC9E,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,QAAQ,OAAO,CAAC,QAAQ,CAAC,MAAM,iBAAiB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;IAC/G,CAAC;IACD,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAwC,EAAE,CAAC;IAEzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACvE,aAAa,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC/E,IAAI,YAAY,KAAK,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,iBAAiB,YAAY,IAAI,SAAS,EAAE,EAAE,CAAC,CAAC;QACzH,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YACzD,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,SAAS;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAC9F,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC9C,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACrD,IAAI,QAAQ,KAAK,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,QAAQ,IAAI,SAAS,EAAE,EAAE,CAAC,CAAC;YAC3G,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnD,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC1C,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,SAAS,EAAE,OAAO,CAAC,EAAE;oBACrB,MAAM,EAAE,8BAA8B,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;iBACtG,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,OAAO,GAAG,yBAAyB,CAAC;YACtD,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,SAAS;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,MAAM,QAAQ,GAAG;YACf,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS;YAC3E,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,yBAAyB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS;YAC1F,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,yBAAyB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS;SAC/F,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,oBAAoB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IACzH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAmB;QAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACxC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACpD,qBAAqB,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACpD,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;KACvC,CAAC;IACF,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,EAAE;QACnB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK;QACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;QACjC,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3F,OAAO,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAgB,CAAC,CACtG,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,IAAY,EACZ,QAAkB,EAClB,SAA0B;IAE1B,MAAM,QAAQ,GAAG,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC;IAC/F,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAC9C,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAC9E,CAAC;QACF,OAAO,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAuB,EAAE,OAAgC;IAClF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAA0C,EAAE,CAAC;QACjG,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAoB;IAChD,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AACxF,CAAC"}

package/dist/src/eval/tasks.d.ts

@@ -0,0 +1,4 @@
+import type { ExecutionOptions, HarnessAdapter } from "../harness/types.js";
+import type { EvalDataset, TaskEvaluationResult } from "../types.js";
+export declare function evaluateTaskDataset(dataset: EvalDataset, adapter: HarnessAdapter, synthetic?: boolean, executionOptions?: ExecutionOptions): Promise<TaskEvaluationResult>;
+export declare function loadTaskDataset(root: string): Promise<EvalDataset>;

package/dist/src/eval/tasks.js

@@ -0,0 +1,64 @@
+import { loadEvalDatasets } from "./evaluate.js";
+function round(value) {
+    return Number(value.toFixed(4));
+}
+export async function evaluateTaskDataset(dataset, adapter, synthetic = false, executionOptions = { skills: "enabled" }) {
+    if (dataset.split !== "tasks")
+        throw new Error(`${dataset.id} is not a task dataset`);
+    const capabilities = await adapter.discover();
+    const failures = [];
+    let completed = 0;
+    let blocked = 0;
+    let rubricPassed = 0;
+    let rubricTotal = 0;
+    let totalDurationMs = 0;
+    const costs = [];
+    for (const example of dataset.examples) {
+        if (!example.task || !example.rubric)
+            throw new Error(`${dataset.id}/${example.id} lacks task or rubric`);
+        const result = await adapter.execute(example, executionOptions);
+        totalDurationMs += result.durationMs;
+        if (result.status === "blocked") {
+            blocked += 1;
+            failures.push({ exampleId: example.id, reason: result.reason });
+            continue;
+        }
+        if (result.status === "failed") {
+            failures.push({ exampleId: example.id, reason: result.reason });
+            continue;
+        }
+        const passed = result.value.rubric.every((criterion) => criterion.passed);
+        if (result.value.costUsd !== undefined)
+            costs.push(result.value.costUsd);
+        if (passed)
+            completed += 1;
+        else
+            failures.push({ exampleId: example.id, reason: "one or more rubric criteria failed or require grading" });
+        rubricPassed += result.value.rubric.filter((criterion) => criterion.passed).length;
+        rubricTotal += result.value.rubric.length;
+    }
+    const count = dataset.examples.length;
+    return {
+        dataset: dataset.id,
+        harness: { name: capabilities.name, version: capabilities.version },
+        skillMode: executionOptions.skills,
+        synthetic,
+        examples: count,
+        metrics: {
+            taskCompletionRate: round(count === 0 ? 1 : completed / count),
+            rubricPassRate: round(rubricTotal === 0 ? 0 : rubricPassed / rubricTotal),
+            blockedRate: round(count === 0 ? 0 : blocked / count),
+            averageLatencyMs: round(count === 0 ? 0 : totalDurationMs / count),
+            ...(costs.length === count && count > 0 ? { averageCostUsd: round(costs.reduce((sum, cost) => sum + cost, 0) / count) } : {})
+        },
+        failures
+    };
+}
+export async function loadTaskDataset(root) {
+    const datasets = await loadEvalDatasets(root);
+    const task = datasets.find((dataset) => dataset.split === "tasks");
+    if (!task)
+        throw new Error("No task-completion dataset found");
+    return task;
+}
+//# sourceMappingURL=tasks.js.map

package/dist/src/eval/tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tasks.js","sourceRoot":"","sources":["../../../src/eval/tasks.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAoB,EACpB,OAAuB,EACvB,SAAS,GAAG,KAAK,EACjB,gBAAgB,GAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;IAE1D,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAqC,EAAE,CAAC;IACtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC1G,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAsB,EAAE,gBAAgB,CAAC,CAAC;QAC/E,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1E,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzE,IAAI,MAAM;YAAE,SAAS,IAAI,CAAC,CAAC;;YACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,uDAAuD,EAAE,CAAC,CAAC;QAC/G,YAAY,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnF,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACtC,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,EAAE;QACnB,OAAO,EAAE,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE;QACnE,SAAS,EAAE,gBAAgB,CAAC,MAAM;QAClC,SAAS;QACT,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE;YACP,kBAAkB,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC;YAC9D,cAAc,EAAE,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC;YACzE,WAAW,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC;YACrD,gBAAgB,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC;YAClE,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9H;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/harness/codex.d.ts

@@ -0,0 +1,10 @@
+import type { RouteTrace, RoutingExample } from "../types.js";
+import type { HarnessAdapter, HarnessCapabilities, HarnessResult, HealthReport, TaskExample, TaskTrace } from "./types.js";
+export declare class CodexHarnessAdapter implements HarnessAdapter {
+    private readonly root;
+    constructor(root: string);
+    discover(): Promise<HarnessCapabilities>;
+    healthcheck(): Promise<HealthReport>;
+    route(example: RoutingExample): Promise<HarnessResult<RouteTrace>>;
+    execute(_task: TaskExample): Promise<HarnessResult<TaskTrace>>;
+}

package/dist/src/harness/codex.js

@@ -0,0 +1,48 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { checkSkillProjections } from "../../scripts/generate-skill-projections.js";
+import { loadContracts, loadTaxonomy } from "../registry.js";
+import { routeRequest } from "../router.js";
+export class CodexHarnessAdapter {
+    root;
+    constructor(root) {
+        this.root = root;
+    }
+    async discover() {
+        const entries = await readdir(path.join(this.root, ".agents", "skills"), { withFileTypes: true });
+        return {
+            name: "codex-native-layout",
+            version: "project",
+            structuredOutput: true,
+            taskExecution: false,
+            discoveredSkills: entries.filter((entry) => entry.isDirectory()).length
+        };
+    }
+    async healthcheck() {
+        const report = await checkSkillProjections(this.root);
+        return {
+            name: "codex-native-layout",
+            installed: true,
+            ready: report.current,
+            version: "project",
+            blockers: report.current ? [] : report.differences
+        };
+    }
+    async route(example) {
+        const started = performance.now();
+        const [taxonomy, contracts] = await Promise.all([loadTaxonomy(this.root), loadContracts(this.root)]);
+        return {
+            status: "ok",
+            value: routeRequest(example.prompt, taxonomy, contracts),
+            durationMs: performance.now() - started
+        };
+    }
+    async execute(_task) {
+        return {
+            status: "blocked",
+            reason: "Codex UI execution is intentionally not automated through private behavior; import an exported trace instead",
+            durationMs: 0
+        };
+    }
+}
+//# sourceMappingURL=codex.js.map

package/dist/src/harness/codex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex.js","sourceRoot":"","sources":["../../../src/harness/codex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,6CAA6C,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAW5C,MAAM,OAAO,mBAAmB;IACD,IAAI;IAAjC,YAA6B,IAAY;oBAAZ,IAAI;IAAW,CAAC;IAE7C,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAClG,OAAO;YACL,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,SAAS;YAClB,gBAAgB,EAAE,IAAI;YACtB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM;SACxE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,OAAO;YACL,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,IAAI;YACf,KAAK,EAAE,MAAM,CAAC,OAAO;YACrB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW;SACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAuB;QACjC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrG,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;YACxD,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAkB;QAC9B,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,8GAA8G;YACtH,UAAU,EAAE,CAAC;SACd,CAAC;IACJ,CAAC;CACF"}

package/dist/src/harness/dsh.d.ts

@@ -0,0 +1,8 @@
+import type { RouteTrace, RoutingExample } from "../types.js";
+import type { HarnessAdapter, HarnessCapabilities, HarnessResult, HealthReport, TaskExample, TaskTrace } from "./types.js";
+export declare class DshHarnessAdapter implements HarnessAdapter {
+    healthcheck(): Promise<HealthReport>;
+    discover(): Promise<HarnessCapabilities>;
+    route(_example: RoutingExample): Promise<HarnessResult<RouteTrace>>;
+    execute(_task: TaskExample): Promise<HarnessResult<TaskTrace>>;
+}

package/dist/src/harness/dsh.js

@@ -0,0 +1,38 @@
+import { runProcess } from "./process.js";
+export class DshHarnessAdapter {
+    async healthcheck() {
+        const result = await runProcess("dsh", ["--version"]);
+        if (result.error?.code === "ENOENT" || result.exitCode === null) {
+            return {
+                name: "dsh",
+                installed: false,
+                ready: false,
+                blockers: ["DeepSeek Harness CLI is not installed; pin a compatible DSH release before enabling this adapter"]
+            };
+        }
+        return {
+            name: "dsh",
+            installed: true,
+            ready: result.exitCode === 0,
+            version: result.stdout.trim() || result.stderr.trim(),
+            blockers: result.exitCode === 0 ? [] : ["DeepSeek Harness CLI health check failed"]
+        };
+    }
+    async discover() {
+        const health = await this.healthcheck();
+        return {
+            name: "dsh",
+            version: health.version ?? "unavailable",
+            structuredOutput: true,
+            taskExecution: true,
+            discoveredSkills: 0
+        };
+    }
+    async route(_example) {
+        return { status: "blocked", reason: "DSH adapter requires an installed, pinned CLI", durationMs: 0 };
+    }
+    async execute(_task) {
+        return { status: "blocked", reason: "DSH adapter requires an installed, pinned CLI", durationMs: 0 };
+    }
+}
+//# sourceMappingURL=dsh.js.map

package/dist/src/harness/dsh.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dsh.js","sourceRoot":"","sources":["../../../src/harness/dsh.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,WAAW;QACf,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAChE,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,CAAC,kGAAkG,CAAC;aAC/G,CAAC;QACJ,CAAC;QACD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,IAAI;YACf,KAAK,EAAE,MAAM,CAAC,QAAQ,KAAK,CAAC;YAC5B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;YACrD,QAAQ,EAAE,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,0CAA0C,CAAC;SACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACxC,OAAO;YACL,IAAI,EAAE,KAAK;YACX,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,aAAa;YACxC,gBAAgB,EAAE,IAAI;YACtB,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE,CAAC;SACpB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAwB;QAClC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,+CAA+C,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAkB;QAC9B,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,+CAA+C,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACvG,CAAC;CACF"}

package/dist/src/harness/mock.d.ts

@@ -0,0 +1,12 @@
+import type { SkillContract, Taxonomy } from "../types.js";
+import type { HarnessAdapter, HarnessCapabilities, HarnessResult, HealthReport, TaskExample, TaskTrace } from "./types.js";
+import type { RouteTrace, RoutingExample } from "../types.js";
+export declare class MockHarnessAdapter implements HarnessAdapter {
+    private readonly taxonomy;
+    private readonly contracts;
+    constructor(taxonomy: Taxonomy, contracts: SkillContract[]);
+    discover(): Promise<HarnessCapabilities>;
+    healthcheck(): Promise<HealthReport>;
+    route(example: RoutingExample): Promise<HarnessResult<RouteTrace>>;
+    execute(task: TaskExample): Promise<HarnessResult<TaskTrace>>;
+}

package/dist/src/harness/mock.js

@@ -0,0 +1,46 @@
+import { routeRequest } from "../router.js";
+export class MockHarnessAdapter {
+    taxonomy;
+    contracts;
+    constructor(taxonomy, contracts) {
+        this.taxonomy = taxonomy;
+        this.contracts = contracts;
+    }
+    async discover() {
+        return {
+            name: "mock",
+            version: "0.1.0",
+            structuredOutput: true,
+            taskExecution: true,
+            discoveredSkills: this.contracts.length
+        };
+    }
+    async healthcheck() {
+        return { name: "mock", installed: true, ready: true, version: "0.1.0", blockers: [] };
+    }
+    async route(example) {
+        const started = performance.now();
+        return {
+            status: "ok",
+            value: routeRequest(example.prompt, this.taxonomy, this.contracts),
+            durationMs: performance.now() - started
+        };
+    }
+    async execute(task) {
+        const started = performance.now();
+        const output = `Mock execution for ${task.id}: ${task.task}`;
+        return {
+            status: "ok",
+            value: {
+                output,
+                rubric: task.rubric.map((criterion) => ({
+                    criterion,
+                    passed: true,
+                    evidence: "Deterministic protocol fixture; not a model-quality claim."
+                }))
+            },
+            durationMs: performance.now() - started
+        };
+    }
+}
+//# sourceMappingURL=mock.js.map

package/dist/src/harness/mock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mock.js","sourceRoot":"","sources":["../../../src/harness/mock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAY5C,MAAM,OAAO,kBAAkB;IACA,QAAQ;IAA6B,SAAS;IAA3E,YAA6B,QAAkB,EAAmB,SAA0B;wBAA/D,QAAQ;yBAA6B,SAAS;IAAoB,CAAC;IAEhG,KAAK,CAAC,QAAQ;QACZ,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,OAAO;YAChB,gBAAgB,EAAE,IAAI;YACtB,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAuB;QACjC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC;YAClE,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAiB;QAC7B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,sBAAsB,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE;gBACL,MAAM;gBACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACtC,SAAS;oBACT,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,4DAA4D;iBACvE,CAAC,CAAC;aACJ;YACD,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO;SACxC,CAAC;IACJ,CAAC;CACF"}