pi-harness-runtime 0.9.4 → 0.9.6

package/packages/evaluation-runner/dist/runner.js

@@ -0,0 +1,153 @@
+/**
+ * Evaluation Runner — Core (RFC-0020)
+ */
+let _runCounter = 0;
+function nextRunId() {
+    return `run-${Date.now()}-${++_runCounter}`;
+}
+export class EvaluationRunner {
+    evaluators = new Map();
+    config;
+    constructor(config = {}) {
+        this.config = {
+            timeoutMs: config.timeoutMs ?? 30_000,
+            retries: config.retries ?? 1,
+            parallel: config.parallel ?? false,
+        };
+    }
+    registerEvaluator(name, fn) {
+        this.evaluators.set(name, fn);
+    }
+    async runSuite(suite, execute) {
+        const results = [];
+        const cases = suite.cases;
+        if (this.config.parallel) {
+            const promises = cases.map((c) => this.runCase(c, execute));
+            const all = await Promise.allSettled(promises);
+            for (const r of all) {
+                if (r.status === "fulfilled")
+                    results.push(...r.value);
+            }
+        }
+        else {
+            for (const c of cases) {
+                const r = await this.runCase(c, execute);
+                results.push(...r);
+            }
+        }
+        const passed = results.filter((r) => r.passed).length;
+        const summary = {};
+        for (const r of results) {
+            if (!summary[r.metric]) {
+                summary[r.metric] = { count: 0, avgScore: 0 };
+            }
+            const s = summary[r.metric];
+            s.count++;
+            s.avgScore = (s.avgScore * (s.count - 1) + r.score) / s.count;
+        }
+        return {
+            suiteId: suite.id,
+            runAt: new Date().toISOString(),
+            total: results.length,
+            passed,
+            failed: results.length - passed,
+            results,
+            summary,
+        };
+    }
+    async runCase(c, execute) {
+        const start = Date.now();
+        let output;
+        try {
+            output = await Promise.race([
+                execute(c.input),
+                new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), this.config.timeoutMs)),
+            ]);
+        }
+        catch (err) {
+            return c.evaluators.map((name) => ({
+                runId: nextRunId(),
+                metric: name,
+                score: 0,
+                passed: false,
+                details: err instanceof Error ? err.message : "unknown error",
+                latencyMs: Date.now() - start,
+            }));
+        }
+        const results = [];
+        for (const name of c.evaluators) {
+            const fn = this.evaluators.get(name);
+            if (!fn) {
+                results.push({
+                    runId: nextRunId(),
+                    metric: name,
+                    score: 0,
+                    passed: false,
+                    details: `Evaluator not registered: ${name}`,
+                    latencyMs: Date.now() - start,
+                });
+                continue;
+            }
+            for (let attempt = 0; attempt < this.config.retries; attempt++) {
+                try {
+                    const r = await fn(c.input, output);
+                    results.push(r);
+                    break;
+                }
+                catch (err) {
+                    if (attempt === this.config.retries - 1) {
+                        results.push({
+                            runId: nextRunId(),
+                            metric: name,
+                            score: 0,
+                            passed: false,
+                            details: err instanceof Error ? err.message : "unknown error",
+                            latencyMs: Date.now() - start,
+                        });
+                    }
+                }
+            }
+        }
+        return results;
+    }
+    listEvaluators() {
+        return [...this.evaluators.keys()];
+    }
+}
+// Built-in evaluators
+export const exactMatchEvaluator = async (input, output) => {
+    const expected = input.expectedOutput ?? "";
+    return {
+        runId: nextRunId(),
+        metric: "exact-match",
+        score: output === expected ? 1 : 0,
+        passed: output === expected,
+        details: output === expected ? "exact match" : "output differs from expected",
+    };
+};
+export const substringMatchEvaluator = async (input, output) => {
+    const expected = input.expectedOutput ?? "";
+    const found = expected.length > 0 && output.includes(expected);
+    return {
+        runId: nextRunId(),
+        metric: "substring-match",
+        score: found ? 1 : 0,
+        passed: found,
+        details: found ? "expected substring found" : "expected substring not found",
+    };
+};
+export const lengthEvaluator = async (input, output) => {
+    const min = input.metadata?.["minLength"] ?? 0;
+    const max = input.metadata?.["maxLength"] ?? Infinity;
+    const len = output.length;
+    const passed = len >= min && len <= max;
+    const score = passed ? 1 : Math.max(0, 1 - Math.abs(len - (min + max) / 2) / max);
+    return {
+        runId: nextRunId(),
+        metric: "length-check",
+        score,
+        passed,
+        details: `length ${len}, expected ${min}-${max}`,
+    };
+};
+//# sourceMappingURL=runner.js.map

package/packages/evaluation-runner/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,SAAS,SAAS;IACjB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,OAAO,gBAAgB;IACpB,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,MAAM,CAA6B;IAE3C,YAAY,MAAM,GAAqB,EAAE;QACxC,IAAI,CAAC,MAAM,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;YAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;SAClC,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAAY,EAAE,EAAe;QAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,QAAQ,CACb,KAAsB,EACtB,OAAoD;QAEpD,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACxD,CAAC;QACF,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACpB,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,OAAO,GAAgC,EAAE,CAAC;QAEhD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC;YACD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QAC/D,CAAC;QAED,OAAO;YACN,OAAO,EAAE,KAAK,CAAC,EAAE;YACjB,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;YAC/B,OAAO;YACP,OAAO;SACP,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACpB,CAAiB,EACjB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACJ,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC3B,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChB,IAAI,OAAO,CAAS,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACjC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CACrE;aACD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,SAAS,EAAE;gBAClB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC7B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,SAAS,EAAE;oBAClB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,6BAA6B,IAAI,EAAE;oBAC5C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAC7B,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACJ,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;wBACzC,OAAO,CAAC,IAAI,CAAC;4BACZ,KAAK,EAAE,SAAS,EAAE;4BAClB,MAAM,EAAE,IAAI;4BACZ,KAAK,EAAE,CAAC;4BACR,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;4BAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;yBAC7B,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,cAAc;QACb,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;CACD;AAED,sBAAsB;AACtB,MAAM,CAAC,MAAM,mBAAmB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,aAAa;QACrB,KAAK,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,MAAM,KAAK,QAAQ;QAC3B,OAAO,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,8BAA8B;KAC7E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,iBAAiB;QACzB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,8BAA8B;KAC5E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACnE,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,QAAQ,CAAC;IAClE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1B,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IAClF,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,cAAc;QACtB,KAAK;QACL,MAAM;QACN,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,IAAI,GAAG,EAAE;KAChD,CAAC;AACH,CAAC,CAAC"}

package/packages/evaluation-runner/dist/types.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+export interface EvaluationInput {
+    prompt: string;
+    expectedOutput?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface EvaluationResult {
+    runId: string;
+    metric: string;
+    score: number;
+    passed: boolean;
+    details?: string;
+    latencyMs?: number;
+}
+export interface EvaluationSuite {
+    id: string;
+    name: string;
+    cases: EvaluationCase[];
+    thresholds: Record<string, number>;
+}
+export interface EvaluationCase {
+    id: string;
+    name: string;
+    input: EvaluationInput;
+    expected?: string;
+    evaluators: string[];
+}
+export type EvaluatorFn = (input: EvaluationInput, output: string) => Promise<EvaluationResult>;
+export interface EvaluationReport {
+    suiteId: string;
+    runAt: string;
+    total: number;
+    passed: number;
+    failed: number;
+    results: EvaluationResult[];
+    summary: Record<string, {
+        count: number;
+        avgScore: number;
+    }>;
+}
+export interface EvaluationConfig {
+    timeoutMs?: number;
+    retries?: number;
+    parallel?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/packages/evaluation-runner/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAE/F,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB"}

package/packages/evaluation-runner/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/packages/evaluation-runner/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/packages/evaluation-runner/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@pi/evaluation-runner",
+  "version": "0.9.6",
+  "type": "module",
+  "description": "Framework-agnostic evaluation framework for AI outputs (RFC-0020)",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc --noEmit",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0"
+  }
+}

package/packages/evaluation-runner/src/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Evaluation Runner — Main Entry (RFC-0020)
+ */
+
+export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
+export type * from "./types.js";

package/packages/evaluation-runner/src/runner.ts

@@ -0,0 +1,182 @@
+/**
+ * Evaluation Runner — Core (RFC-0020)
+ */
+
+import type {
+	EvaluationCase,
+	EvaluationConfig,
+	EvaluationInput,
+	EvaluatorFn,
+	EvaluationReport,
+	EvaluationResult,
+	EvaluationSuite,
+} from "./types.js";
+
+let _runCounter = 0;
+function nextRunId(): string {
+	return `run-${Date.now()}-${++_runCounter}`;
+}
+
+export class EvaluationRunner {
+	private evaluators = new Map<string, EvaluatorFn>();
+	private config: Required<EvaluationConfig>;
+
+	constructor(config: EvaluationConfig = {}) {
+		this.config = {
+			timeoutMs: config.timeoutMs ?? 30_000,
+			retries: config.retries ?? 1,
+			parallel: config.parallel ?? false,
+		};
+	}
+
+	registerEvaluator(name: string, fn: EvaluatorFn): void {
+		this.evaluators.set(name, fn);
+	}
+
+	async runSuite(
+		suite: EvaluationSuite,
+		execute: (input: EvaluationInput) => Promise<string>,
+	): Promise<EvaluationReport> {
+		const results: EvaluationResult[] = [];
+		const cases = suite.cases;
+
+		if (this.config.parallel) {
+			const promises = cases.map((c) => this.runCase(c, execute));
+			const all = await Promise.allSettled(promises);
+			for (const r of all) {
+				if (r.status === "fulfilled") results.push(...r.value);
+			}
+		} else {
+			for (const c of cases) {
+				const r = await this.runCase(c, execute);
+				results.push(...r);
+			}
+		}
+
+		const passed = results.filter((r) => r.passed).length;
+		const summary: EvaluationReport["summary"] = {};
+
+		for (const r of results) {
+			if (!summary[r.metric]) {
+				summary[r.metric] = { count: 0, avgScore: 0 };
+			}
+			const s = summary[r.metric];
+			s.count++;
+			s.avgScore = (s.avgScore * (s.count - 1) + r.score) / s.count;
+		}
+
+		return {
+			suiteId: suite.id,
+			runAt: new Date().toISOString(),
+			total: results.length,
+			passed,
+			failed: results.length - passed,
+			results,
+			summary,
+		};
+	}
+
+	private async runCase(
+		c: EvaluationCase,
+		execute: (input: EvaluationInput) => Promise<string>,
+	): Promise<EvaluationResult[]> {
+		const start = Date.now();
+		let output: string;
+		try {
+			output = await Promise.race([
+				execute(c.input),
+				new Promise<string>((_, reject) =>
+					setTimeout(() => reject(new Error("timeout")), this.config.timeoutMs),
+				),
+			]);
+		} catch (err) {
+			return c.evaluators.map((name) => ({
+				runId: nextRunId(),
+				metric: name,
+				score: 0,
+				passed: false,
+				details: err instanceof Error ? err.message : "unknown error",
+				latencyMs: Date.now() - start,
+			}));
+		}
+
+		const results: EvaluationResult[] = [];
+		for (const name of c.evaluators) {
+			const fn = this.evaluators.get(name);
+			if (!fn) {
+				results.push({
+					runId: nextRunId(),
+					metric: name,
+					score: 0,
+					passed: false,
+					details: `Evaluator not registered: ${name}`,
+					latencyMs: Date.now() - start,
+				});
+				continue;
+			}
+
+			for (let attempt = 0; attempt < this.config.retries; attempt++) {
+				try {
+					const r = await fn(c.input, output);
+					results.push(r);
+					break;
+				} catch (err) {
+					if (attempt === this.config.retries - 1) {
+						results.push({
+							runId: nextRunId(),
+							metric: name,
+							score: 0,
+							passed: false,
+							details: err instanceof Error ? err.message : "unknown error",
+							latencyMs: Date.now() - start,
+						});
+					}
+				}
+			}
+		}
+		return results;
+	}
+
+	listEvaluators(): string[] {
+		return [...this.evaluators.keys()];
+	}
+}
+
+// Built-in evaluators
+export const exactMatchEvaluator: EvaluatorFn = async (input, output) => {
+	const expected = input.expectedOutput ?? "";
+	return {
+		runId: nextRunId(),
+		metric: "exact-match",
+		score: output === expected ? 1 : 0,
+		passed: output === expected,
+		details: output === expected ? "exact match" : "output differs from expected",
+	};
+};
+
+export const substringMatchEvaluator: EvaluatorFn = async (input, output) => {
+	const expected = input.expectedOutput ?? "";
+	const found = expected.length > 0 && output.includes(expected);
+	return {
+		runId: nextRunId(),
+		metric: "substring-match",
+		score: found ? 1 : 0,
+		passed: found,
+		details: found ? "expected substring found" : "expected substring not found",
+	};
+};
+
+export const lengthEvaluator: EvaluatorFn = async (input, output) => {
+	const min = (input.metadata?.["minLength"] as number) ?? 0;
+	const max = (input.metadata?.["maxLength"] as number) ?? Infinity;
+	const len = output.length;
+	const passed = len >= min && len <= max;
+	const score = passed ? 1 : Math.max(0, 1 - Math.abs(len - (min + max) / 2) / max);
+	return {
+		runId: nextRunId(),
+		metric: "length-check",
+		score,
+		passed,
+		details: `length ${len}, expected ${min}-${max}`,
+	};
+};

package/packages/evaluation-runner/src/types.ts

@@ -0,0 +1,51 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+
+export interface EvaluationInput {
+	prompt: string;
+	expectedOutput?: string;
+	metadata?: Record<string, unknown>;
+}
+
+export interface EvaluationResult {
+	runId: string;
+	metric: string;
+	score: number;
+	passed: boolean;
+	details?: string;
+	latencyMs?: number;
+}
+
+export interface EvaluationSuite {
+	id: string;
+	name: string;
+	cases: EvaluationCase[];
+	thresholds: Record<string, number>;
+}
+
+export interface EvaluationCase {
+	id: string;
+	name: string;
+	input: EvaluationInput;
+	expected?: string;
+	evaluators: string[];
+}
+
+export type EvaluatorFn = (input: EvaluationInput, output: string) => Promise<EvaluationResult>
+
+export interface EvaluationReport {
+	suiteId: string;
+	runAt: string;
+	total: number;
+	passed: number;
+	failed: number;
+	results: EvaluationResult[];
+	summary: Record<string, { count: number; avgScore: number }>;
+}
+
+export interface EvaluationConfig {
+	timeoutMs?: number;
+	retries?: number;
+	parallel?: boolean;
+}

package/packages/evaluation-runner/test/runner.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Evaluation Runner Tests (RFC-0020)
+ */
+
+import { describe, it, expect, beforeEach } from "bun:test";
+import {
+  EvaluationRunner,
+  exactMatchEvaluator,
+  substringMatchEvaluator,
+  lengthEvaluator,
+} from "../src/index.js";
+import type { EvaluationSuite } from "../src/index.js";
+
+describe("EvaluationRunner", () => {
+  let runner: EvaluationRunner;
+
+  beforeEach(() => { runner = new EvaluationRunner(); });
+
+  it("registers and lists evaluators", () => {
+    runner.registerEvaluator("custom", async () => ({ runId: "r1", metric: "custom", score: 0.5, passed: true }));
+    expect(runner.listEvaluators()).toContain("custom");
+  });
+
+  it("runs suite and returns report", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s1",
+      name: "Basic",
+      cases: [{
+        id: "c1",
+        name: "exact match",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: { "exact-match": 1 },
+    };
+    const report = await runner.runSuite(suite, async () => "world");
+    expect(report.total).toBe(1);
+    expect(report.passed).toBe(1);
+    expect(report.results[0].score).toBe(1);
+  });
+
+  it("marks failed when output differs", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s2",
+      name: "Fail",
+      cases: [{
+        id: "c1",
+        name: "differs",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "other");
+    expect(report.failed).toBe(1);
+    expect(report.passed).toBe(0);
+  });
+
+  it("handles timeout", async () => {
+    runner = new EvaluationRunner({ timeoutMs: 10 });
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s3",
+      name: "Timeout",
+      cases: [{
+        id: "c1",
+        name: "slow",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => new Promise((r) => setTimeout(() => r("world"), 200)));
+    expect(report.failed).toBe(1);
+    expect(report.results[0].details).toContain("timeout");
+  });
+
+  it("registers unknown evaluator gracefully", async () => {
+    const suite: EvaluationSuite = {
+      id: "s4",
+      name: "Unknown",
+      cases: [{
+        id: "c1",
+        name: "unknown",
+        input: { prompt: "hello" },
+        evaluators: ["not-registered"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "output");
+    expect(report.total).toBe(1);
+    expect(report.results[0].details).toContain("not registered");
+  });
+
+  it("calculates summary averages", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s5",
+      name: "Multi",
+      cases: [
+        { id: "c1", name: "a", input: { prompt: "a", expectedOutput: "x" }, evaluators: ["exact"] },
+        { id: "c2", name: "b", input: { prompt: "b", expectedOutput: "y" }, evaluators: ["exact"] },
+      ],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "x");
+    expect(report.summary["exact-match"]?.count).toBe(2);
+  });
+});
+
+describe("exactMatchEvaluator", () => {
+  it("scores 1 for exact match", async () => {
+    const result = await exactMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world" },
+      "world",
+    );
+    expect(result.score).toBe(1);
+    expect(result.passed).toBe(true);
+  });
+
+  it("scores 0 for mismatch", async () => {
+    const result = await exactMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world" },
+      "other",
+    );
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+});
+
+describe("substringMatchEvaluator", () => {
+  it("passes when expected is substring", async () => {
+    const result = await substringMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world peace" },
+      "hello world peace today",
+    );
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it("fails when expected not found", async () => {
+    const result = await substringMatchEvaluator(
+      { prompt: "hello", expectedOutput: "xyz" },
+      "abc def",
+    );
+    expect(result.passed).toBe(false);
+  });
+});
+
+describe("lengthEvaluator", () => {
+  it("passes when within range", async () => {
+    const result = await lengthEvaluator(
+      { prompt: "hello", metadata: { minLength: 5, maxLength: 20 } },
+      "hello world",
+    );
+    expect(result.passed).toBe(true);
+  });
+
+  it("fails when too long", async () => {
+    const result = await lengthEvaluator(
+      { prompt: "hello", metadata: { minLength: 5, maxLength: 10 } },
+      "this is a very long response",
+    );
+    expect(result.passed).toBe(false);
+  });
+});

package/packages/evaluation-runner/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": { "outDir": "dist", "rootDir": "src" },
+  "include": ["src/**/*"]
+}

package/packages/experience-replay/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/experience-replay",
-  "version": "0.9.4",
+  "version": "0.9.6",
   "description": "Experience replay for debugging and reproducibility (RFC-0059)",
   "type": "module",
   "main": "./dist/index.js",