pi-harness-runtime 0.9.2 → 0.9.5

package/packages/doc-generator/test/docs.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Doc Generator Tests (RFC-0014)
+ */
+
+import { describe, it, expect } from "bun:test";
+import {
+	detectProjectType,
+	parseSignals,
+	detectSymbols,
+	generateDocs,
+} from "../src/index.js";
+import type { SourceFile } from "../src/index.js";
+
+const src = (content: string, path = "src/utils.ts"): SourceFile => ({
+	path,
+	content,
+	language: "typescript",
+});
+
+describe("detectProjectType", () => {
+	it("detects Frappe/ERPNext", () => {
+		const signals = ["frappe-bench/apps/myapp/hooks.py", "doctype/CRM"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("frappe_erpnext");
+		expect(result.signals).toContain("frappe-bench/apps/myapp/hooks.py");
+	});
+
+	it("detects Next.js", () => {
+		const signals = ["next.config.js", "app/layout.tsx", "pages/index.ts"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("nextjs");
+	});
+
+	it("detects React/Vite", () => {
+		const signals = ["vite.config.ts", "src/main.tsx", "index.html"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("react_vite");
+	});
+
+	it("detects Django", () => {
+		const signals = ["manage.py", "settings.py"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("django");
+	});
+
+	it("detects Laravel", () => {
+		const signals = ["artisan", "database/seeders/UserSeeder.php"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("laravel");
+	});
+
+	it("returns unknown for empty signals", () => {
+		const result = detectProjectType([]);
+		expect(result.projectType).toBe("unknown");
+		expect(result.confidence).toBe(0);
+	});
+
+	it("returns generic_web for package.json", () => {
+		const result = detectProjectType(["package.json", "index.js"]);
+		expect(result.projectType).toBe("generic_web");
+	});
+
+	it("includes recommended seed strategy", () => {
+		const result = detectProjectType(["manage.py"]);
+		expect(result.recommendedSeedStrategy).toBe("django_migrations");
+	});
+
+	it("includes recommended e2e strategy", () => {
+		const result = detectProjectType(["next.config.js"]);
+		expect(result.recommendedE2EStrategy).toBe("nextjs_dev_server_flow");
+	});
+
+	it("confidence is 0-1", () => {
+		const result = detectProjectType(["doctype/CRM"]);
+		expect(result.confidence).toBeGreaterThanOrEqual(0);
+		expect(result.confidence).toBeLessThanOrEqual(1);
+	});
+});
+
+describe("parseSignals", () => {
+	it("filters paths with dots or slashes", () => {
+		const entries = ["frappe-bench", "doctype", "README.md", "apps/main.py"];
+		const signals = parseSignals(entries);
+		expect(signals).toContain("README.md");
+		expect(signals).toContain("apps/main.py");
+		expect(signals).not.toContain("doctype");
+	});
+});
+
+describe("detectSymbols", () => {
+	it("detects function declarations", () => {
+		const s = src(
+			"export function add(a: number, b: number): number { return a + b; }",
+		);
+		const symbols = detectSymbols(s);
+		const fn = symbols.find((sym) => sym.kind === "function");
+		expect(fn).toBeDefined();
+		expect(fn!.name).toBe("add");
+		expect(fn!.signature).toContain("add");
+	});
+
+	it("detects async functions", () => {
+		const s = src("async function fetchData(): Promise<string> { return ''; }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "function")).toBe(true);
+	});
+
+	it("detects class declarations", () => {
+		const s = src("export class UserService { }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "class")).toBe(true);
+	});
+
+	it("detects interface declarations", () => {
+		const s = src("export interface Config { port: number; }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "interface")).toBe(true);
+	});
+
+	it("detects type aliases", () => {
+		const s = src("export type ID = string | number;");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "type")).toBe(true);
+	});
+
+	it("detects const exports", () => {
+		const s = src("export const MAX_RETRIES = 3;");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "constant")).toBe(true);
+	});
+
+	it("includes file and line number", () => {
+		const s = src("function a() {}\nfunction b() {}");
+		const symbols = detectSymbols(s);
+		const b = symbols.find((sym) => sym.name === "b");
+		expect(b!.line).toBe(2);
+	});
+});
+
+describe("generateDocs", () => {
+	it("generates overview section", () => {
+		const sources = [
+			src("export function hello() {}", "src/a.ts"),
+			src("export type Config = {}", "src/b.ts"),
+		];
+		const docs = generateDocs(sources, ["next.config.js"]);
+		expect(docs.sections.some((s) => s.heading === "Overview")).toBe(true);
+		expect(docs.sections.some((s) => s.heading === "Functions")).toBe(true);
+	});
+
+	it("sets project type from signals", () => {
+		const docs = generateDocs([], ["vite.config.ts", "src/main.tsx"]);
+		expect(docs.projectType).toBe("react_vite");
+	});
+
+	it("lists exported symbols", () => {
+		const sources = [src("export function fn() {}")];
+		const docs = generateDocs(sources, []);
+		expect(docs.symbols.length).toBeGreaterThan(0);
+	});
+
+	it("includes exported functions in sections", () => {
+		const sources = [src("export function publicFn() {}")];
+		const docs = generateDocs(sources, []);
+		const fnSection = docs.sections.find((s) => s.heading === "Functions");
+		expect(fnSection!.content).toContain("publicFn");
+	});
+});

package/packages/doc-generator/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": { "outDir": "./dist", "rootDir": "./src", "types": ["node"], "lib": ["ES2022"] },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "test"]
+}

package/packages/evaluation-engine/dist/engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACX,iBAAiB,EACjB,gBAAgB,EAKhB,eAAe,EAIf,cAAc,EACd,MAAM,YAAY,CAAC;AA2TpB,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,KAAK,CAA8B;gBAE/B,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAI7C;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IAyHtD;;OAEG;IACH,QAAQ,IAAI,eAAe;IAI3B;;OAEG;IACH,KAAK,IAAI,IAAI;CAGb;AAID,wBAAgB,sBAAsB,CACrC,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,gBAAgB,CAElB"}
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACX,iBAAiB,EACjB,gBAAgB,EAKhB,eAAe,EAIf,cAAc,EACd,MAAM,YAAY,CAAC;AA2TpB,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,KAAK,CAA8B;IAE3C,YAAY,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,EAE5C;IAED;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,CAuHrD;IAED;;OAEG;IACH,QAAQ,IAAI,eAAe,CAE1B;IAED;;OAEG;IACH,KAAK,IAAI,IAAI,CAEZ;CACD;AAID,wBAAgB,sBAAsB,CACrC,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,gBAAgB,CAElB"}

package/packages/evaluation-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/evaluation-engine",
-  "version": "0.9.1",
+  "version": "0.9.5",
   "description": "Evaluation engine for pi-harness runtime (RFC-0057)",
   "type": "module",
   "main": "./dist/index.js",
@@ -18,6 +18,6 @@
   },
   "devDependencies": {
     "@types/node": "^22.0.0",
-    "typescript": "^5.5.0"
+    "typescript": "^7.0.2"
   }
 }

package/packages/evaluation-runner/dist/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Evaluation Runner — Main Entry (RFC-0020)
+ */
+export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
+export type * from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/packages/evaluation-runner/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9G,mBAAmB,YAAY,CAAC"}

package/packages/evaluation-runner/dist/index.js

@@ -0,0 +1,5 @@
+/**
+ * Evaluation Runner — Main Entry (RFC-0020)
+ */
+export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
+//# sourceMappingURL=index.js.map

package/packages/evaluation-runner/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC"}

package/packages/evaluation-runner/dist/runner.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Evaluation Runner — Core (RFC-0020)
+ */
+import type { EvaluationConfig, EvaluationInput, EvaluatorFn, EvaluationReport, EvaluationSuite } from "./types.js";
+export declare class EvaluationRunner {
+    private evaluators;
+    private config;
+    constructor(config?: EvaluationConfig);
+    registerEvaluator(name: string, fn: EvaluatorFn): void;
+    runSuite(suite: EvaluationSuite, execute: (input: EvaluationInput) => Promise<string>): Promise<EvaluationReport>;
+    private runCase;
+    listEvaluators(): string[];
+}
+export declare const exactMatchEvaluator: EvaluatorFn;
+export declare const substringMatchEvaluator: EvaluatorFn;
+export declare const lengthEvaluator: EvaluatorFn;
+//# sourceMappingURL=runner.d.ts.map

package/packages/evaluation-runner/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAEX,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAEhB,eAAe,EACf,MAAM,YAAY,CAAC;AAOpB,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAA6B;IAE3C,YAAY,MAAM,GAAE,gBAAqB,EAMxC;IAED,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAErD;IAEK,QAAQ,CACb,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,GAClD,OAAO,CAAC,gBAAgB,CAAC,CAsC3B;YAEa,OAAO;IA6DrB,cAAc,IAAI,MAAM,EAAE,CAEzB;CACD;AAGD,eAAO,MAAM,mBAAmB,EAAE,WASjC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,WAUrC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,WAa7B,CAAC"}

package/packages/evaluation-runner/dist/runner.js

@@ -0,0 +1,153 @@
+/**
+ * Evaluation Runner — Core (RFC-0020)
+ */
+let _runCounter = 0;
+function nextRunId() {
+    return `run-${Date.now()}-${++_runCounter}`;
+}
+export class EvaluationRunner {
+    evaluators = new Map();
+    config;
+    constructor(config = {}) {
+        this.config = {
+            timeoutMs: config.timeoutMs ?? 30_000,
+            retries: config.retries ?? 1,
+            parallel: config.parallel ?? false,
+        };
+    }
+    registerEvaluator(name, fn) {
+        this.evaluators.set(name, fn);
+    }
+    async runSuite(suite, execute) {
+        const results = [];
+        const cases = suite.cases;
+        if (this.config.parallel) {
+            const promises = cases.map((c) => this.runCase(c, execute));
+            const all = await Promise.allSettled(promises);
+            for (const r of all) {
+                if (r.status === "fulfilled")
+                    results.push(...r.value);
+            }
+        }
+        else {
+            for (const c of cases) {
+                const r = await this.runCase(c, execute);
+                results.push(...r);
+            }
+        }
+        const passed = results.filter((r) => r.passed).length;
+        const summary = {};
+        for (const r of results) {
+            if (!summary[r.metric]) {
+                summary[r.metric] = { count: 0, avgScore: 0 };
+            }
+            const s = summary[r.metric];
+            s.count++;
+            s.avgScore = (s.avgScore * (s.count - 1) + r.score) / s.count;
+        }
+        return {
+            suiteId: suite.id,
+            runAt: new Date().toISOString(),
+            total: results.length,
+            passed,
+            failed: results.length - passed,
+            results,
+            summary,
+        };
+    }
+    async runCase(c, execute) {
+        const start = Date.now();
+        let output;
+        try {
+            output = await Promise.race([
+                execute(c.input),
+                new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), this.config.timeoutMs)),
+            ]);
+        }
+        catch (err) {
+            return c.evaluators.map((name) => ({
+                runId: nextRunId(),
+                metric: name,
+                score: 0,
+                passed: false,
+                details: err instanceof Error ? err.message : "unknown error",
+                latencyMs: Date.now() - start,
+            }));
+        }
+        const results = [];
+        for (const name of c.evaluators) {
+            const fn = this.evaluators.get(name);
+            if (!fn) {
+                results.push({
+                    runId: nextRunId(),
+                    metric: name,
+                    score: 0,
+                    passed: false,
+                    details: `Evaluator not registered: ${name}`,
+                    latencyMs: Date.now() - start,
+                });
+                continue;
+            }
+            for (let attempt = 0; attempt < this.config.retries; attempt++) {
+                try {
+                    const r = await fn(c.input, output);
+                    results.push(r);
+                    break;
+                }
+                catch (err) {
+                    if (attempt === this.config.retries - 1) {
+                        results.push({
+                            runId: nextRunId(),
+                            metric: name,
+                            score: 0,
+                            passed: false,
+                            details: err instanceof Error ? err.message : "unknown error",
+                            latencyMs: Date.now() - start,
+                        });
+                    }
+                }
+            }
+        }
+        return results;
+    }
+    listEvaluators() {
+        return [...this.evaluators.keys()];
+    }
+}
+// Built-in evaluators
+export const exactMatchEvaluator = async (input, output) => {
+    const expected = input.expectedOutput ?? "";
+    return {
+        runId: nextRunId(),
+        metric: "exact-match",
+        score: output === expected ? 1 : 0,
+        passed: output === expected,
+        details: output === expected ? "exact match" : "output differs from expected",
+    };
+};
+export const substringMatchEvaluator = async (input, output) => {
+    const expected = input.expectedOutput ?? "";
+    const found = expected.length > 0 && output.includes(expected);
+    return {
+        runId: nextRunId(),
+        metric: "substring-match",
+        score: found ? 1 : 0,
+        passed: found,
+        details: found ? "expected substring found" : "expected substring not found",
+    };
+};
+export const lengthEvaluator = async (input, output) => {
+    const min = input.metadata?.["minLength"] ?? 0;
+    const max = input.metadata?.["maxLength"] ?? Infinity;
+    const len = output.length;
+    const passed = len >= min && len <= max;
+    const score = passed ? 1 : Math.max(0, 1 - Math.abs(len - (min + max) / 2) / max);
+    return {
+        runId: nextRunId(),
+        metric: "length-check",
+        score,
+        passed,
+        details: `length ${len}, expected ${min}-${max}`,
+    };
+};
+//# sourceMappingURL=runner.js.map

package/packages/evaluation-runner/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,SAAS,SAAS;IACjB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,OAAO,gBAAgB;IACpB,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,MAAM,CAA6B;IAE3C,YAAY,MAAM,GAAqB,EAAE;QACxC,IAAI,CAAC,MAAM,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;YAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;SAClC,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAAY,EAAE,EAAe;QAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,QAAQ,CACb,KAAsB,EACtB,OAAoD;QAEpD,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACxD,CAAC;QACF,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACpB,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,OAAO,GAAgC,EAAE,CAAC;QAEhD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC;YACD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QAC/D,CAAC;QAED,OAAO;YACN,OAAO,EAAE,KAAK,CAAC,EAAE;YACjB,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;YAC/B,OAAO;YACP,OAAO;SACP,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACpB,CAAiB,EACjB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACJ,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC3B,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChB,IAAI,OAAO,CAAS,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACjC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CACrE;aACD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,SAAS,EAAE;gBAClB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC7B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,SAAS,EAAE;oBAClB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,6BAA6B,IAAI,EAAE;oBAC5C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAC7B,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACJ,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;wBACzC,OAAO,CAAC,IAAI,CAAC;4BACZ,KAAK,EAAE,SAAS,EAAE;4BAClB,MAAM,EAAE,IAAI;4BACZ,KAAK,EAAE,CAAC;4BACR,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;4BAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;yBAC7B,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,cAAc;QACb,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;CACD;AAED,sBAAsB;AACtB,MAAM,CAAC,MAAM,mBAAmB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,aAAa;QACrB,KAAK,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,MAAM,KAAK,QAAQ;QAC3B,OAAO,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,8BAA8B;KAC7E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,iBAAiB;QACzB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,8BAA8B;KAC5E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACnE,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,QAAQ,CAAC;IAClE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1B,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IAClF,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,cAAc;QACtB,KAAK;QACL,MAAM;QACN,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,IAAI,GAAG,EAAE;KAChD,CAAC;AACH,CAAC,CAAC"}

package/packages/evaluation-runner/dist/types.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+export interface EvaluationInput {
+    prompt: string;
+    expectedOutput?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface EvaluationResult {
+    runId: string;
+    metric: string;
+    score: number;
+    passed: boolean;
+    details?: string;
+    latencyMs?: number;
+}
+export interface EvaluationSuite {
+    id: string;
+    name: string;
+    cases: EvaluationCase[];
+    thresholds: Record<string, number>;
+}
+export interface EvaluationCase {
+    id: string;
+    name: string;
+    input: EvaluationInput;
+    expected?: string;
+    evaluators: string[];
+}
+export type EvaluatorFn = (input: EvaluationInput, output: string) => Promise<EvaluationResult>;
+export interface EvaluationReport {
+    suiteId: string;
+    runAt: string;
+    total: number;
+    passed: number;
+    failed: number;
+    results: EvaluationResult[];
+    summary: Record<string, {
+        count: number;
+        avgScore: number;
+    }>;
+}
+export interface EvaluationConfig {
+    timeoutMs?: number;
+    retries?: number;
+    parallel?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/packages/evaluation-runner/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAE/F,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB"}

package/packages/evaluation-runner/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/packages/evaluation-runner/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/packages/evaluation-runner/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@pi/evaluation-runner",
+  "version": "0.9.5",
+  "type": "module",
+  "description": "Framework-agnostic evaluation framework for AI outputs (RFC-0020)",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc --noEmit",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0"
+  }
+}

package/packages/evaluation-runner/src/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Evaluation Runner — Main Entry (RFC-0020)
+ */
+
+export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
+export type * from "./types.js";