pi-harness-runtime 0.10.7 → 0.10.11

package/packages/evaluation-engine/src/types.ts

@@ -1,136 +0,0 @@
-/**
- * Evaluation Engine Types (RFC-0057)
- *
- * Interfaces for task evaluation and acceptance criteria checking.
- */
-
-// --- Task Output Types ---------------------------------------------------------
-
-export interface TaskOutputArtifact {
-	id: string;
-	type: "file" | "directory" | "url" | "test-result" | "screenshot" | "log";
-	path?: string;
-	url?: string;
-	description: string;
-	hash?: string;
-	timestamp?: string;
-}
-
-export interface TestResult {
-	id: string;
-	testFile: string;
-	status: "passed" | "failed" | "skipped" | "error";
-	durationMs?: number;
-	errorMessage?: string;
-	testCount?: number;
-	passedCount?: number;
-	failedCount?: number;
-}
-
-export interface PolicyFinding {
-	ruleId: string;
-	severity: "critical" | "high" | "medium" | "low";
-	message: string;
-	filePath?: string;
-	line?: number;
-}
-
-export interface CompiledTask {
-	id: string;
-	title: string;
-	description: string;
-	acceptanceCriteria: string[];
-	requiredArtifacts: string[];
-	mandatoryStages: string[];
-}
-
-// --- Evaluation Types ---------------------------------------------------------
-
-export interface EvaluationRequest {
-	jobId: string;
-	task: CompiledTask;
-	worktreePath: string;
-	outputs: TaskOutputArtifact[];
-	testResults: TestResult[];
-	policyFindings: PolicyFinding[];
-}
-
-export type EvaluationStatus = "passed" | "failed" | "needs_review";
-
-export interface EvaluationFinding {
-	dimension: EvaluationDimension;
-	severity: "critical" | "high" | "medium" | "low";
-	message: string;
-	evidence?: string;
-	criterion?: string;
-}
-
-export type EvaluationDimension =
-	| "correctness"
-	| "test_completeness"
-	| "acceptance_coverage"
-	| "policy_compliance"
-	| "code_quality"
-	| "security_risk"
-	| "regression_risk"
-	| "documentation";
-
-export interface EvaluationEvidence {
-	kind: "test" | "diff" | "file" | "policy" | "screenshot" | "trace";
-	path?: string;
-	description: string;
-	hash?: string;
-}
-
-export type EvaluationResultStatus = "passed" | "failed" | "needs_review";
-
-export interface EvaluationResult {
-	status: EvaluationResultStatus;
-	score: number;
-	findings: EvaluationFinding[];
-	evidence: EvaluationEvidence[];
-	recommendedAction: "continue" | "repair" | "human_review";
-	evaluationId: string;
-	jobId: string;
-	taskId: string;
-	evaluatedAt: string;
-}
-
-export type EvaluationFailureClass =
-	| "test_failure"
-	| "missing_output"
-	| "acceptance_gap"
-	| "policy_violation"
-	| "security_issue"
-	| "regression_risk"
-	| "insufficient_evidence";
-
-export type EvaluationState =
-	| "pending"
-	| "collecting_evidence"
-	| "evaluating"
-	| "passed"
-	| "failed"
-	| "needs_review";
-
-// --- Scoring Weights -----------------------------------------------------------
-
-export interface ScoringWeights {
-	correctness: number;
-	test: number;
-	acceptance: number;
-	quality: number;
-	policyPenalty: number;
-	securityPenalty: number;
-	regressionPenalty: number;
-}
-
-export const DEFAULT_SCORING_WEIGHTS: ScoringWeights = {
-	correctness: 30,
-	test: 20,
-	acceptance: 20,
-	quality: 15,
-	policyPenalty: 20,
-	securityPenalty: 30,
-	regressionPenalty: 15,
-};

package/packages/evaluation-engine/test/engine.test.ts

@@ -1,277 +0,0 @@
-/**
- * Evaluation Engine Tests (RFC-0057)
- */
-
-import { describe, it, expect, beforeEach } from "@jest/globals";
-import { type EvaluationEngine, createEvaluationEngine } from "../src/index.js";
-import type { EvaluationRequest } from "../src/types.js";
-
-describe("EvaluationEngine", () => {
-	let engine: EvaluationEngine;
-	let defaultTask: EvaluationRequest["task"];
-	let defaultOutputs: EvaluationRequest["outputs"];
-	let defaultTestResults: EvaluationRequest["testResults"];
-	let defaultPolicyFindings: EvaluationRequest["policyFindings"];
-
-	beforeEach(() => {
-		engine = createEvaluationEngine();
-		defaultTask = {
-			id: "task-1",
-			title: "Test Task",
-			description: "A test task",
-			acceptanceCriteria: ["Output file exists", "Tests pass"],
-			requiredArtifacts: ["output.txt"],
-			mandatoryStages: [],
-		};
-		defaultOutputs = [
-			{
-				id: "out-1",
-				type: "file",
-				path: "output.txt",
-				description: "Generated output file",
-			},
-		];
-		defaultTestResults = [
-			{
-				id: "test-1",
-				testFile: "test.spec.ts",
-				status: "passed",
-				durationMs: 100,
-			},
-		];
-		defaultPolicyFindings = [];
-	});
-
-	describe("evaluate", () => {
-		it("passes when all criteria are met", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("passed");
-			expect(result.score).toBeGreaterThanOrEqual(70);
-			expect(result.recommendedAction).toBe("continue");
-			expect(result.evidence.length).toBeGreaterThan(0);
-		});
-
-		it("fails when required artifact is missing", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: [], // No outputs
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.message.includes("missing"))).toBe(
-				true,
-			);
-		});
-
-		it("fails on critical policy violation", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: [
-					{
-						ruleId: "SEC-001",
-						severity: "critical",
-						message: "Hardcoded API key detected",
-						filePath: "src/config.ts",
-						line: 10,
-					},
-				],
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.severity === "critical")).toBe(true);
-			expect(result.recommendedAction).toBe("repair");
-		});
-
-		it("fails when tests fail", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: [
-					{
-						id: "test-1",
-						testFile: "test.spec.ts",
-						status: "failed",
-						errorMessage: "Assertion failed: expected 2 to equal 3",
-					},
-				],
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.message.includes("failed"))).toBe(
-				true,
-			);
-		});
-
-		it("returns needs_review for ambiguous criteria", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: {
-					...defaultTask,
-					acceptanceCriteria: ["Do something good", "Make it work well"],
-				},
-				worktreePath: "/tmp/worktree",
-				outputs: [
-					{
-						id: "out-1",
-						type: "file",
-						path: "output.txt",
-						description: "Some output",
-					},
-				],
-				testResults: [
-					{
-						id: "test-1",
-						testFile: "test.spec.ts",
-						status: "passed",
-					},
-				],
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			// Should pass or needs_review depending on coverage
-			expect(["passed", "needs_review"]).toContain(result.status);
-		});
-
-		it("includes evidence in result", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.evidence).toBeDefined();
-			expect(Array.isArray(result.evidence)).toBe(true);
-		});
-
-		it("maps acceptance criteria to evidence", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: {
-					...defaultTask,
-					acceptanceCriteria: ["Output file exists"],
-				},
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			const acceptanceFinding = result.findings.find(
-				(f) => f.dimension === "acceptance_coverage",
-			);
-
-			// Should either pass or have unmet criteria finding
-			expect(result.status).toBe("passed");
-		});
-	});
-
-	describe("state management", () => {
-		it("starts in pending state", () => {
-			const freshEngine = createEvaluationEngine();
-			expect(freshEngine.getState()).toBe("pending");
-		});
-
-		it("transitions through states during evaluation", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			engine.evaluate(request);
-
-			expect(["passed", "failed", "needs_review"]).toContain(engine.getState());
-		});
-
-		it("resets state correctly", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			engine.evaluate(request);
-			engine.reset();
-
-			expect(engine.getState()).toBe("pending");
-		});
-	});
-
-	describe("scoring", () => {
-		it("returns score between 0 and 100", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.score).toBeGreaterThanOrEqual(0);
-			expect(result.score).toBeLessThanOrEqual(100);
-		});
-
-		it("penalizes for missing tests", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: [], // No tests
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			// Missing tests should lower score
-			expect(
-				result.findings.some((f) => f.message.includes("No test results")),
-			).toBe(true);
-		});
-	});
-});

package/packages/evaluation-engine/tsconfig.json

@@ -1,8 +0,0 @@
-{
-  "extends": "../../tsconfig.base.json",
-  "compilerOptions": {
-    "outDir": "./dist",
-    "rootDir": "./src"
-  },
-  "include": ["src/**/*"]
-}

package/packages/evaluation-runner/dist/index.d.ts

@@ -1,6 +0,0 @@
-/**
- * Evaluation Runner — Main Entry (RFC-0020)
- */
-export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
-export type * from "./types.js";
-//# sourceMappingURL=index.d.ts.map

package/packages/evaluation-runner/dist/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9G,mBAAmB,YAAY,CAAC"}

package/packages/evaluation-runner/dist/index.js

@@ -1,5 +0,0 @@
-/**
- * Evaluation Runner — Main Entry (RFC-0020)
- */
-export { EvaluationRunner, exactMatchEvaluator, substringMatchEvaluator, lengthEvaluator } from "./runner.js";
-//# sourceMappingURL=index.js.map

package/packages/evaluation-runner/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC"}

package/packages/evaluation-runner/dist/runner.d.ts

@@ -1,17 +0,0 @@
-/**
- * Evaluation Runner — Core (RFC-0020)
- */
-import type { EvaluationConfig, EvaluationInput, EvaluatorFn, EvaluationReport, EvaluationSuite } from "./types.js";
-export declare class EvaluationRunner {
-    private evaluators;
-    private config;
-    constructor(config?: EvaluationConfig);
-    registerEvaluator(name: string, fn: EvaluatorFn): void;
-    runSuite(suite: EvaluationSuite, execute: (input: EvaluationInput) => Promise<string>): Promise<EvaluationReport>;
-    private runCase;
-    listEvaluators(): string[];
-}
-export declare const exactMatchEvaluator: EvaluatorFn;
-export declare const substringMatchEvaluator: EvaluatorFn;
-export declare const lengthEvaluator: EvaluatorFn;
-//# sourceMappingURL=runner.d.ts.map

package/packages/evaluation-runner/dist/runner.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAEX,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAEhB,eAAe,EACf,MAAM,YAAY,CAAC;AAOpB,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,MAAM,CAA6B;IAE3C,YAAY,MAAM,GAAE,gBAAqB,EAMxC;IAED,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,GAAG,IAAI,CAErD;IAEK,QAAQ,CACb,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,GAClD,OAAO,CAAC,gBAAgB,CAAC,CAsC3B;YAEa,OAAO;IA6DrB,cAAc,IAAI,MAAM,EAAE,CAEzB;CACD;AAGD,eAAO,MAAM,mBAAmB,EAAE,WASjC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,WAUrC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,WAa7B,CAAC"}

package/packages/evaluation-runner/dist/runner.js

@@ -1,153 +0,0 @@
-/**
- * Evaluation Runner — Core (RFC-0020)
- */
-let _runCounter = 0;
-function nextRunId() {
-    return `run-${Date.now()}-${++_runCounter}`;
-}
-export class EvaluationRunner {
-    evaluators = new Map();
-    config;
-    constructor(config = {}) {
-        this.config = {
-            timeoutMs: config.timeoutMs ?? 30_000,
-            retries: config.retries ?? 1,
-            parallel: config.parallel ?? false,
-        };
-    }
-    registerEvaluator(name, fn) {
-        this.evaluators.set(name, fn);
-    }
-    async runSuite(suite, execute) {
-        const results = [];
-        const cases = suite.cases;
-        if (this.config.parallel) {
-            const promises = cases.map((c) => this.runCase(c, execute));
-            const all = await Promise.allSettled(promises);
-            for (const r of all) {
-                if (r.status === "fulfilled")
-                    results.push(...r.value);
-            }
-        }
-        else {
-            for (const c of cases) {
-                const r = await this.runCase(c, execute);
-                results.push(...r);
-            }
-        }
-        const passed = results.filter((r) => r.passed).length;
-        const summary = {};
-        for (const r of results) {
-            if (!summary[r.metric]) {
-                summary[r.metric] = { count: 0, avgScore: 0 };
-            }
-            const s = summary[r.metric];
-            s.count++;
-            s.avgScore = (s.avgScore * (s.count - 1) + r.score) / s.count;
-        }
-        return {
-            suiteId: suite.id,
-            runAt: new Date().toISOString(),
-            total: results.length,
-            passed,
-            failed: results.length - passed,
-            results,
-            summary,
-        };
-    }
-    async runCase(c, execute) {
-        const start = Date.now();
-        let output;
-        try {
-            output = await Promise.race([
-                execute(c.input),
-                new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), this.config.timeoutMs)),
-            ]);
-        }
-        catch (err) {
-            return c.evaluators.map((name) => ({
-                runId: nextRunId(),
-                metric: name,
-                score: 0,
-                passed: false,
-                details: err instanceof Error ? err.message : "unknown error",
-                latencyMs: Date.now() - start,
-            }));
-        }
-        const results = [];
-        for (const name of c.evaluators) {
-            const fn = this.evaluators.get(name);
-            if (!fn) {
-                results.push({
-                    runId: nextRunId(),
-                    metric: name,
-                    score: 0,
-                    passed: false,
-                    details: `Evaluator not registered: ${name}`,
-                    latencyMs: Date.now() - start,
-                });
-                continue;
-            }
-            for (let attempt = 0; attempt < this.config.retries; attempt++) {
-                try {
-                    const r = await fn(c.input, output);
-                    results.push(r);
-                    break;
-                }
-                catch (err) {
-                    if (attempt === this.config.retries - 1) {
-                        results.push({
-                            runId: nextRunId(),
-                            metric: name,
-                            score: 0,
-                            passed: false,
-                            details: err instanceof Error ? err.message : "unknown error",
-                            latencyMs: Date.now() - start,
-                        });
-                    }
-                }
-            }
-        }
-        return results;
-    }
-    listEvaluators() {
-        return [...this.evaluators.keys()];
-    }
-}
-// Built-in evaluators
-export const exactMatchEvaluator = async (input, output) => {
-    const expected = input.expectedOutput ?? "";
-    return {
-        runId: nextRunId(),
-        metric: "exact-match",
-        score: output === expected ? 1 : 0,
-        passed: output === expected,
-        details: output === expected ? "exact match" : "output differs from expected",
-    };
-};
-export const substringMatchEvaluator = async (input, output) => {
-    const expected = input.expectedOutput ?? "";
-    const found = expected.length > 0 && output.includes(expected);
-    return {
-        runId: nextRunId(),
-        metric: "substring-match",
-        score: found ? 1 : 0,
-        passed: found,
-        details: found ? "expected substring found" : "expected substring not found",
-    };
-};
-export const lengthEvaluator = async (input, output) => {
-    const min = input.metadata?.["minLength"] ?? 0;
-    const max = input.metadata?.["maxLength"] ?? Infinity;
-    const len = output.length;
-    const passed = len >= min && len <= max;
-    const score = passed ? 1 : Math.max(0, 1 - Math.abs(len - (min + max) / 2) / max);
-    return {
-        runId: nextRunId(),
-        metric: "length-check",
-        score,
-        passed,
-        details: `length ${len}, expected ${min}-${max}`,
-    };
-};
-//# sourceMappingURL=runner.js.map

package/packages/evaluation-runner/dist/runner.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,SAAS,SAAS;IACjB,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,OAAO,gBAAgB;IACpB,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,MAAM,CAA6B;IAE3C,YAAY,MAAM,GAAqB,EAAE;QACxC,IAAI,CAAC,MAAM,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;YAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;SAClC,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAAY,EAAE,EAAe;QAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,QAAQ,CACb,KAAsB,EACtB,OAAoD;QAEpD,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;oBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACxD,CAAC;QACF,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACpB,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,OAAO,GAAgC,EAAE,CAAC;QAEhD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC;YACD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QAC/D,CAAC;QAED,OAAO;YACN,OAAO,EAAE,KAAK,CAAC,EAAE;YACjB,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;YAC/B,OAAO;YACP,OAAO;SACP,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACpB,CAAiB,EACjB,OAAoD;QAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACJ,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC3B,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChB,IAAI,OAAO,CAAS,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACjC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CACrE;aACD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,SAAS,EAAE;gBAClB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAC7B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,SAAS,EAAE;oBAClB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,6BAA6B,IAAI,EAAE;oBAC5C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAC7B,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;gBAChE,IAAI,CAAC;oBACJ,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;wBACzC,OAAO,CAAC,IAAI,CAAC;4BACZ,KAAK,EAAE,SAAS,EAAE;4BAClB,MAAM,EAAE,IAAI;4BACZ,KAAK,EAAE,CAAC;4BACR,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;4BAC7D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;yBAC7B,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,cAAc;QACb,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;CACD;AAED,sBAAsB;AACtB,MAAM,CAAC,MAAM,mBAAmB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,aAAa;QACrB,KAAK,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,MAAM,KAAK,QAAQ;QAC3B,OAAO,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,8BAA8B;KAC7E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/D,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,iBAAiB;QACzB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,8BAA8B;KAC5E,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAgB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;IACnE,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAY,IAAI,QAAQ,CAAC;IAClE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1B,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IAClF,OAAO;QACN,KAAK,EAAE,SAAS,EAAE;QAClB,MAAM,EAAE,cAAc;QACtB,KAAK;QACL,MAAM;QACN,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,IAAI,GAAG,EAAE;KAChD,CAAC;AACH,CAAC,CAAC"}

package/packages/evaluation-runner/dist/types.d.ts

@@ -1,48 +0,0 @@
-/**
- * Evaluation Runner — Types (RFC-0020)
- */
-export interface EvaluationInput {
-    prompt: string;
-    expectedOutput?: string;
-    metadata?: Record<string, unknown>;
-}
-export interface EvaluationResult {
-    runId: string;
-    metric: string;
-    score: number;
-    passed: boolean;
-    details?: string;
-    latencyMs?: number;
-}
-export interface EvaluationSuite {
-    id: string;
-    name: string;
-    cases: EvaluationCase[];
-    thresholds: Record<string, number>;
-}
-export interface EvaluationCase {
-    id: string;
-    name: string;
-    input: EvaluationInput;
-    expected?: string;
-    evaluators: string[];
-}
-export type EvaluatorFn = (input: EvaluationInput, output: string) => Promise<EvaluationResult>;
-export interface EvaluationReport {
-    suiteId: string;
-    runAt: string;
-    total: number;
-    passed: number;
-    failed: number;
-    results: EvaluationResult[];
-    summary: Record<string, {
-        count: number;
-        avgScore: number;
-    }>;
-}
-export interface EvaluationConfig {
-    timeoutMs?: number;
-    retries?: number;
-    parallel?: boolean;
-}
-//# sourceMappingURL=types.d.ts.map

package/packages/evaluation-runner/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAE/F,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB"}

package/packages/evaluation-runner/dist/types.js

@@ -1,5 +0,0 @@
-/**
- * Evaluation Runner — Types (RFC-0020)
- */
-export {};
-//# sourceMappingURL=types.js.map