pi-harness-runtime 1.1.439 → 1.1.441

package/packages/evaluation-engine/test/engine.test.ts

@@ -1,277 +0,0 @@
-/**
- * Evaluation Engine Tests (RFC-0057)
- */
-
-import { describe, it, expect, beforeEach } from "@jest/globals";
-import { type EvaluationEngine, createEvaluationEngine } from "../src/index.js";
-import type { EvaluationRequest } from "../src/types.js";
-
-describe("EvaluationEngine", () => {
-	let engine: EvaluationEngine;
-	let defaultTask: EvaluationRequest["task"];
-	let defaultOutputs: EvaluationRequest["outputs"];
-	let defaultTestResults: EvaluationRequest["testResults"];
-	let defaultPolicyFindings: EvaluationRequest["policyFindings"];
-
-	beforeEach(() => {
-		engine = createEvaluationEngine();
-		defaultTask = {
-			id: "task-1",
-			title: "Test Task",
-			description: "A test task",
-			acceptanceCriteria: ["Output file exists", "Tests pass"],
-			requiredArtifacts: ["output.txt"],
-			mandatoryStages: [],
-		};
-		defaultOutputs = [
-			{
-				id: "out-1",
-				type: "file",
-				path: "output.txt",
-				description: "Generated output file",
-			},
-		];
-		defaultTestResults = [
-			{
-				id: "test-1",
-				testFile: "test.spec.ts",
-				status: "passed",
-				durationMs: 100,
-			},
-		];
-		defaultPolicyFindings = [];
-	});
-
-	describe("evaluate", () => {
-		it("passes when all criteria are met", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("passed");
-			expect(result.score).toBeGreaterThanOrEqual(70);
-			expect(result.recommendedAction).toBe("continue");
-			expect(result.evidence.length).toBeGreaterThan(0);
-		});
-
-		it("fails when required artifact is missing", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: [], // No outputs
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.message.includes("missing"))).toBe(
-				true,
-			);
-		});
-
-		it("fails on critical policy violation", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: [
-					{
-						ruleId: "SEC-001",
-						severity: "critical",
-						message: "Hardcoded API key detected",
-						filePath: "src/config.ts",
-						line: 10,
-					},
-				],
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.severity === "critical")).toBe(true);
-			expect(result.recommendedAction).toBe("repair");
-		});
-
-		it("fails when tests fail", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: [
-					{
-						id: "test-1",
-						testFile: "test.spec.ts",
-						status: "failed",
-						errorMessage: "Assertion failed: expected 2 to equal 3",
-					},
-				],
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.status).toBe("failed");
-			expect(result.findings.some((f) => f.message.includes("failed"))).toBe(
-				true,
-			);
-		});
-
-		it("returns needs_review for ambiguous criteria", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: {
-					...defaultTask,
-					acceptanceCriteria: ["Do something good", "Make it work well"],
-				},
-				worktreePath: "/tmp/worktree",
-				outputs: [
-					{
-						id: "out-1",
-						type: "file",
-						path: "output.txt",
-						description: "Some output",
-					},
-				],
-				testResults: [
-					{
-						id: "test-1",
-						testFile: "test.spec.ts",
-						status: "passed",
-					},
-				],
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			// Should pass or needs_review depending on coverage
-			expect(["passed", "needs_review"]).toContain(result.status);
-		});
-
-		it("includes evidence in result", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.evidence).toBeDefined();
-			expect(Array.isArray(result.evidence)).toBe(true);
-		});
-
-		it("maps acceptance criteria to evidence", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: {
-					...defaultTask,
-					acceptanceCriteria: ["Output file exists"],
-				},
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			const acceptanceFinding = result.findings.find(
-				(f) => f.dimension === "acceptance_coverage",
-			);
-
-			// Should either pass or have unmet criteria finding
-			expect(result.status).toBe("passed");
-		});
-	});
-
-	describe("state management", () => {
-		it("starts in pending state", () => {
-			const freshEngine = createEvaluationEngine();
-			expect(freshEngine.getState()).toBe("pending");
-		});
-
-		it("transitions through states during evaluation", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			engine.evaluate(request);
-
-			expect(["passed", "failed", "needs_review"]).toContain(engine.getState());
-		});
-
-		it("resets state correctly", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			engine.evaluate(request);
-			engine.reset();
-
-			expect(engine.getState()).toBe("pending");
-		});
-	});
-
-	describe("scoring", () => {
-		it("returns score between 0 and 100", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: defaultTestResults,
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			expect(result.score).toBeGreaterThanOrEqual(0);
-			expect(result.score).toBeLessThanOrEqual(100);
-		});
-
-		it("penalizes for missing tests", () => {
-			const request: EvaluationRequest = {
-				jobId: "job-1",
-				task: defaultTask,
-				worktreePath: "/tmp/worktree",
-				outputs: defaultOutputs,
-				testResults: [], // No tests
-				policyFindings: defaultPolicyFindings,
-			};
-
-			const result = engine.evaluate(request);
-
-			// Missing tests should lower score
-			expect(
-				result.findings.some((f) => f.message.includes("No test results")),
-			).toBe(true);
-		});
-	});
-});

package/packages/evaluation-runner/test/runner.test.ts

@@ -1,168 +0,0 @@
-/**
- * Evaluation Runner Tests (RFC-0020)
- */
-
-import { describe, it, expect, beforeEach } from "bun:test";
-import {
-  EvaluationRunner,
-  exactMatchEvaluator,
-  substringMatchEvaluator,
-  lengthEvaluator,
-} from "../src/index.js";
-import type { EvaluationSuite } from "../src/index.js";
-
-describe("EvaluationRunner", () => {
-  let runner: EvaluationRunner;
-
-  beforeEach(() => { runner = new EvaluationRunner(); });
-
-  it("registers and lists evaluators", () => {
-    runner.registerEvaluator("custom", async () => ({ runId: "r1", metric: "custom", score: 0.5, passed: true }));
-    expect(runner.listEvaluators()).toContain("custom");
-  });
-
-  it("runs suite and returns report", async () => {
-    runner.registerEvaluator("exact", exactMatchEvaluator);
-    const suite: EvaluationSuite = {
-      id: "s1",
-      name: "Basic",
-      cases: [{
-        id: "c1",
-        name: "exact match",
-        input: { prompt: "hello", expectedOutput: "world" },
-        evaluators: ["exact"],
-      }],
-      thresholds: { "exact-match": 1 },
-    };
-    const report = await runner.runSuite(suite, async () => "world");
-    expect(report.total).toBe(1);
-    expect(report.passed).toBe(1);
-    expect(report.results[0].score).toBe(1);
-  });
-
-  it("marks failed when output differs", async () => {
-    runner.registerEvaluator("exact", exactMatchEvaluator);
-    const suite: EvaluationSuite = {
-      id: "s2",
-      name: "Fail",
-      cases: [{
-        id: "c1",
-        name: "differs",
-        input: { prompt: "hello", expectedOutput: "world" },
-        evaluators: ["exact"],
-      }],
-      thresholds: {},
-    };
-    const report = await runner.runSuite(suite, async () => "other");
-    expect(report.failed).toBe(1);
-    expect(report.passed).toBe(0);
-  });
-
-  it("handles timeout", async () => {
-    runner = new EvaluationRunner({ timeoutMs: 10 });
-    runner.registerEvaluator("exact", exactMatchEvaluator);
-    const suite: EvaluationSuite = {
-      id: "s3",
-      name: "Timeout",
-      cases: [{
-        id: "c1",
-        name: "slow",
-        input: { prompt: "hello", expectedOutput: "world" },
-        evaluators: ["exact"],
-      }],
-      thresholds: {},
-    };
-    const report = await runner.runSuite(suite, async () => new Promise((r) => setTimeout(() => r("world"), 200)));
-    expect(report.failed).toBe(1);
-    expect(report.results[0].details).toContain("timeout");
-  });
-
-  it("registers unknown evaluator gracefully", async () => {
-    const suite: EvaluationSuite = {
-      id: "s4",
-      name: "Unknown",
-      cases: [{
-        id: "c1",
-        name: "unknown",
-        input: { prompt: "hello" },
-        evaluators: ["not-registered"],
-      }],
-      thresholds: {},
-    };
-    const report = await runner.runSuite(suite, async () => "output");
-    expect(report.total).toBe(1);
-    expect(report.results[0].details).toContain("not registered");
-  });
-
-  it("calculates summary averages", async () => {
-    runner.registerEvaluator("exact", exactMatchEvaluator);
-    const suite: EvaluationSuite = {
-      id: "s5",
-      name: "Multi",
-      cases: [
-        { id: "c1", name: "a", input: { prompt: "a", expectedOutput: "x" }, evaluators: ["exact"] },
-        { id: "c2", name: "b", input: { prompt: "b", expectedOutput: "y" }, evaluators: ["exact"] },
-      ],
-      thresholds: {},
-    };
-    const report = await runner.runSuite(suite, async () => "x");
-    expect(report.summary["exact-match"]?.count).toBe(2);
-  });
-});
-
-describe("exactMatchEvaluator", () => {
-  it("scores 1 for exact match", async () => {
-    const result = await exactMatchEvaluator(
-      { prompt: "hello", expectedOutput: "world" },
-      "world",
-    );
-    expect(result.score).toBe(1);
-    expect(result.passed).toBe(true);
-  });
-
-  it("scores 0 for mismatch", async () => {
-    const result = await exactMatchEvaluator(
-      { prompt: "hello", expectedOutput: "world" },
-      "other",
-    );
-    expect(result.score).toBe(0);
-    expect(result.passed).toBe(false);
-  });
-});
-
-describe("substringMatchEvaluator", () => {
-  it("passes when expected is substring", async () => {
-    const result = await substringMatchEvaluator(
-      { prompt: "hello", expectedOutput: "world peace" },
-      "hello world peace today",
-    );
-    expect(result.passed).toBe(true);
-    expect(result.score).toBe(1);
-  });
-
-  it("fails when expected not found", async () => {
-    const result = await substringMatchEvaluator(
-      { prompt: "hello", expectedOutput: "xyz" },
-      "abc def",
-    );
-    expect(result.passed).toBe(false);
-  });
-});
-
-describe("lengthEvaluator", () => {
-  it("passes when within range", async () => {
-    const result = await lengthEvaluator(
-      { prompt: "hello", metadata: { minLength: 5, maxLength: 20 } },
-      "hello world",
-    );
-    expect(result.passed).toBe(true);
-  });
-
-  it("fails when too long", async () => {
-    const result = await lengthEvaluator(
-      { prompt: "hello", metadata: { minLength: 5, maxLength: 10 } },
-      "this is a very long response",
-    );
-    expect(result.passed).toBe(false);
-  });
-});

package/packages/event-bus/test/bus.test.ts

@@ -1,100 +0,0 @@
-/// <reference types="node" />
-import { describe, it } from "node:test";
-import { equal, deepEqual } from "node:assert";
-import { EventBus } from "../src/bus.js";
-import type { EventPayload } from "../src/types.js";
-
-describe("EventBus", () => {
-	const bus = new EventBus();
-
-	it("publish returns eventId", () => {
-		const id = bus.publish("test.topic", { msg: "hello" });
-		equal(typeof id, "string");
-		equal(id.length > 0, true);
-	});
-
-	it("subscriber receives event", () => {
-		// The value is held in a box: a variable that is only assigned in a
-		// callback would be narrowed to null by the type checker.
-		const received: { value: EventPayload<{ data: string }> | null } = {
-			value: null,
-		};
-		bus.subscribe<{ data: string }>("test.1", (p) => {
-			received.value = p;
-		});
-		bus.publish("test.1", { data: "hello" });
-		// The subscriber receives the envelope of the event, not the raw data.
-		equal(received.value?.topic, "test.1");
-		equal(received.value?.data.data, "hello");
-		equal(typeof received.value?.eventId, "string");
-		equal(typeof received.value?.timestamp, "string");
-		// A published event comes from the system unless a source is given.
-		equal(received.value?.source, "system");
-	});
-
-	it("multiple subscribers all called", () => {
-		const calls: number[] = [];
-		bus.subscribe("test.2", () => {
-			calls.push(1);
-		});
-		bus.subscribe("test.2", () => {
-			calls.push(2);
-		});
-		bus.publish("test.2", { x: 1 });
-		deepEqual(calls, [1, 2]);
-	});
-
-	it("unsubscribe stops notifications", () => {
-		let called = false;
-		const id = bus.subscribe("test.3", () => {
-			called = true;
-		});
-		bus.unsubscribe(id);
-		bus.publish("test.3", {});
-		equal(called, false);
-	});
-
-	it("filter predicates work", () => {
-		let called = false;
-		bus.subscribe(
-			"test.4",
-			() => {
-				called = true;
-			},
-			(d) => (d as { ok: boolean }).ok === true,
-		);
-		bus.publish("test.4", { ok: false });
-		equal(called, false);
-		bus.publish("test.4", { ok: true });
-		equal(called, true);
-	});
-
-	it("priority ordering respected", () => {
-		const order: number[] = [];
-		bus.subscribe(
-			"test.5",
-			() => {
-				order.push(1);
-			},
-			undefined,
-			1,
-		);
-		bus.subscribe(
-			"test.5",
-			() => {
-				order.push(2);
-			},
-			undefined,
-			2,
-		);
-		bus.publish("test.5", {});
-		deepEqual(order, [2, 1]); // higher priority first
-	});
-
-	it("unsubscribeAll(topic) removes all for topic", () => {
-		bus.subscribe("test.6", () => {});
-		bus.subscribe("test.6", () => {});
-		bus.unsubscribeAll("test.6");
-		equal(bus.getSubscribers("test.6").length, 0);
-	});
-});

package/packages/event-store/test/store.test.ts

@@ -1,115 +0,0 @@
-import { describe, it, beforeEach, afterEach } from "node:test";
-import { equal } from "node:assert";
-import { join } from "node:path";
-import { mkdirSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { EventStore } from "../src/store.js";
-
-describe("EventStore", () => {
-	const dir = join(tmpdir(), "event-store-test-" + Date.now());
-	beforeEach(() => mkdirSync(dir, { recursive: true }));
-	afterEach(() => rmSync(dir, { recursive: true, force: true }));
-
-	const store = new EventStore({ sessionsDir: dir });
-
-	it("append generates id and timestamp", async () => {
-		const result = await store.append({
-			sessionId: "test-1",
-			type: "message",
-			role: "user",
-			content: "hello",
-		});
-		equal(typeof result.id, "string");
-		equal(result.id.length > 0, true);
-		equal(typeof result.timestamp, "string");
-	});
-
-	it("read returns appended events", async () => {
-		await store.append({
-			sessionId: "s1",
-			type: "message",
-			role: "user",
-			content: "hello",
-		});
-		await store.append({
-			sessionId: "s1",
-			type: "message",
-			role: "assistant",
-			content: "hi",
-		});
-		const events = await store.read({ sessionId: "s1" });
-		equal(events.length, 2);
-		equal(events[0].content, "hello");
-	});
-
-	it("read filters by type", async () => {
-		await store.append({
-			sessionId: "s2",
-			type: "message",
-			role: "user",
-			content: "a",
-		});
-		await store.append({
-			sessionId: "s2",
-			type: "tool_start",
-			tool: "bash",
-			content: "",
-		});
-		const events = await store.read({ sessionId: "s2", types: ["message"] });
-		equal(events.length, 1);
-	});
-
-	it("search finds matching content", async () => {
-		await store.append({
-			sessionId: "s3",
-			type: "message",
-			role: "user",
-			content: "fix the login bug",
-		});
-		await store.append({
-			sessionId: "s3",
-			type: "message",
-			role: "user",
-			content: "add dark mode",
-		});
-		const events = await store.search({ query: "login" });
-		equal(events.length >= 1, true);
-		equal(events[0].content?.includes("login"), true);
-	});
-
-	it("stats returns counts", async () => {
-		await store.append({
-			sessionId: "s4",
-			type: "message",
-			role: "user",
-			content: "a",
-		});
-		await store.append({
-			sessionId: "s4",
-			type: "message",
-			role: "user",
-			content: "b",
-		});
-		const st = store.stats("s4");
-		equal(st.totalEvents, 2);
-		equal(st.sizeBytes > 0, true);
-	});
-
-	it("list returns session IDs", async () => {
-		await store.append({
-			sessionId: "alpha",
-			type: "message",
-			role: "user",
-			content: "x",
-		});
-		await store.append({
-			sessionId: "beta",
-			type: "message",
-			role: "user",
-			content: "y",
-		});
-		const sessions = store.list();
-		equal(sessions.includes("alpha"), true);
-		equal(sessions.includes("beta"), true);
-	});
-});