pi-harness-runtime 0.10.43 → 1.1.1

package/packages/doc-generator/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@pi-harness/doc-generator",
+  "version": "0.10.30",
+  "description": "Auto-Doc Generator — generate documentation from source code (RFC-0014)",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0"
+  }
+}

package/packages/doc-generator/test/docs.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Doc Generator Tests (RFC-0014)
+ */
+
+import { describe, it, expect } from "bun:test";
+import {
+	detectProjectType,
+	parseSignals,
+	detectSymbols,
+	generateDocs,
+} from "../src/index.js";
+import type { SourceFile } from "../src/index.js";
+
+const src = (content: string, path = "src/utils.ts"): SourceFile => ({
+	path,
+	content,
+	language: "typescript",
+});
+
+describe("detectProjectType", () => {
+	it("detects Frappe/ERPNext", () => {
+		const signals = ["frappe-bench/apps/myapp/hooks.py", "doctype/CRM"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("frappe_erpnext");
+		expect(result.signals).toContain("frappe-bench/apps/myapp/hooks.py");
+	});
+
+	it("detects Next.js", () => {
+		const signals = ["next.config.js", "app/layout.tsx", "pages/index.ts"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("nextjs");
+	});
+
+	it("detects React/Vite", () => {
+		const signals = ["vite.config.ts", "src/main.tsx", "index.html"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("react_vite");
+	});
+
+	it("detects Django", () => {
+		const signals = ["manage.py", "settings.py"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("django");
+	});
+
+	it("detects Laravel", () => {
+		const signals = ["artisan", "database/seeders/UserSeeder.php"];
+		const result = detectProjectType(signals);
+		expect(result.projectType).toBe("laravel");
+	});
+
+	it("returns unknown for empty signals", () => {
+		const result = detectProjectType([]);
+		expect(result.projectType).toBe("unknown");
+		expect(result.confidence).toBe(0);
+	});
+
+	it("returns generic_web for package.json", () => {
+		const result = detectProjectType(["package.json", "index.js"]);
+		expect(result.projectType).toBe("generic_web");
+	});
+
+	it("includes recommended seed strategy", () => {
+		const result = detectProjectType(["manage.py"]);
+		expect(result.recommendedSeedStrategy).toBe("django_migrations");
+	});
+
+	it("includes recommended e2e strategy", () => {
+		const result = detectProjectType(["next.config.js"]);
+		expect(result.recommendedE2EStrategy).toBe("nextjs_dev_server_flow");
+	});
+
+	it("confidence is 0-1", () => {
+		const result = detectProjectType(["doctype/CRM"]);
+		expect(result.confidence).toBeGreaterThanOrEqual(0);
+		expect(result.confidence).toBeLessThanOrEqual(1);
+	});
+});
+
+describe("parseSignals", () => {
+	it("filters paths with dots or slashes", () => {
+		const entries = ["frappe-bench", "doctype", "README.md", "apps/main.py"];
+		const signals = parseSignals(entries);
+		expect(signals).toContain("README.md");
+		expect(signals).toContain("apps/main.py");
+		expect(signals).not.toContain("doctype");
+	});
+});
+
+describe("detectSymbols", () => {
+	it("detects function declarations", () => {
+		const s = src(
+			"export function add(a: number, b: number): number { return a + b; }",
+		);
+		const symbols = detectSymbols(s);
+		const fn = symbols.find((sym) => sym.kind === "function");
+		expect(fn).toBeDefined();
+		expect(fn!.name).toBe("add");
+		expect(fn!.signature).toContain("add");
+	});
+
+	it("detects async functions", () => {
+		const s = src("async function fetchData(): Promise<string> { return ''; }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "function")).toBe(true);
+	});
+
+	it("detects class declarations", () => {
+		const s = src("export class UserService { }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "class")).toBe(true);
+	});
+
+	it("detects interface declarations", () => {
+		const s = src("export interface Config { port: number; }");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "interface")).toBe(true);
+	});
+
+	it("detects type aliases", () => {
+		const s = src("export type ID = string | number;");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "type")).toBe(true);
+	});
+
+	it("detects const exports", () => {
+		const s = src("export const MAX_RETRIES = 3;");
+		const symbols = detectSymbols(s);
+		expect(symbols.some((sym) => sym.kind === "constant")).toBe(true);
+	});
+
+	it("includes file and line number", () => {
+		const s = src("function a() {}\nfunction b() {}");
+		const symbols = detectSymbols(s);
+		const b = symbols.find((sym) => sym.name === "b");
+		expect(b!.line).toBe(2);
+	});
+});
+
+describe("generateDocs", () => {
+	it("generates overview section", () => {
+		const sources = [
+			src("export function hello() {}", "src/a.ts"),
+			src("export type Config = {}", "src/b.ts"),
+		];
+		const docs = generateDocs(sources, ["next.config.js"]);
+		expect(docs.sections.some((s) => s.heading === "Overview")).toBe(true);
+		expect(docs.sections.some((s) => s.heading === "Functions")).toBe(true);
+	});
+
+	it("sets project type from signals", () => {
+		const docs = generateDocs([], ["vite.config.ts", "src/main.tsx"]);
+		expect(docs.projectType).toBe("react_vite");
+	});
+
+	it("lists exported symbols", () => {
+		const sources = [src("export function fn() {}")];
+		const docs = generateDocs(sources, []);
+		expect(docs.symbols.length).toBeGreaterThan(0);
+	});
+
+	it("includes exported functions in sections", () => {
+		const sources = [src("export function publicFn() {}")];
+		const docs = generateDocs(sources, []);
+		const fnSection = docs.sections.find((s) => s.heading === "Functions");
+		expect(fnSection!.content).toContain("publicFn");
+	});
+});

package/packages/evaluation-engine/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@pi-harness/evaluation-engine",
+  "version": "0.10.30",
+  "description": "Evaluation engine for pi-harness runtime (RFC-0057)",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js",
+    "lint": "eslint src --ext .ts"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^7.0.2"
+  }
+}

package/packages/evaluation-engine/test/engine.test.ts

@@ -0,0 +1,277 @@
+/**
+ * Evaluation Engine Tests (RFC-0057)
+ */
+
+import { describe, it, expect, beforeEach } from "@jest/globals";
+import { type EvaluationEngine, createEvaluationEngine } from "../src/index.js";
+import type { EvaluationRequest } from "../src/types.js";
+
+describe("EvaluationEngine", () => {
+	let engine: EvaluationEngine;
+	let defaultTask: EvaluationRequest["task"];
+	let defaultOutputs: EvaluationRequest["outputs"];
+	let defaultTestResults: EvaluationRequest["testResults"];
+	let defaultPolicyFindings: EvaluationRequest["policyFindings"];
+
+	beforeEach(() => {
+		engine = createEvaluationEngine();
+		defaultTask = {
+			id: "task-1",
+			title: "Test Task",
+			description: "A test task",
+			acceptanceCriteria: ["Output file exists", "Tests pass"],
+			requiredArtifacts: ["output.txt"],
+			mandatoryStages: [],
+		};
+		defaultOutputs = [
+			{
+				id: "out-1",
+				type: "file",
+				path: "output.txt",
+				description: "Generated output file",
+			},
+		];
+		defaultTestResults = [
+			{
+				id: "test-1",
+				testFile: "test.spec.ts",
+				status: "passed",
+				durationMs: 100,
+			},
+		];
+		defaultPolicyFindings = [];
+	});
+
+	describe("evaluate", () => {
+		it("passes when all criteria are met", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.status).toBe("passed");
+			expect(result.score).toBeGreaterThanOrEqual(70);
+			expect(result.recommendedAction).toBe("continue");
+			expect(result.evidence.length).toBeGreaterThan(0);
+		});
+
+		it("fails when required artifact is missing", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: [], // No outputs
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.status).toBe("failed");
+			expect(result.findings.some((f) => f.message.includes("missing"))).toBe(
+				true,
+			);
+		});
+
+		it("fails on critical policy violation", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: [
+					{
+						ruleId: "SEC-001",
+						severity: "critical",
+						message: "Hardcoded API key detected",
+						filePath: "src/config.ts",
+						line: 10,
+					},
+				],
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.status).toBe("failed");
+			expect(result.findings.some((f) => f.severity === "critical")).toBe(true);
+			expect(result.recommendedAction).toBe("repair");
+		});
+
+		it("fails when tests fail", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: [
+					{
+						id: "test-1",
+						testFile: "test.spec.ts",
+						status: "failed",
+						errorMessage: "Assertion failed: expected 2 to equal 3",
+					},
+				],
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.status).toBe("failed");
+			expect(result.findings.some((f) => f.message.includes("failed"))).toBe(
+				true,
+			);
+		});
+
+		it("returns needs_review for ambiguous criteria", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: {
+					...defaultTask,
+					acceptanceCriteria: ["Do something good", "Make it work well"],
+				},
+				worktreePath: "/tmp/worktree",
+				outputs: [
+					{
+						id: "out-1",
+						type: "file",
+						path: "output.txt",
+						description: "Some output",
+					},
+				],
+				testResults: [
+					{
+						id: "test-1",
+						testFile: "test.spec.ts",
+						status: "passed",
+					},
+				],
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			// Should pass or needs_review depending on coverage
+			expect(["passed", "needs_review"]).toContain(result.status);
+		});
+
+		it("includes evidence in result", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.evidence).toBeDefined();
+			expect(Array.isArray(result.evidence)).toBe(true);
+		});
+
+		it("maps acceptance criteria to evidence", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: {
+					...defaultTask,
+					acceptanceCriteria: ["Output file exists"],
+				},
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			const acceptanceFinding = result.findings.find(
+				(f) => f.dimension === "acceptance_coverage",
+			);
+
+			// Should either pass or have unmet criteria finding
+			expect(result.status).toBe("passed");
+		});
+	});
+
+	describe("state management", () => {
+		it("starts in pending state", () => {
+			const freshEngine = createEvaluationEngine();
+			expect(freshEngine.getState()).toBe("pending");
+		});
+
+		it("transitions through states during evaluation", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			engine.evaluate(request);
+
+			expect(["passed", "failed", "needs_review"]).toContain(engine.getState());
+		});
+
+		it("resets state correctly", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			engine.evaluate(request);
+			engine.reset();
+
+			expect(engine.getState()).toBe("pending");
+		});
+	});
+
+	describe("scoring", () => {
+		it("returns score between 0 and 100", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: defaultTestResults,
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			expect(result.score).toBeGreaterThanOrEqual(0);
+			expect(result.score).toBeLessThanOrEqual(100);
+		});
+
+		it("penalizes for missing tests", () => {
+			const request: EvaluationRequest = {
+				jobId: "job-1",
+				task: defaultTask,
+				worktreePath: "/tmp/worktree",
+				outputs: defaultOutputs,
+				testResults: [], // No tests
+				policyFindings: defaultPolicyFindings,
+			};
+
+			const result = engine.evaluate(request);
+
+			// Missing tests should lower score
+			expect(
+				result.findings.some((f) => f.message.includes("No test results")),
+			).toBe(true);
+		});
+	});
+});

package/packages/evaluation-runner/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@pi/evaluation-runner",
+  "version": "0.10.30",
+  "type": "module",
+  "description": "Framework-agnostic evaluation framework for AI outputs (RFC-0020)",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc --noEmit",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0"
+  }
+}

package/packages/evaluation-runner/test/runner.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Evaluation Runner Tests (RFC-0020)
+ */
+
+import { describe, it, expect, beforeEach } from "bun:test";
+import {
+  EvaluationRunner,
+  exactMatchEvaluator,
+  substringMatchEvaluator,
+  lengthEvaluator,
+} from "../src/index.js";
+import type { EvaluationSuite } from "../src/index.js";
+
+describe("EvaluationRunner", () => {
+  let runner: EvaluationRunner;
+
+  beforeEach(() => { runner = new EvaluationRunner(); });
+
+  it("registers and lists evaluators", () => {
+    runner.registerEvaluator("custom", async () => ({ runId: "r1", metric: "custom", score: 0.5, passed: true }));
+    expect(runner.listEvaluators()).toContain("custom");
+  });
+
+  it("runs suite and returns report", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s1",
+      name: "Basic",
+      cases: [{
+        id: "c1",
+        name: "exact match",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: { "exact-match": 1 },
+    };
+    const report = await runner.runSuite(suite, async () => "world");
+    expect(report.total).toBe(1);
+    expect(report.passed).toBe(1);
+    expect(report.results[0].score).toBe(1);
+  });
+
+  it("marks failed when output differs", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s2",
+      name: "Fail",
+      cases: [{
+        id: "c1",
+        name: "differs",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "other");
+    expect(report.failed).toBe(1);
+    expect(report.passed).toBe(0);
+  });
+
+  it("handles timeout", async () => {
+    runner = new EvaluationRunner({ timeoutMs: 10 });
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s3",
+      name: "Timeout",
+      cases: [{
+        id: "c1",
+        name: "slow",
+        input: { prompt: "hello", expectedOutput: "world" },
+        evaluators: ["exact"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => new Promise((r) => setTimeout(() => r("world"), 200)));
+    expect(report.failed).toBe(1);
+    expect(report.results[0].details).toContain("timeout");
+  });
+
+  it("registers unknown evaluator gracefully", async () => {
+    const suite: EvaluationSuite = {
+      id: "s4",
+      name: "Unknown",
+      cases: [{
+        id: "c1",
+        name: "unknown",
+        input: { prompt: "hello" },
+        evaluators: ["not-registered"],
+      }],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "output");
+    expect(report.total).toBe(1);
+    expect(report.results[0].details).toContain("not registered");
+  });
+
+  it("calculates summary averages", async () => {
+    runner.registerEvaluator("exact", exactMatchEvaluator);
+    const suite: EvaluationSuite = {
+      id: "s5",
+      name: "Multi",
+      cases: [
+        { id: "c1", name: "a", input: { prompt: "a", expectedOutput: "x" }, evaluators: ["exact"] },
+        { id: "c2", name: "b", input: { prompt: "b", expectedOutput: "y" }, evaluators: ["exact"] },
+      ],
+      thresholds: {},
+    };
+    const report = await runner.runSuite(suite, async () => "x");
+    expect(report.summary["exact-match"]?.count).toBe(2);
+  });
+});
+
+describe("exactMatchEvaluator", () => {
+  it("scores 1 for exact match", async () => {
+    const result = await exactMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world" },
+      "world",
+    );
+    expect(result.score).toBe(1);
+    expect(result.passed).toBe(true);
+  });
+
+  it("scores 0 for mismatch", async () => {
+    const result = await exactMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world" },
+      "other",
+    );
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+});
+
+describe("substringMatchEvaluator", () => {
+  it("passes when expected is substring", async () => {
+    const result = await substringMatchEvaluator(
+      { prompt: "hello", expectedOutput: "world peace" },
+      "hello world peace today",
+    );
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it("fails when expected not found", async () => {
+    const result = await substringMatchEvaluator(
+      { prompt: "hello", expectedOutput: "xyz" },
+      "abc def",
+    );
+    expect(result.passed).toBe(false);
+  });
+});
+
+describe("lengthEvaluator", () => {
+  it("passes when within range", async () => {
+    const result = await lengthEvaluator(
+      { prompt: "hello", metadata: { minLength: 5, maxLength: 20 } },
+      "hello world",
+    );
+    expect(result.passed).toBe(true);
+  });
+
+  it("fails when too long", async () => {
+    const result = await lengthEvaluator(
+      { prompt: "hello", metadata: { minLength: 5, maxLength: 10 } },
+      "this is a very long response",
+    );
+    expect(result.passed).toBe(false);
+  });
+});

package/packages/event-bus/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@pi/event-bus",
+  "version": "0.10.30",
+  "type": "module",
+  "main": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "node --test"
+  }
+}