pi-harness-runtime 0.10.11 → 0.10.12

package/packages/evaluation-runner/src/runner.ts

@@ -0,0 +1,182 @@
+/**
+ * Evaluation Runner — Core (RFC-0020)
+ */
+
+import type {
+	EvaluationCase,
+	EvaluationConfig,
+	EvaluationInput,
+	EvaluatorFn,
+	EvaluationReport,
+	EvaluationResult,
+	EvaluationSuite,
+} from "./types.js";
+
+let _runCounter = 0;
+function nextRunId(): string {
+	return `run-${Date.now()}-${++_runCounter}`;
+}
+
+export class EvaluationRunner {
+	private evaluators = new Map<string, EvaluatorFn>();
+	private config: Required<EvaluationConfig>;
+
+	constructor(config: EvaluationConfig = {}) {
+		this.config = {
+			timeoutMs: config.timeoutMs ?? 30_000,
+			retries: config.retries ?? 1,
+			parallel: config.parallel ?? false,
+		};
+	}
+
+	registerEvaluator(name: string, fn: EvaluatorFn): void {
+		this.evaluators.set(name, fn);
+	}
+
+	async runSuite(
+		suite: EvaluationSuite,
+		execute: (input: EvaluationInput) => Promise<string>,
+	): Promise<EvaluationReport> {
+		const results: EvaluationResult[] = [];
+		const cases = suite.cases;
+
+		if (this.config.parallel) {
+			const promises = cases.map((c) => this.runCase(c, execute));
+			const all = await Promise.allSettled(promises);
+			for (const r of all) {
+				if (r.status === "fulfilled") results.push(...r.value);
+			}
+		} else {
+			for (const c of cases) {
+				const r = await this.runCase(c, execute);
+				results.push(...r);
+			}
+		}
+
+		const passed = results.filter((r) => r.passed).length;
+		const summary: EvaluationReport["summary"] = {};
+
+		for (const r of results) {
+			if (!summary[r.metric]) {
+				summary[r.metric] = { count: 0, avgScore: 0 };
+			}
+			const s = summary[r.metric];
+			s.count++;
+			s.avgScore = (s.avgScore * (s.count - 1) + r.score) / s.count;
+		}
+
+		return {
+			suiteId: suite.id,
+			runAt: new Date().toISOString(),
+			total: results.length,
+			passed,
+			failed: results.length - passed,
+			results,
+			summary,
+		};
+	}
+
+	private async runCase(
+		c: EvaluationCase,
+		execute: (input: EvaluationInput) => Promise<string>,
+	): Promise<EvaluationResult[]> {
+		const start = Date.now();
+		let output: string;
+		try {
+			output = await Promise.race([
+				execute(c.input),
+				new Promise<string>((_, reject) =>
+					setTimeout(() => reject(new Error("timeout")), this.config.timeoutMs),
+				),
+			]);
+		} catch (err) {
+			return c.evaluators.map((name) => ({
+				runId: nextRunId(),
+				metric: name,
+				score: 0,
+				passed: false,
+				details: err instanceof Error ? err.message : "unknown error",
+				latencyMs: Date.now() - start,
+			}));
+		}
+
+		const results: EvaluationResult[] = [];
+		for (const name of c.evaluators) {
+			const fn = this.evaluators.get(name);
+			if (!fn) {
+				results.push({
+					runId: nextRunId(),
+					metric: name,
+					score: 0,
+					passed: false,
+					details: `Evaluator not registered: ${name}`,
+					latencyMs: Date.now() - start,
+				});
+				continue;
+			}
+
+			for (let attempt = 0; attempt < this.config.retries; attempt++) {
+				try {
+					const r = await fn(c.input, output);
+					results.push(r);
+					break;
+				} catch (err) {
+					if (attempt === this.config.retries - 1) {
+						results.push({
+							runId: nextRunId(),
+							metric: name,
+							score: 0,
+							passed: false,
+							details: err instanceof Error ? err.message : "unknown error",
+							latencyMs: Date.now() - start,
+						});
+					}
+				}
+			}
+		}
+		return results;
+	}
+
+	listEvaluators(): string[] {
+		return [...this.evaluators.keys()];
+	}
+}
+
+// Built-in evaluators
+export const exactMatchEvaluator: EvaluatorFn = async (input, output) => {
+	const expected = input.expectedOutput ?? "";
+	return {
+		runId: nextRunId(),
+		metric: "exact-match",
+		score: output === expected ? 1 : 0,
+		passed: output === expected,
+		details: output === expected ? "exact match" : "output differs from expected",
+	};
+};
+
+export const substringMatchEvaluator: EvaluatorFn = async (input, output) => {
+	const expected = input.expectedOutput ?? "";
+	const found = expected.length > 0 && output.includes(expected);
+	return {
+		runId: nextRunId(),
+		metric: "substring-match",
+		score: found ? 1 : 0,
+		passed: found,
+		details: found ? "expected substring found" : "expected substring not found",
+	};
+};
+
+export const lengthEvaluator: EvaluatorFn = async (input, output) => {
+	const min = (input.metadata?.["minLength"] as number) ?? 0;
+	const max = (input.metadata?.["maxLength"] as number) ?? Infinity;
+	const len = output.length;
+	const passed = len >= min && len <= max;
+	const score = passed ? 1 : Math.max(0, 1 - Math.abs(len - (min + max) / 2) / max);
+	return {
+		runId: nextRunId(),
+		metric: "length-check",
+		score,
+		passed,
+		details: `length ${len}, expected ${min}-${max}`,
+	};
+};

package/packages/evaluation-runner/src/types.ts

@@ -0,0 +1,51 @@
+/**
+ * Evaluation Runner — Types (RFC-0020)
+ */
+
+export interface EvaluationInput {
+	prompt: string;
+	expectedOutput?: string;
+	metadata?: Record<string, unknown>;
+}
+
+export interface EvaluationResult {
+	runId: string;
+	metric: string;
+	score: number;
+	passed: boolean;
+	details?: string;
+	latencyMs?: number;
+}
+
+export interface EvaluationSuite {
+	id: string;
+	name: string;
+	cases: EvaluationCase[];
+	thresholds: Record<string, number>;
+}
+
+export interface EvaluationCase {
+	id: string;
+	name: string;
+	input: EvaluationInput;
+	expected?: string;
+	evaluators: string[];
+}
+
+export type EvaluatorFn = (input: EvaluationInput, output: string) => Promise<EvaluationResult>
+
+export interface EvaluationReport {
+	suiteId: string;
+	runAt: string;
+	total: number;
+	passed: number;
+	failed: number;
+	results: EvaluationResult[];
+	summary: Record<string, { count: number; avgScore: number }>;
+}
+
+export interface EvaluationConfig {
+	timeoutMs?: number;
+	retries?: number;
+	parallel?: boolean;
+}

package/packages/event-bus/dist/src/bus.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Event Bus — pub/sub implementation
+ */
+import type { Subscriber, Subscription, EventPayload } from "./types.js";
+export declare class EventBus {
+    private readonly subs;
+    private readonly byTopic;
+    publish<T>(topic: string, data: T, source?: string): string;
+    subscribe<T>(topic: string, subscriber: Subscriber<T>, filter?: (data: T) => boolean, priority?: number): string;
+    unsubscribe(subscriptionId: string): void;
+    unsubscribeAll(topic?: string): void;
+    replay(topic: string, events: EventPayload<unknown>[]): void;
+    getSubscribers(topic?: string): Subscription[];
+}
+//# sourceMappingURL=bus.d.ts.map

package/packages/event-bus/dist/src/bus.js

@@ -0,0 +1,98 @@
+/**
+ * Event Bus — pub/sub implementation
+ */
+import { randomUUID } from "node:crypto";
+export class EventBus {
+    subs = new Map();
+    byTopic = new Map();
+    publish(topic, data, source = "system") {
+        const eventId = randomUUID();
+        const timestamp = new Date().toISOString();
+        const payload = {
+            topic,
+            data,
+            timestamp,
+            eventId,
+            source,
+        };
+        const topicIds = this.byTopic.get(topic) ?? new Set();
+        const wildcardIds = this.byTopic.get("*") ?? new Set();
+        const matches = [];
+        for (const sid of [...topicIds, ...wildcardIds]) {
+            const sub = this.subs.get(sid);
+            if (!sub || !sub.active)
+                continue;
+            if (sub.filter && !sub.filter(data))
+                continue;
+            matches.push({ sub, payload });
+        }
+        matches.sort((a, b) => b.sub.priority - a.sub.priority);
+        for (const { sub } of matches) {
+            try {
+                sub.subscriber(payload);
+            }
+            catch {
+                sub.active = false;
+                this.subs.delete(sub.id);
+                this.byTopic.get(sub.topic)?.delete(sub.id);
+            }
+        }
+        return eventId;
+    }
+    subscribe(topic, subscriber, filter, priority = 0) {
+        const id = randomUUID();
+        const sub = {
+            id,
+            topic,
+            subscriber: subscriber,
+            filter: filter,
+            priority,
+            active: true,
+        };
+        this.subs.set(id, sub);
+        if (!this.byTopic.has(topic))
+            this.byTopic.set(topic, new Set());
+        this.byTopic.get(topic).add(id);
+        return id;
+    }
+    unsubscribe(subscriptionId) {
+        const sub = this.subs.get(subscriptionId);
+        if (!sub)
+            return;
+        sub.active = false;
+        this.subs.delete(subscriptionId);
+        this.byTopic.get(sub.topic)?.delete(subscriptionId);
+    }
+    unsubscribeAll(topic) {
+        if (topic) {
+            const ids = this.byTopic.get(topic);
+            if (ids) {
+                for (const id of ids)
+                    this.unsubscribe(id);
+                this.byTopic.delete(topic);
+            }
+        }
+        else {
+            for (const id of this.subs.keys())
+                this.unsubscribe(id);
+            this.byTopic.clear();
+        }
+    }
+    replay(topic, events) {
+        for (const event of events) {
+            if (event.topic === topic || topic === "*") {
+                this.publish(event.topic, event.data, event.source);
+            }
+        }
+    }
+    getSubscribers(topic) {
+        if (topic) {
+            const ids = this.byTopic.get(topic) ?? new Set();
+            return [...ids]
+                .map((id) => this.subs.get(id))
+                .filter((s) => s !== undefined);
+        }
+        return [...this.subs.values()].filter((s) => s.active);
+    }
+}
+//# sourceMappingURL=bus.js.map

package/packages/event-bus/dist/src/herdr-bus.d.ts

@@ -0,0 +1,121 @@
+/**
+ * HerdrEventBus — File-based cross-process event bus for herdr tabs.
+ *
+ * Uses JSONL files in a shared workspace so separate pi processes
+ * (coding agent, review agent, etc.) can communicate.
+ *
+ * Architecture:
+ *   Workspace (shared dir)
+ *   ├── events.jsonl          # Append-only event log
+ *   ├── subscriptions/       # Per-agent subscription markers
+ *   └── payloads/             # Event payload files
+ *
+ * Flow:
+ *   1. Agent publishes event → append to events.jsonl + write payload
+ *   2. Other agents poll events.jsonl → process matching events
+ *   3. Each agent tracks last-read offset in subscriptions/{agentId}.json
+ */
+import type { EventPayload } from "./types.js";
+export interface HerdrBusConfig {
+    workspace: string;
+    pollIntervalMs?: number;
+    agentId: string;
+}
+export interface HerdrSubscription {
+    id: string;
+    topic: string;
+    lastOffset: number;
+    filter?: (data: unknown) => boolean;
+}
+export declare class HerdrEventBus {
+    private readonly workspace;
+    private readonly agentId;
+    private readonly pollIntervalMs;
+    private readonly subscriptions;
+    private polling;
+    private pollTimer?;
+    constructor(config: HerdrBusConfig);
+    private ensureDirs;
+    subscribe(topic: string, filter?: (data: unknown) => boolean): string;
+    unsubscribe(id: string): void;
+    publish<T>(topic: string, data: T): string;
+    startPolling(handler: (payload: EventPayload<unknown>) => void | Promise<void>): void;
+    stopPolling(): void;
+    private pollEvents;
+    private readNewEvents;
+    private subscriptionsPath;
+    private saveSubscriptions;
+    private loadSubscriptions;
+    getSubscribedTopics(): string[];
+    getWorkspace(): string;
+}
+export declare function getHerdrWorkspace(): string;
+export declare function createHerdrBus(agentId: string): HerdrEventBus;
+export interface HerdrWorkspace {
+    root: string;
+    code: string;
+    reviews: string;
+    payloads: string;
+    subscriptions: string;
+}
+export declare function getHerdrWorkspacePaths(): HerdrWorkspace;
+export declare function ensureHerdrWorkspace(): HerdrWorkspace;
+export type LoopVerdict = "approved" | "changes_requested" | "blocked";
+export interface LoopConfig {
+    loopId: string;
+    writeCount: number;
+    reviewCount: number;
+    prompt: string;
+    nextReviewAfter: number;
+    createdAt: string;
+}
+export interface CodeTickPayload {
+    loopId: string;
+    iteration: number;
+    prompt: string;
+}
+export interface CodeWrittenPayload {
+    loopId: string;
+    iteration: number;
+    files: string[];
+    summary?: string;
+}
+export interface ReviewTickPayload {
+    loopId: string;
+    iteration: number;
+    codeFiles: string[];
+}
+export interface ReviewCompletedPayload {
+    loopId: string;
+    iteration: number;
+    verdict: LoopVerdict;
+    message: string;
+    reportFile: string;
+}
+export interface LoopEarlyExitPayload {
+    loopId: string;
+    reason: LoopVerdict;
+    message: string;
+}
+export interface LoopFinishedPayload {
+    loopId: string;
+    summary: string;
+    iterations: {
+        writes: number;
+        reviews: number;
+        finalVerdict: LoopVerdict;
+    };
+}
+export declare function publishLoopStarted(bus: HerdrEventBus, config: LoopConfig): string;
+export declare function publishCodeTick(bus: HerdrEventBus, loopId: string, iteration: number, prompt: string): string;
+export declare function publishCodeWritten(bus: HerdrEventBus, loopId: string, iteration: number, files: string[], summary?: string): string;
+export declare function publishReviewTick(bus: HerdrEventBus, loopId: string, iteration: number, codeFiles: string[]): string;
+export declare function publishReviewCompleted(bus: HerdrEventBus, loopId: string, iteration: number, verdict: LoopVerdict, message: string, reportFile: string): string;
+export declare function publishLoopEarlyExit(bus: HerdrEventBus, loopId: string, reason: LoopVerdict, message: string): string;
+export declare function publishLoopFinished(bus: HerdrEventBus, loopId: string, summary: string, writes: number, reviews: number, finalVerdict: LoopVerdict): string;
+export declare function parseVerdict(content: string): LoopVerdict | null;
+export declare function parseVerdictMessage(content: string): string;
+export declare function publishCodeWrittenSimple(bus: HerdrEventBus, taskId: string, files: string[], branch?: string): string;
+export declare function publishReviewRequestedSimple(bus: HerdrEventBus, taskId: string, codeTaskId: string): string;
+export declare function publishReviewCompletedSimple(bus: HerdrEventBus, taskId: string, reportFile: string, status: "approved" | "changes_requested" | "failed"): string;
+//# sourceMappingURL=herdr-bus.d.ts.map