pi-harness-runtime 1.1.394 → 1.1.396

package/packages/jev-judge/src/continuation-controller.ts

@@ -1,5 +1,14 @@
 import type { TodoItem, TodoContext, TodoProvider } from "./todo-provider.js";
 import { createTodoProvider } from "./todo-provider.js";
+import {
+	buildValidationSteerSuffix,
+	formatDuration,
+	parseValidationPlan,
+	validationBudgetStatus,
+	type ValidationBudgetStatus,
+	type ValidationState,
+} from "./validation-plan.js";
+import type { ValidationLedger } from "./validation-ledger.js";
 
 /**
  * Simple debug logger that writes to file, not stdout
@@ -88,8 +97,24 @@ export interface ContinuationConfig {
 	 * A dispatch that never settles must not block every later continuation.
 	 */
 	inFlightTimeoutMs?: number;
+	/**
+	 * Ledger of validation runs.
+	 *
+	 * When a ledger is wired, the controller knows which tier already ran and
+	 * whether the final tier is still pending, and it can pause an item whose
+	 * declared time budget is spent.
+	 */
+	validationLedger?: ValidationLedger;
 }
 
+/**
+ * What a dispatch wants to send.
+ *
+ * A dispatch may also abort: the message builder returns the reason and the
+ * dispatch is dropped before anything is put on the wire.
+ */
+type DispatchPayload = { send: string } | { abort: string };
+
 /**
  * Continuation state
  */
@@ -148,6 +173,8 @@ export interface ContinuationResult {
 	nextItemId?: number;
 	question?: string;
 	reason?: string;
+	/** Budget status of the item the decision was about. */
+	budget?: ValidationBudgetStatus;
 }
 
 /**
@@ -198,9 +225,11 @@ export class TodoContinuationController {
 	private _circuitBreakerCooldownMs = 5 * 60 * 1000;  // How long the breaker stays open
 	private _inFlightTimeoutMs = INFLIGHT_TIMEOUT_MS;  // Watchdog for a hanging dispatch
 	private _attemptsByItem = new Map<number, number>();  // Successful dispatches per item
+	private _itemStartedAt = new Map<number, number>();  // First continuation of an item (elapsed-time budget)
 	private _totalAttempts = 0;  // Successful dispatches in this session (runaway guard)
 	private _maxTotalSteerAttempts = 9;  // Session-wide cap on successful dispatches
 	private _disposed = false;  // Set on shutdown: no dispatch and no timer may act anymore
+	private _validationLedger?: ValidationLedger;  // Optional ledger of validation runs
 
 	/** True while a dispatch is on the wire */
 	private get inFlight(): boolean {
@@ -215,6 +244,7 @@ export class TodoContinuationController {
 		this._circuitBreakerCooldownMs = config?.circuitBreakerCooldownMs ?? 5 * 60 * 1000;
 		this._maxConsecutiveFailures = config?.maxConsecutiveFailures ?? 3;
 		this._inFlightTimeoutMs = config?.inFlightTimeoutMs ?? INFLIGHT_TIMEOUT_MS;
+		this._validationLedger = config?.validationLedger;
 		this.config = {
 			jevFallbackMinWaitMs: 5 * 60 * 1000, // 5 minutes
 			routineThreshold: 0.7,
@@ -237,6 +267,56 @@ export class TodoContinuationController {
 		return this._attemptsByItem.get(itemId) ?? 0;
 	}
 
+	/**
+	 * Elapsed work time on an item, measured from the first continuation.
+	 *
+	 * `undefined` means the item was never continued, so no elapsed time can
+	 * be reported yet.
+	 */
+	private elapsedForItem(itemId: number): number | undefined {
+		const startedAt = this._itemStartedAt.get(itemId);
+		if (startedAt === undefined) return undefined;
+		return Date.now() - startedAt;
+	}
+
+	/**
+	 * Remember when work on an item started (first continuation only).
+	 */
+	private markItemStarted(itemId: number): void {
+		if (!this._itemStartedAt.has(itemId)) this._itemStartedAt.set(itemId, Date.now());
+	}
+
+	/**
+	 * The validation state of an item, read from the ledger.
+	 *
+	 * `undefined` when no ledger is wired: the caller then falls back to the
+	 * declared plan alone.
+	 */
+	private validationStateFor(item: TodoItem | undefined): ValidationState | undefined {
+		if (!item || !this._validationLedger) return undefined;
+		return this._validationLedger.stateFor(item);
+	}
+
+	/**
+	 * The budget status of an item.
+	 *
+	 * Only a declared `budgetMinutes` arms a budget - the harness never
+	 * invents one, because an invented budget could pause legitimate work.
+	 */
+	private budgetStatusFor(item: TodoItem | undefined): ValidationBudgetStatus {
+		const plan = parseValidationPlan(item);
+		return validationBudgetStatus(plan, item ? this.elapsedForItem(item.id) : undefined);
+	}
+
+	/**
+	 * The reason reported when an item ran out of its declared time budget.
+	 */
+	private budgetAbortReason(item: TodoItem, budget: ValidationBudgetStatus): string {
+		const spent = formatDuration(budget.elapsedMs ?? 0);
+		const total = formatDuration(budget.budgetMs ?? 0);
+		return `budget exhausted for task ${item.id} (${spent} of ${total}) - pausing at a safe boundary`;
+	}
+
 	/**
 	 * True while the circuit breaker forbids a send.
 	 *
@@ -469,7 +549,18 @@ export class TodoContinuationController {
 			if (this._totalAttempts >= this._maxTotalSteerAttempts) {
 				return { action: "stop", reason: `max total attempts (${this._maxTotalSteerAttempts}) reached` };
 			}
-			
+
+			// A declared time budget that is spent pauses the item at a safe
+			// boundary: the work stays open, the report says what is pending.
+			const budget = this.budgetStatusFor(item);
+			if (budget.exhausted) {
+				return {
+					action: "stop",
+					reason: this.budgetAbortReason(item, budget),
+					budget,
+				};
+			}
+
 			this.state.currentItemId = item.id;
 			this.state.nextItemId = item.id;
 			return { action: "continue", nextItemId: item.id };
@@ -492,7 +583,18 @@ export class TodoContinuationController {
 			if (this._totalAttempts >= this._maxTotalSteerAttempts) {
 				return { action: "stop", reason: `max total attempts (${this._maxTotalSteerAttempts}) reached` };
 			}
-			
+
+			// A declared time budget that is spent pauses the item at a safe
+			// boundary: the work stays open, the report says what is pending.
+			const budget = this.budgetStatusFor(item);
+			if (budget.exhausted) {
+				return {
+					action: "stop",
+					reason: this.budgetAbortReason(item, budget),
+					budget,
+				};
+			}
+
 			this.state.currentItemId = item.id;
 			this.state.nextItemId = item.id;
 			return { action: "continue", nextItemId: item.id };
@@ -530,20 +632,41 @@ export class TodoContinuationController {
 			return;
 		}
 
-		// Build continuation message
-		const message = item.description
-			? `Continue task: ${item.subject}\n\n${item.description}`
-			: `Continue task: ${item.subject}`;
-		// A retry after a stalled dispatch gets the same duplicate warning as
-		// the automatic continuation path.
-		const finalMessage =
-			this._stalledItemId === item.id
-				? `${message} (retry after a stalled dispatch - a duplicate continuation for this item may still arrive)`
-				: message;
+		// The message is built when the dispatch is about to be sent: the
+		// validation context must reflect the state at that moment.
+		this.markItemStarted(item.id);
 
 		debugLog("TodoContinuation", "Sending continuation:", item.subject);
 
-		const result = await this.guardedDispatch(item.id, finalMessage);
+		const result = await this.guardedDispatch(item.id, (dispatched) => {
+			const target = dispatched ?? item;
+			const baseMessage = target.description
+				? `Continue task: ${target.subject}\n\n${target.description}`
+				: `Continue task: ${target.subject}`;
+			// A retry after a stalled dispatch gets the same duplicate warning
+			// as the automatic continuation path.
+			const retryNote =
+				this._stalledItemId === item.id
+					? " (retry after a stalled dispatch - a duplicate continuation for this item may still arrive)"
+					: "";
+			// A spent time budget pauses the item: nothing goes on the wire.
+			const budget = this.budgetStatusFor(target);
+			if (budget.exhausted) {
+				return { abort: this.budgetAbortReason(target, budget) };
+			}
+			// A declared staged-validation plan rides along with the
+			// continuation: the agent must know which tier is cheap before it
+			// picks a command.
+			const validation = buildValidationSteerSuffix(target, {
+				elapsedMs: this.elapsedForItem(target.id),
+				state: this.validationStateFor(target),
+			});
+			return {
+				send: validation
+					? `${baseMessage}${retryNote}\n\n${validation}`
+					: `${baseMessage}${retryNote}`,
+			};
+		});
 		if (result.dispatched) {
 			// Only a dispatch that actually reached the agent consumes an attempt
 			this.recordAttempt(item.id);
@@ -566,8 +689,15 @@ export class TodoContinuationController {
 	/**
 	 * Guarded dispatch - checks all conditions before sending
 	 * Uses in-flight guard to prevent race conditions
+	 *
+	 * The message is built by `buildMessage` at the moment the dispatch is
+	 * about to send, so the payload always reflects the state at that moment
+	 * (and the item comes from the snapshot read this dispatch already does).
 	 */
-	private async guardedDispatch(itemId: number, message: string): Promise<GuardedDispatchResult> {
+	private async guardedDispatch(
+		itemId: number,
+		buildMessage: (item: TodoItem | undefined) => DispatchPayload,
+	): Promise<GuardedDispatchResult> {
 		// 0. A disposed controller never dispatches.
 		if (this._disposed) {
 			debugLog("TodoContinuation", "Guarded dispatch: controller disposed, skipping");
@@ -716,6 +846,18 @@ export class TodoContinuationController {
 			this.scheduleGuardRelease();
 
 			try {
+				const payload = buildMessage(item);
+				if ("abort" in payload) {
+					// The message builder refused the dispatch (e.g. the item
+					// ran out of its declared time budget). Nothing goes on the
+					// wire, and the guard is released again.
+					debugLog("TodoContinuation", "Guarded dispatch: aborted before send:", payload.abort);
+					this.state.continuationQueued = false;
+					this.cancelGuardTimer();
+					this._blockedDispatches++;
+					return { dispatched: false, reason: payload.abort, blocked: true };
+				}
+				const message = payload.send;
 				await this.config.dispatch(message);
 				this.state.lastSteerMessage = message;
 				// Only the dispatch that owns the guard may reset the
@@ -824,14 +966,31 @@ export class TodoContinuationController {
 			};
 		}
 
-		const message =
-			this._stalledItemId === itemId
-				? `Continue todo item ID ${itemId}. (retry after a stalled dispatch - a duplicate continuation for this item may still arrive)`
-				: `Continue todo item ID ${itemId}.`;
+		this.markItemStarted(itemId);
 		debugLog("TodoContinuation", "Attempting steer to continue item", itemId);
 
-		// Use guarded dispatch to verify all conditions
-		const result = await this.guardedDispatch(itemId, message);
+		// The message is built when the dispatch is about to be sent, from the
+		// item the dispatch itself read: one snapshot read, no extra latency,
+		// and the validation context is always current.
+		const result = await this.guardedDispatch(itemId, (item) => {
+			const baseMessage =
+				this._stalledItemId === itemId
+					? `Continue todo item ID ${itemId}. (retry after a stalled dispatch - a duplicate continuation for this item may still arrive)`
+					: `Continue todo item ID ${itemId}.`;
+			// A spent time budget pauses the item: nothing goes on the wire.
+			const budget = this.budgetStatusFor(item);
+			if (budget.exhausted && item) {
+				return { abort: this.budgetAbortReason(item, budget) };
+			}
+			// A declared staged-validation plan rides along with the
+			// continuation: the agent must know which tier is cheap before it
+			// picks a command.
+			const validation = buildValidationSteerSuffix(item, {
+				elapsedMs: this.elapsedForItem(itemId),
+				state: this.validationStateFor(item),
+			});
+			return { send: validation ? `${baseMessage}\n\n${validation}` : baseMessage };
+		});
 		if (result.dispatched) {
 			// Only a dispatch that actually reached the agent consumes an attempt
 			this.recordAttempt(itemId);

package/packages/jev-judge/src/index.ts

@@ -728,6 +728,47 @@ export type { ContinuationConfig, ContinuationState, ContinuationResult } from "
 export { createTodoProvider } from "./todo-provider.js";
 export type { TodoProvider, TodoAvailability, TodoItem, TodoStatus, TodoSnapshot, TodoContext } from "./todo-provider.js";
 
+// Export staged validation plans
+export {
+	parseValidationPlan,
+	formatValidationContext,
+	buildValidationSteerSuffix,
+	describeTier,
+	parseTier,
+	parseDurationMs,
+	formatDuration,
+	summarizeValidationState,
+	validationBudgetStatus,
+} from "./validation-plan.js";
+export type {
+	ValidationPlan,
+	ValidationTier,
+	ValidationTierKind,
+	ValidationState,
+	ValidationTierState,
+	ValidationBudgetStatus,
+} from "./validation-plan.js";
+
+// Export the validation ledger (durable record of the tiers that ran)
+export {
+	ValidationLedger,
+	defaultLedgerPath,
+	parseRunRecord,
+	parseLedgerContent,
+} from "./validation-ledger.js";
+export type {
+	ValidationRunRecord,
+	ValidationRunStatus,
+	ValidationLedgerOptions,
+} from "./validation-ledger.js";
+
+// Export the validation tie-breaker (typed verdict about one item)
+export {
+	tieBreakValidation,
+	formatTieBreaker,
+} from "./validation-tie-breaker.js";
+export type { ValidationVerdict, ValidationTieBreaker } from "./validation-tie-breaker.js";
+
 // Export Task Intake
 export { TaskIntakeController, createTaskIntake } from "./task-intake.js";
 export type { TaskIntakeConfig, TaskIntakeResult } from "./task-intake.js";

package/packages/jev-judge/src/validation-ledger.ts

@@ -0,0 +1,290 @@
+/**
+ * Validation Ledger - a durable record of which validation tier ran, when,
+ * how long it took and whether it passed.
+ *
+ * The harness never runs the tests itself, so the ledger is written by the
+ * agent (or by any tool that knows the result) through
+ * `scripts/validation-run.ts`. The ledger is the only mechanical source for
+ * the question "did the final tier already pass?" - without it the harness
+ * would have to trust the agent's own report, and a todo could be completed
+ * while its final acceptance check is still pending.
+ *
+ * Storage is plain NDJSON (one JSON object per line) so the file stays
+ * inspectable with `tail`, `jq` or `less`:
+ *
+ * ```text
+ * ~/.pi-harness-runtime/logs/validation-runs.jsonl
+ * ```
+ *
+ * The file is rotated when it grows past `maxBytes` (the previous content
+ * becomes `<path>.1`).
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import type { TodoItem } from "./todo-provider.js";
+import {
+	parseValidationPlan,
+	summarizeValidationState,
+	type ValidationState,
+	type ValidationTierKind,
+	type ValidationTierState,
+} from "./validation-plan.js";
+
+/**
+ * Result of a single validation run.
+ */
+export type ValidationRunStatus = "passed" | "failed" | "timeout" | "error" | "skipped";
+
+/**
+ * One recorded validation run.
+ */
+export interface ValidationRunRecord {
+	/** ISO timestamp of the end of the run. */
+	ts: string;
+	/** Which tier was executed. */
+	tier: ValidationTierKind;
+	/** Outcome of the run. */
+	status: ValidationRunStatus;
+	/** Todo item the run belongs to. */
+	itemId?: number;
+	/** Subject of the todo item (denormalized for readable logs). */
+	itemSubject?: string;
+	/** The command that was executed. */
+	command?: string;
+	/** Human label of the tier. */
+	label?: string;
+	/** Wall-clock duration of the run in milliseconds. */
+	durationMs?: number;
+	/** Number of records the tier exercised, when it is data-volume bound. */
+	datasetSize?: number;
+	/** Exit code of the command, when it produced one. */
+	exitCode?: number;
+	/** Free-form note (e.g. "5 of 320 tests failed"). */
+	note?: string;
+}
+
+/**
+ * Aggregated validation state of a todo item.
+ */
+export type { ValidationState, ValidationTierState };
+
+/**
+ * Options for the ledger.
+ */
+export interface ValidationLedgerOptions {
+	/** Path of the NDJSON ledger file. */
+	path?: string;
+	/** Rotation threshold in bytes (default 5 MB). */
+	maxBytes?: number;
+	/**
+	 * Mirror sink: called for every recorded run.
+	 *
+	 * The harness uses it to ship the ledger through the central logging
+	 * pipeline, so the runs are archived next to the other application logs.
+	 */
+	onRecord?: (record: ValidationRunRecord) => void;
+}
+
+const DEFAULT_MAX_BYTES = 5 * 1024 * 1024;
+
+/**
+ * Default location of the ledger.
+ */
+export function defaultLedgerPath(): string {
+	return join(homedir(), ".pi-harness-runtime", "logs", "validation-runs.jsonl");
+}
+
+function isRunStatus(value: unknown): value is ValidationRunStatus {
+	return (
+		value === "passed" ||
+		value === "failed" ||
+		value === "timeout" ||
+		value === "error" ||
+		value === "skipped"
+	);
+}
+
+/**
+ * Copy the optional fields of a raw ledger object onto a typed record.
+ */
+function copyOptionalFields(record: Record<string, unknown>, out: ValidationRunRecord): void {
+	if (typeof record.itemId === "number" && Number.isFinite(record.itemId)) out.itemId = record.itemId;
+	if (typeof record.itemSubject === "string") out.itemSubject = record.itemSubject;
+	if (typeof record.command === "string") out.command = record.command;
+	if (typeof record.label === "string") out.label = record.label;
+	if (typeof record.durationMs === "number" && Number.isFinite(record.durationMs)) {
+		out.durationMs = record.durationMs;
+	}
+	if (typeof record.datasetSize === "number" && Number.isFinite(record.datasetSize)) {
+		out.datasetSize = record.datasetSize;
+	}
+	if (typeof record.exitCode === "number" && Number.isFinite(record.exitCode)) {
+		out.exitCode = record.exitCode;
+	}
+	if (typeof record.note === "string") out.note = record.note;
+}
+
+/**
+ * Parse one ledger line. Returns `null` for malformed lines.
+ */
+export function parseRunRecord(line: string): ValidationRunRecord | null {
+	const text = line.trim();
+	if (!text) return null;
+	let raw: unknown;
+	try {
+		raw = JSON.parse(text);
+	} catch {
+		return null;
+	}
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+	const record = raw as Record<string, unknown>;
+	const tier = record.tier === "fast" || record.tier === "full" ? record.tier : undefined;
+	if (!tier) return null;
+	if (!isRunStatus(record.status)) return null;
+
+	const out: ValidationRunRecord = {
+		ts: typeof record.ts === "string" ? record.ts : new Date(0).toISOString(),
+		tier,
+		status: record.status,
+	};
+	copyOptionalFields(record, out);
+	return out;
+}
+
+/**
+ * Parse a whole ledger file. Malformed lines are skipped, not fatal.
+ */
+export function parseLedgerContent(content: string): ValidationRunRecord[] {
+	const records: ValidationRunRecord[] = [];
+	for (const line of content.split("\n")) {
+		const record = parseRunRecord(line);
+		if (record) records.push(record);
+	}
+	return records;
+}
+
+/**
+ * Durable ledger of validation runs.
+ */
+export class ValidationLedger {
+	private readonly path: string;
+	private readonly maxBytes: number;
+	private readonly onRecord?: (record: ValidationRunRecord) => void;
+
+	constructor(options: ValidationLedgerOptions = {}) {
+		this.path = options.path ?? process.env.PI_VALIDATION_LEDGER ?? defaultLedgerPath();
+		this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+		this.onRecord = options.onRecord;
+	}
+
+	/** Path of the ledger file. */
+	get filePath(): string {
+		return this.path;
+	}
+
+	/**
+	 * Record a validation run.
+	 *
+	 * Never throws: a broken ledger must not break the work it describes.
+	 */
+	record(
+		input: Omit<ValidationRunRecord, "ts"> & { ts?: string },
+	): ValidationRunRecord | null {
+		const record: ValidationRunRecord = {
+			ts: input.ts ?? new Date().toISOString(),
+			tier: input.tier,
+			status: input.status,
+		};
+		if (input.itemId !== undefined) record.itemId = input.itemId;
+		if (input.itemSubject !== undefined) record.itemSubject = input.itemSubject;
+		if (input.command !== undefined) record.command = input.command;
+		if (input.label !== undefined) record.label = input.label;
+		if (input.durationMs !== undefined) record.durationMs = input.durationMs;
+		if (input.datasetSize !== undefined) record.datasetSize = input.datasetSize;
+		if (input.exitCode !== undefined) record.exitCode = input.exitCode;
+		if (input.note !== undefined) record.note = input.note;
+
+		try {
+			this.append(record);
+		} catch {
+			// A ledger write failure is not a reason to fail the run it describes.
+		}
+		try {
+			this.onRecord?.(record);
+		} catch {
+			// The mirror sink must never break the producer either.
+		}
+		return record;
+	}
+
+	/**
+	 * Read every recorded run.
+	 */
+	read(): ValidationRunRecord[] {
+		try {
+			if (!existsSync(this.path)) return [];
+			return parseLedgerContent(readFileSync(this.path, "utf-8"));
+		} catch {
+			return [];
+		}
+	}
+
+	/**
+	 * Every run recorded for one todo item.
+	 */
+	runsFor(itemId: number): ValidationRunRecord[] {
+		return this.read().filter((run) => run.itemId === itemId);
+	}
+
+	/**
+	 * The latest run of one tier for one todo item.
+	 */
+	latestFor(itemId: number, tier: ValidationTierKind): ValidationRunRecord | undefined {
+		const runs = this.runsFor(itemId).filter((run) => run.tier === tier);
+		if (runs.length === 0) return undefined;
+		// The ledger is append-only: with an equal timestamp the later run wins.
+		return runs.reduce((a, b) => (a.ts > b.ts ? a : b));
+	}
+
+	/**
+	 * The aggregated validation state of a todo item.
+	 */
+	stateFor(item: TodoItem | null | undefined): ValidationState {
+		const plan = parseValidationPlan(item);
+		const runs = item?.id === undefined ? [] : this.runsFor(item.id);
+		return summarizeValidationState(plan, runs);
+	}
+
+	/**
+	 * True when the item must not be completed yet because its final tier is
+	 * still pending.
+	 */
+	isFinalTierPending(item: TodoItem | null | undefined): boolean {
+		return this.stateFor(item).finalTierPending;
+	}
+
+	private append(record: ValidationRunRecord): void {
+		const dir = dirname(this.path);
+		if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+		this.rotateIfNeeded();
+		appendFileSync(this.path, `${JSON.stringify(record)}\n`, "utf-8");
+	}
+
+	private rotateIfNeeded(): void {
+		if (!existsSync(this.path)) return;
+		let size = 0;
+		try {
+			size = statSync(this.path).size;
+		} catch {
+			return;
+		}
+		if (size < this.maxBytes) return;
+		try {
+			renameSync(this.path, `${this.path}.1`);
+		} catch {
+			// Rotation is best effort: an unwritable directory must not lose runs.
+		}
+	}
+}