pi-harness-runtime 0.3.2-beta.2 → 0.4.0

package/harness/index.ts

@@ -0,0 +1,105 @@
+/**
+ * Harness Runtime — Core Module Index
+ *
+ * Re-exports all harness components for easy importing.
+ */
+
+// Re-export types from the types package
+export type {
+	JobStatus,
+	TaskStatus,
+	RuntimeCheckpoint,
+	RuntimeTask,
+	RuntimeEvent,
+	TaskGraph,
+	TaskNode,
+	RepairTask,
+	FailureType,
+	AttemptedFix,
+	RetryPolicy,
+	QuotaSignal,
+	QuotaState,
+	ProviderConfig,
+	ProviderCapability,
+	ProviderRequest,
+	ProviderResponse,
+	E2EScenario,
+	E2EStep,
+	E2EResult,
+	E2EReport,
+} from "../packages/types/src/runtime-types.js";
+
+// State Machine
+export { JobStateMachine } from "./job-state-machine.js";
+export type {
+	StateTransition,
+	TransitionResult,
+} from "./job-state-machine.js";
+export type { CheckpointManager } from "./job-state-machine.js";
+
+// Task Graph
+export { TaskGraphManager } from "./task-graph.js";
+export type { TaskGraphOptions } from "./task-graph.js";
+
+// Loop Runtime
+export { LoopRuntime } from "./loop-runtime.js";
+export type { LoopResult } from "./loop-runtime.js";
+
+// Repair Engine
+export { RepairEngine } from "./repair-engine.js";
+export type { RepairResult } from "./repair-engine.js";
+
+// Master Planner
+export { MasterPlanner } from "./master-planner.js";
+export type { PlanResult } from "./master-planner.js";
+
+// Context Window Manager
+export { ContextWindowManager } from "./context-window-manager.js";
+
+// Blackboard
+export { createBlackboard, SharedBlackboard } from "./blackboard.js";
+
+// Agent Handoff
+export { AgentHandoffProtocol } from "./agent-handoff.js";
+
+// Auto Compact (RFC-0019)
+export { AutoCompactEngine } from "./auto-compact.js";
+export type {
+	CompactionEvent,
+	CompactionConfig,
+	ContinuePrompt,
+} from "./auto-compact.js";
+
+// Output Limit Handler (RFC-0020)
+export { OutputLimitHandler } from "./output-limit-handler.js";
+export type {
+	OutputLimitConfig,
+	OutputLimitEvent,
+	ExpectedOutput,
+} from "./output-limit-handler.js";
+
+// Partial Recovery (RFC-0021)
+export { PartialRecovery } from "./partial-recovery.js";
+export type {
+	PartialResponse,
+	RecoveryStatus,
+	MergeOptions,
+} from "./partial-recovery.js";
+
+// Notification Events (RFC-0022)
+export {
+	HarnessNotificationEvents,
+	createNotificationConfigFromEnv,
+} from "./notification-events.js";
+
+// E2E Testing
+export { E2ETestEngine } from "./e2e/test-engine.js";
+export { PlaywrightE2ERunner } from "./e2e/playwright-runner.js";
+export { MiniMaxQuotaScraper } from "./e2e/playwright-runner.js";
+export type {
+	E2ERunner,
+	PlaywrightRunnerConfig,
+} from "./e2e/playwright-runner.js";
+
+// Project Detector
+export { ProjectDetector } from "./project-detector/detector.js";

package/harness/notification-events.ts

@@ -0,0 +1,299 @@
+/**
+ * Harness Notification Events — RFC-0022
+ *
+ * Integrates NotificationCenter with the harness runtime.
+ * Sends mobile alerts for all runtime events.
+ *
+ * Events:
+ *   JobStarted, TaskCompleted, TaskFailed, QuotaPaused,
+ *   ResumeScheduled, ContextCompacted, OutputLimitContinued,
+ *   E2EFailed, HumanReviewNeeded, ReadyForClient, JobCancelled, Error
+ */
+
+import { NotificationCenter } from "../packages/notification/notification-center.js";
+import type {
+	NotificationConfig,
+	NotificationEvent,
+	NotificationContext,
+} from "../packages/notification/types.js";
+import type { JobStateMachine } from "./job-state-machine.js";
+import type { TaskGraphManager } from "./task-graph.js";
+
+export interface NotificationEventsConfig {
+	jobId: string;
+	requirement: string;
+	notificationConfig?: NotificationConfig;
+	// Optional: path to config file for persistent settings
+	configPath?: string;
+}
+
+export class HarnessNotificationEvents {
+	private center: NotificationCenter;
+	private config: NotificationEventsConfig;
+	private machine: JobStateMachine | null = null;
+	private graph: TaskGraphManager | null = null;
+
+	constructor(config: NotificationEventsConfig) {
+		this.config = config;
+		this.center = new NotificationCenter(config.notificationConfig);
+	}
+
+	/**
+	 * Initialize the notification center
+	 */
+	async initialize(): Promise<void> {
+		await this.center.initialize();
+	}
+
+	/**
+	 * Attach to job state machine to receive events
+	 */
+	attachToMachine(machine: JobStateMachine): void {
+		this.machine = machine;
+	}
+
+	/**
+	 * Attach to task graph to get task info
+	 */
+	attachToGraph(graph: TaskGraphManager): void {
+		this.graph = graph;
+	}
+
+	/**
+	 * Check if notifications are configured
+	 */
+	hasChannels(): boolean {
+		return this.center.hasChannels();
+	}
+
+	/**
+	 * List configured channels
+	 */
+	listChannels(): string[] {
+		return this.center.listChannels();
+	}
+
+	// ─── Event Emitters ────────────────────────────────────────────────
+
+	/**
+	 * Emit JobStarted event
+	 */
+	async emitJobStarted(): Promise<void> {
+		await this.emit("JobStarted");
+	}
+
+	/**
+	 * Emit TaskCompleted event
+	 */
+	async emitTaskCompleted(taskId: string, taskTitle?: string): Promise<void> {
+		await this.emit("TaskCompleted", { taskId, taskTitle });
+	}
+
+	/**
+	 * Emit TaskFailed event
+	 */
+	async emitTaskFailed(
+		taskId: string,
+		taskTitle?: string,
+		error?: string,
+	): Promise<void> {
+		await this.emit("TaskFailed", { taskId, taskTitle, error });
+	}
+
+	/**
+	 * Emit QuotaPaused event
+	 */
+	async emitQuotaPaused(resumeAt?: string): Promise<void> {
+		await this.emit("QuotaPaused", {
+			error: resumeAt ? `Resumes at ${resumeAt}` : undefined,
+		});
+	}
+
+	/**
+	 * Emit ResumeScheduled event
+	 */
+	async emitResumeScheduled(resumeAt: string): Promise<void> {
+		await this.emit("ResumeScheduled", { error: resumeAt });
+	}
+
+	/**
+	 * Emit ContextCompacted event
+	 */
+	async emitContextCompacted(tokensCompacted?: number): Promise<void> {
+		await this.emit("ContextCompacted", {
+			error: tokensCompacted
+				? `Compacted ${tokensCompacted.toLocaleString()} tokens`
+				: undefined,
+		});
+	}
+
+	/**
+	 * Emit OutputLimitContinued event
+	 */
+	async emitOutputLimitContinued(attempt: number): Promise<void> {
+		await this.emit("OutputLimitContinued", {
+			error: `Attempt ${attempt}`,
+		});
+	}
+
+	/**
+	 * Emit E2EFailed event
+	 */
+	async emitE2EFailed(scenarioId?: string): Promise<void> {
+		await this.emit("E2EFailed", {
+			error: scenarioId ? `Scenario: ${scenarioId}` : undefined,
+		});
+	}
+
+	/**
+	 * Emit HumanReviewNeeded event
+	 */
+	async emitHumanReviewNeeded(taskId: string, reason: string): Promise<void> {
+		await this.emit("HumanReviewNeeded", { taskId, error: reason });
+	}
+
+	/**
+	 * Emit ReadyForClient event
+	 */
+	async emitReadyForClient(): Promise<void> {
+		await this.emit("ReadyForClient");
+	}
+
+	/**
+	 * Emit JobCancelled event
+	 */
+	async emitJobCancelled(reason?: string): Promise<void> {
+		await this.emit("JobCancelled", { error: reason });
+	}
+
+	/**
+	 * Emit Error event
+	 */
+	async emitError(error: string): Promise<void> {
+		await this.emit("Error", { error });
+	}
+
+	// ─── State Machine Event Listeners ─────────────────────────────────
+
+	/**
+	 * Wire up with JobStateMachine to emit events on transitions
+	 */
+	wireWithStateMachine(machine: JobStateMachine): void {
+		this.machine = machine;
+
+		// Listen to state machine events via polling or callback
+		// This would need the state machine to emit events
+	}
+
+	/**
+	 * Check current state and emit appropriate events
+	 */
+	async checkAndEmitStateChange(
+		oldState: string,
+		newState: string,
+	): Promise<void> {
+		// Map state machine states to notification events
+		const stateEventMap: Record<string, NotificationEvent> = {
+			paused_quota: "QuotaPaused",
+			waiting_human: "HumanReviewNeeded",
+			ready_for_client: "ReadyForClient",
+			cancelled: "JobCancelled",
+			planning: "JobStarted",
+		};
+
+		const event = stateEventMap[newState];
+		if (event) {
+			await this.emit(event);
+		}
+	}
+
+	// ─── Private Methods ───────────────────────────────────────────────
+
+	private async emit(
+		event: NotificationEvent,
+		extra?: Partial<NotificationContext>,
+	): Promise<void> {
+		if (!this.center.hasChannels()) {
+			return; // No channels configured, skip
+		}
+
+		const context: NotificationContext = {
+			jobId: this.config.jobId,
+			requirement: this.config.requirement,
+			...extra,
+		};
+
+		try {
+			const results = await this.center.notify(event, context);
+
+			// Log results (but don't fail if notification fails)
+			for (const result of results) {
+				if (!result.success) {
+					console.warn(
+						`[NotificationEvents] Failed to send ${event} to ${result.channel}: ${result.error}`,
+					);
+				}
+			}
+		} catch (error) {
+			// Never crash the runtime due to notification failure
+			console.error(`[NotificationEvents] Notification error: ${error}`);
+		}
+	}
+}
+
+/**
+ * Create notification config from environment variables
+ */
+export function createNotificationConfigFromEnv():
+	| NotificationConfig
+	| undefined {
+	const channels: NotificationConfig["channels"] = [];
+
+	// Telegram
+	if (process.env.TELEGRAM_BOT_TOKEN && process.env.TELEGRAM_CHAT_ID) {
+		channels.push({
+			id: "telegram",
+			type: "telegram",
+			enabled: true,
+			config: {
+				botToken: process.env.TELEGRAM_BOT_TOKEN,
+				chatId: process.env.TELEGRAM_CHAT_ID,
+			},
+		});
+	}
+
+	// Ntfy
+	if (process.env.NTFY_TOPIC) {
+		channels.push({
+			id: "ntfy",
+			type: "ntfy",
+			enabled: true,
+			config: {
+				server: process.env.NTFY_SERVER ?? "https://ntfy.sh",
+				topic: process.env.NTFY_TOPIC,
+				authToken: process.env.NTFY_TOKEN,
+			},
+		});
+	}
+
+	// Webhook
+	if (process.env.NOTIFICATION_WEBHOOK_URL) {
+		channels.push({
+			id: "webhook",
+			type: "webhook",
+			enabled: true,
+			config: {
+				url: process.env.NOTIFICATION_WEBHOOK_URL,
+				method:
+					(process.env.NOTIFICATION_WEBHOOK_METHOD as "POST" | "PUT") ?? "POST",
+				authToken: process.env.NOTIFICATION_WEBHOOK_TOKEN,
+			},
+		});
+	}
+
+	if (channels.length === 0) {
+		return undefined;
+	}
+
+	return { channels, enabled: true };
+}

package/harness/output-limit-handler.ts

@@ -0,0 +1,316 @@
+/**
+ * Output Token Limit Handler — RFC-0020
+ *
+ * Handles model responses that stop because the maximum output token limit was reached.
+ * Coordinates with AutoCompactEngine (RFC-0019) and PartialRecovery (RFC-0021).
+ *
+ * Classification Matrix:
+ * | Failure Type       | Runtime Action           |
+ * |--------------------|--------------------------|
+ * | Quota exhausted    | pause until reset        |
+ * | Context full       | compact and resume       |
+ * | Output token limit | continue same task       |
+ * | Unknown error      | retry or escalate        |
+ */
+
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import type { CompactionEvent } from "./auto-compact.js";
+
+export interface OutputLimitConfig {
+	jobId: string;
+	taskId: string;
+	rootDir?: string;
+	maxContinueAttempts?: number;
+	continuationBackoffMs?: number;
+	requireExpectedOutputValidation?: boolean;
+}
+
+export interface OutputLimitEvent {
+	timestamp: string;
+	jobId: string;
+	taskId: string;
+	reason: string;
+	partialContent: string;
+	finishReason: "length" | "stop" | "content_filter" | "error";
+	attempts: number;
+	continueSucceeded?: boolean;
+}
+
+export interface ExpectedOutput {
+	description: string;
+	validate?: (content: string) => boolean;
+}
+
+const OUTPUT_LIMIT_PATTERNS = [
+	/reached the maximum output token limit/i,
+	/output token limit/i,
+	/response may be incomplete/i,
+	/stop_reason.*length/i,
+	/finish_reason.*length/i,
+	/max_tokens.*exceeded/i,
+	/completion.*truncated/i,
+	/model.*stopped.*token/i,
+];
+
+export class OutputLimitHandler {
+	private readonly rootDir: string;
+	private readonly jobId: string;
+	private readonly taskId: string;
+	private readonly maxAttempts: number;
+	private readonly backoffMs: number;
+	private readonly requireValidation: boolean;
+	private attempts = 0;
+	private partials: string[] = [];
+
+	constructor(config: OutputLimitConfig) {
+		this.rootDir =
+			config.rootDir ??
+			join(homedir(), ".pi", "harness", config.jobId, "partial", config.taskId);
+		this.jobId = config.jobId;
+		this.taskId = config.taskId;
+		this.maxAttempts = config.maxContinueAttempts ?? 5;
+		this.backoffMs = config.continuationBackoffMs ?? 1000;
+		this.requireValidation = config.requireExpectedOutputValidation ?? true;
+		this.ensureDir();
+	}
+
+	/**
+	 * Detect if an error or response indicates output limit was reached
+	 */
+	detectOutputLimit(
+		error: unknown,
+		response?: { finishReason?: string },
+	): boolean {
+		// Check error message
+		if (error) {
+			const errorStr = String(error).toLowerCase();
+			for (const pattern of OUTPUT_LIMIT_PATTERNS) {
+				if (pattern.test(errorStr)) {
+					return true;
+				}
+			}
+		}
+
+		// Check finish reason
+		if (response?.finishReason === "length") {
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Classify the type of failure for proper handling
+	 */
+	classifyFailure(
+		error: unknown,
+		response?: { finishReason?: string; content?: string },
+	): "quota_exhausted" | "context_full" | "output_limit" | "unknown" {
+		const errorStr = String(error ?? "").toLowerCase();
+
+		// Check for quota exhaustion
+		if (
+			/error.*quota/i.test(errorStr) ||
+			/error.*2056/i.test(errorStr) ||
+			/error.*insufficient_quota/i.test(errorStr)
+		) {
+			return "quota_exhausted";
+		}
+
+		// Check for context/sequence length
+		if (
+			/error.*context.*length/i.test(errorStr) ||
+			/error.*too many tokens/i.test(errorStr) ||
+			/error.*maximum context/i.test(errorStr)
+		) {
+			return "context_full";
+		}
+
+		// Check for output token limit
+		if (
+			this.detectOutputLimit(error, response) ||
+			response?.finishReason === "length"
+		) {
+			return "output_limit";
+		}
+
+		return "unknown";
+	}
+
+	/**
+	 * Handle output limit - save partial and prepare continuation
+	 */
+	async handleOutputLimit(
+		partialContent: string,
+		finishReason: string = "length",
+	): Promise<OutputLimitEvent> {
+		this.attempts++;
+		const timestamp = new Date().toISOString();
+
+		// Save partial response
+		const partialPath = this.savePartial(partialContent, this.attempts);
+		this.partials.push(partialPath);
+
+		// Create event
+		const event: OutputLimitEvent = {
+			timestamp,
+			jobId: this.jobId,
+			taskId: this.taskId,
+			reason: "output_token_limit",
+			partialContent,
+			finishReason: finishReason as OutputLimitEvent["finishReason"],
+			attempts: this.attempts,
+		};
+
+		// Save event
+		this.saveEvent(event);
+
+		// Wait with backoff before continuing
+		await this.backoff();
+
+		return event;
+	}
+
+	/**
+	 * Check if we should continue (respects max attempts)
+	 */
+	shouldContinue(): boolean {
+		return this.attempts < this.maxAttempts;
+	}
+
+	/**
+	 * Get current attempt count
+	 */
+	getAttempts(): number {
+		return this.attempts;
+	}
+
+	/**
+	 * Get all partial responses
+	 */
+	getPartials(): string[] {
+		return [...this.partials];
+	}
+
+	/**
+	 * Merge partial responses
+	 */
+	mergePartials(): string {
+		const merged: string[] = [];
+
+		for (const partialPath of this.partials) {
+			if (existsSync(partialPath)) {
+				const content = readFileSync(partialPath, "utf-8");
+				merged.push(content);
+			}
+		}
+
+		// v0.1: simple concatenation as markdown sections
+		return merged
+			.map((p, i) => `## Partial ${i + 1}\n\n${p}`)
+			.join("\n\n---\n\n");
+	}
+
+	/**
+	 * Build continue message for the next turn
+	 */
+	buildContinueMessage(additionalContext?: string): string {
+		const merged = this.mergePartials();
+
+		const lines = [
+			"# Continue From Partial Response",
+			"",
+			"The previous response was truncated due to output token limit.",
+			"",
+			"## Merged Partial Content",
+			"",
+			merged,
+		];
+
+		if (additionalContext) {
+			lines.push("", "## Additional Context");
+			lines.push("", additionalContext);
+		}
+
+		lines.push("", "## Instructions");
+		lines.push("");
+		lines.push("1. Review the partial content above");
+		lines.push("2. Continue from where the response was truncated");
+		lines.push("3. Do not repeat content that already appears above");
+		lines.push(
+			`4. This is attempt ${this.attempts + 1} of ${this.maxAttempts}`,
+		);
+
+		return lines.join("\n");
+	}
+
+	/**
+	 * Validate expected output if configured
+	 */
+	validateOutput(content: string, expected: ExpectedOutput): boolean {
+		if (!this.requireValidation) {
+			return true;
+		}
+
+		if (expected.validate) {
+			return expected.validate(content);
+		}
+
+		// Basic validation: content should be longer than partial
+		return content.length > this.getPartials()[0]?.length ?? 0;
+	}
+
+	/**
+	 * Reset for a new task
+	 */
+	reset(): void {
+		this.attempts = 0;
+		this.partials = [];
+	}
+
+	/**
+	 * Check if escalation is needed (max attempts reached)
+	 */
+	shouldEscalate(): boolean {
+		return this.attempts >= this.maxAttempts;
+	}
+
+	// ─── Private Methods ────────────────────────────────────────────────
+
+	private ensureDir(): void {
+		if (!existsSync(this.rootDir)) {
+			mkdirSync(this.rootDir, { recursive: true });
+		}
+	}
+
+	private savePartial(content: string, attempt: number): string {
+		const filename = `partial_${String(attempt).padStart(3, "0")}.md`;
+		const path = join(this.rootDir, filename);
+		writeFileSync(path, content, "utf-8");
+		return path;
+	}
+
+	private saveEvent(event: OutputLimitEvent): void {
+		const eventsPath = join(this.rootDir, "events.jsonl");
+		writeFileSync(eventsPath, JSON.stringify(event) + "\n", "utf-8");
+
+		// Also save recovery status
+		const statusPath = join(this.rootDir, "recovery_status.json");
+		const status = {
+			taskId: this.taskId,
+			status: this.shouldContinue() ? "continuing" : "escalated",
+			partials: this.partials.map((p) => p.split("/").pop()),
+			mergedOutput: "merged.md",
+			attempts: this.attempts,
+			lastError: event.reason,
+		};
+		writeFileSync(statusPath, JSON.stringify(status, null, 2) + "\n", "utf-8");
+	}
+
+	private async backoff(): Promise<void> {
+		const delay = this.backoffMs * 2 ** (this.attempts - 1);
+		await new Promise((resolve) => setTimeout(resolve, delay));
+	}
+}