pi-harness-runtime 0.10.13 → 0.10.14

package/harness/e2e/test-engine.js

@@ -0,0 +1,290 @@
+/**
+ * E2E Test Engine — RFC-0013
+ *
+ * Browser-based end-to-end testing before work is marked ready for client.
+ *
+ * Artifact Layout:
+ *   harness/e2e/
+ *     scenarios/
+ *     test-data/
+ *     reports/
+ *     artifacts/screenshots/
+ *     artifacts/traces/
+ *     artifacts/videos/
+ */
+import { writeJson, appendJsonl } from "../../cli.js";
+import { join } from "node:path";
+export class E2ETestEngine {
+    rootDir;
+    config;
+    runner = null;
+    constructor(rootDir, config) {
+        this.rootDir = rootDir;
+        this.config = {
+            headless: true,
+            screenshotOnFailure: true,
+            videoOnFailure: false,
+            traceOnFailure: false,
+            timeout: 30000,
+            ...config,
+        };
+    }
+    /**
+     * Set the Playwright runner
+     */
+    setRunner(runner) {
+        this.runner = runner;
+    }
+    /**
+     * Run a single E2E scenario
+     */
+    async runScenario(scenario, context) {
+        if (!this.runner) {
+            return this.createErrorResult(scenario.id, "No runner configured");
+        }
+        const startTime = Date.now();
+        let stepsExecuted = 0;
+        let stepsPassed = 0;
+        let stepsFailed = 0;
+        let screenshotPath;
+        let failedStep;
+        try {
+            for (let i = 0; i < scenario.steps.length; i++) {
+                const step = scenario.steps[i];
+                stepsExecuted++;
+                try {
+                    const success = await this.executeStep(step, context);
+                    if (success) {
+                        stepsPassed++;
+                    }
+                    else {
+                        stepsFailed++;
+                        failedStep = i;
+                        if (this.config.screenshotOnFailure) {
+                            screenshotPath = await this.captureScreenshot(scenario.id, i);
+                        }
+                        break; // Stop on first failure
+                    }
+                }
+                catch (error) {
+                    stepsFailed++;
+                    failedStep = i;
+                    if (this.config.screenshotOnFailure) {
+                        screenshotPath = await this.captureScreenshot(scenario.id, i);
+                    }
+                    break;
+                }
+            }
+            const duration = Date.now() - startTime;
+            return {
+                scenarioId: scenario.id,
+                status: stepsFailed > 0 ? "failed" : "passed",
+                duration,
+                stepsExecuted,
+                stepsPassed,
+                stepsFailed,
+                screenshotPath,
+                executedAt: new Date().toISOString(),
+                failedStep,
+            };
+        }
+        catch (error) {
+            return this.createErrorResult(scenario.id, String(error));
+        }
+    }
+    /**
+     * Run all scenarios for a job
+     */
+    async runAllScenarios(jobId, scenarios, context) {
+        const results = [];
+        let totalDuration = 0;
+        for (const scenario of scenarios) {
+            if (!scenario.required) {
+                // Skip non-required scenarios on failure of required ones
+                const result = await this.runScenario(scenario, context);
+                results.push(result);
+                totalDuration += result.duration;
+            }
+            else {
+                const result = await this.runScenario(scenario, context);
+                results.push(result);
+                totalDuration += result.duration;
+                // Stop on first required failure
+                if (result.status === "failed") {
+                    // Run remaining non-required scenarios
+                    for (const nextScenario of scenarios.slice(scenarios.indexOf(scenario) + 1)) {
+                        if (!nextScenario.required) {
+                            const nextResult = await this.runScenario(nextScenario, context);
+                            results.push(nextResult);
+                            totalDuration += nextResult.duration;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        const report = {
+            jobId,
+            scenarios,
+            results,
+            summary: {
+                total: results.length,
+                passed: results.filter((r) => r.status === "passed").length,
+                failed: results.filter((r) => r.status === "failed").length,
+                skipped: results.filter((r) => r.status === "skipped").length,
+                duration: totalDuration,
+            },
+            createdAt: new Date().toISOString(),
+        };
+        // Save report
+        this.saveReport(report);
+        return report;
+    }
+    /**
+     * Create a scenario from a natural language description
+     */
+    createScenario(id, name, description, required = true) {
+        return {
+            id,
+            name,
+            description,
+            steps: [],
+            required,
+        };
+    }
+    /**
+     * Add a step to a scenario
+     */
+    addStep(scenario, action, options) {
+        scenario.steps.push({
+            action,
+            selector: options?.selector,
+            value: options?.value,
+            timeout: options?.timeout,
+            assertCondition: options?.assertCondition,
+        });
+    }
+    /**
+     * Execute a single step
+     */
+    async executeStep(step, context) {
+        if (!this.runner)
+            return false;
+        const timeout = step.timeout ?? this.config.timeout ?? 30000;
+        switch (step.action) {
+            case "navigate":
+                await this.runner.navigate(this.resolveValue(step.value ?? "", context));
+                return true;
+            case "click":
+                await this.runner.wait(step.selector, timeout);
+                await this.runner.click(step.selector);
+                return true;
+            case "type":
+                await this.runner.wait(step.selector, timeout);
+                await this.runner.type(step.selector, this.resolveValue(step.value ?? "", context));
+                return true;
+            case "wait":
+                await this.runner.wait(step.selector, timeout);
+                return true;
+            case "screenshot": {
+                const path = this.resolveValue(step.value ?? "screenshot.png", context);
+                await this.runner.screenshot(path);
+                return true;
+            }
+            case "assert":
+                return await this.runner.assert(step.assertCondition ?? "true", `Assertion failed: ${step.assertCondition}`);
+            case "hover":
+                await this.runner.wait(step.selector, timeout);
+                // await this.runner.hover(step.selector!);
+                return true;
+            case "select":
+                await this.runner.wait(step.selector, timeout);
+                // await this.runner.select(step.selector!, step.value!);
+                return true;
+            case "upload":
+                await this.runner.wait(step.selector, timeout);
+                // await this.runner.upload(step.selector!, step.value!);
+                return true;
+            default:
+                console.warn(`Unknown step action: ${step.action}`);
+                return false;
+        }
+    }
+    /**
+     * Capture screenshot
+     */
+    async captureScreenshot(scenarioId, stepIndex) {
+        const path = join(this.rootDir, "harness", "e2e", "artifacts", "screenshots", `${scenarioId}-step-${stepIndex}.png`);
+        if (this.runner) {
+            await this.runner.screenshot(path);
+        }
+        return path;
+    }
+    /**
+     * Create error result
+     */
+    createErrorResult(scenarioId, error) {
+        return {
+            scenarioId,
+            status: "error",
+            duration: 0,
+            stepsExecuted: 0,
+            stepsPassed: 0,
+            stepsFailed: 0,
+            errorMessage: error,
+            executedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Save report to file
+     */
+    saveReport(report) {
+        const reportPath = join(this.rootDir, "harness", "e2e", "reports", `${report.jobId}-${Date.now()}.json`);
+        writeJson(reportPath, report);
+        // Also append to a log
+        const logPath = join(this.rootDir, "harness", "e2e", "reports", `${report.jobId}.jsonl`);
+        appendJsonl(logPath, report);
+    }
+    /**
+     * Resolve variables in values
+     */
+    resolveValue(value, context) {
+        if (!context)
+            return value;
+        let result = value;
+        for (const [key, val] of Object.entries(context)) {
+            result = result.replace(new RegExp(`{{${key}}}`, "g"), String(val));
+        }
+        return result;
+    }
+    /**
+     * Load scenarios from directory
+     */
+    loadScenarios(_scenariosDir) {
+        // In practice, this would read from the scenarios directory
+        return [];
+    }
+}
+/**
+ * Default E2E steps for common workflows
+ */
+export const CommonSteps = {
+    login: (username, password) => [
+        { action: "navigate", value: "/login" },
+        { action: "type", selector: "#username", value: username },
+        { action: "type", selector: "#password", value: password },
+        { action: "click", selector: 'button[type="submit"]' },
+        { action: "wait", selector: ".dashboard", timeout: 10000 },
+    ],
+    logout: () => [
+        { action: "click", selector: ".user-menu" },
+        { action: "click", selector: 'a[href="/logout"]' },
+    ],
+    fillForm: (fields) => {
+        const steps = [];
+        for (const [selector, value] of Object.entries(fields)) {
+            steps.push({ action: "type", selector, value });
+        }
+        return steps;
+    },
+};

package/harness/forked-summarizer.js

@@ -0,0 +1,212 @@
+/**
+ * Forked Summarizer — RFC-0028
+ *
+ * Summarizes old messages via a separate LLM call (forked context).
+ * This prevents the summarization request from hitting the same
+ * context-too-long error that triggered compaction.
+ */
+// --- Default System Prompt ---------------------------------------------------
+const DEFAULT_SUMMARY_SYSTEM = `You are a conversation summarizer. Your task is to create concise, accurate summaries that preserve essential context for continuing a conversation.
+
+Focus on:
+- What was the goal/problem being addressed?
+- What approaches were tried and what were the results?
+- What decisions were made and why?
+- What remains to be done?
+- What files or code sections are relevant?
+
+IMPORTANT: Do NOT include tool_result content verbatim. Instead, summarize the outcomes.`;
+const WORK_DONE_FOCUS = `Focus specifically on:
+1. What work was accomplished?
+2. What files were created, modified, or deleted?
+3. What tests passed or failed?
+4. What errors or issues were encountered and resolved?
+5. What is the current state of the project?`;
+const DECISIONS_FOCUS = `Focus specifically on:
+1. What architectural decisions were made?
+2. What approaches were chosen over alternatives?
+3. What was the rationale for each decision?
+4. What constraints or requirements influenced the decisions?
+5. Any lessons learned or trade-offs considered?`;
+// --- Forked Summarizer ------------------------------------------------------
+export class ForkedSummarizer {
+    config;
+    invokeAgent;
+    constructor(config, invokeAgent) {
+        this.config = {
+            provider: "openai",
+            focusOn: "all",
+            ...config,
+        };
+        this.invokeAgent = invokeAgent;
+    }
+    /**
+     * Summarize old messages, keeping recent ones as context
+     */
+    async summarize(messages, options) {
+        const keepRecentCount = options?.keepRecentCount ?? 5;
+        const focusOn = options?.focusOn ?? this.config.focusOn ?? "all";
+        // Split messages
+        const recentMessages = messages.slice(-keepRecentCount);
+        const oldMessages = messages.slice(0, -keepRecentCount);
+        if (oldMessages.length === 0) {
+            return { summary: "", droppedCount: 0 };
+        }
+        // Build summary prompt
+        const summaryPrompt = this.buildSummaryPrompt(oldMessages, { focusOn });
+        // Create summarization messages
+        const summarizationMessages = [
+            {
+                role: "system",
+                content: this.config.systemPrompt ?? DEFAULT_SUMMARY_SYSTEM,
+            },
+            {
+                role: "user",
+                content: summaryPrompt,
+            },
+        ];
+        // Call summarization model
+        try {
+            const result = await this.invokeAgent({
+                messages: summarizationMessages,
+                model: this.config.model,
+                maxOutputTokens: this.config.maxSummaryTokens,
+            });
+            if (!result.success || !result.output) {
+                throw new Error(`Summarization failed: ${result.error ?? "unknown"}`);
+            }
+            return {
+                summary: result.output.trim(),
+                droppedCount: oldMessages.length,
+                tokensUsed: result.usage?.totalTokens,
+            };
+        }
+        catch (error) {
+            // Return heuristic summary on failure
+            return {
+                summary: this.heuristicSummary(oldMessages, focusOn),
+                droppedCount: oldMessages.length,
+            };
+        }
+    }
+    /**
+     * Build the summary prompt for the LLM
+     */
+    buildSummaryPrompt(messages, options) {
+        const focusOn = options.focusOn ?? "all";
+        const formattedMessages = messages
+            .map((m) => `[${m.role}]\n${this.truncateContent(m.content, 2000)}`)
+            .join("\n\n---\n\n");
+        const focusInstructions = {
+            work_done: WORK_DONE_FOCUS,
+            decisions: DECISIONS_FOCUS,
+            all: "Provide a comprehensive summary covering all important aspects of the conversation.",
+        }[focusOn];
+        return `Summarize the following conversation concisely.
+
+${focusInstructions}
+
+## Conversation to Summarize
+${formattedMessages}
+
+## Output Format
+Provide a summary in 500-1000 tokens that:
+1. Captures the essential context
+2. Preserves key decisions and their rationale
+3. Notes any ongoing work or next steps
+4. Lists any important files or code sections referenced
+
+Do not include verbatim tool results — summarize their outcomes instead.`;
+    }
+    /**
+     * Truncate content to max length
+     */
+    truncateContent(content, maxLength) {
+        if (content.length <= maxLength)
+            return content;
+        return content.substring(0, maxLength) + "... [truncated]";
+    }
+    /**
+     * Heuristic summary when LLM summarization fails
+     */
+    heuristicSummary(messages, focusOn) {
+        const summaries = [];
+        const keyFiles = new Set();
+        const decisions = [];
+        // Extract patterns from last 20 messages
+        for (const msg of messages.slice(-20)) {
+            const content = msg.content;
+            // Look for file references
+            const filePatterns = [
+                /created (?:file|module):?\s*([^\n.]+)/gi,
+                /modified (?:file|module):?\s*([^\n.]+)/gi,
+                /wrote to ([\w./-]+)/gi,
+                /updated ([\w./-]+)/gi,
+            ];
+            for (const pattern of filePatterns) {
+                for (const match of content.matchAll(pattern)) {
+                    if (match[1])
+                        keyFiles.add(match[1].trim());
+                }
+            }
+            // Look for decisions
+            const decisionPatterns = [
+                /(?:decided|choosing|choice|chose):?\s*([^\n.]+)/gi,
+                /(?:approach|strategy):?\s*([^\n.]+)/gi,
+            ];
+            for (const pattern of decisionPatterns) {
+                for (const match of content.matchAll(pattern)) {
+                    if (match[1])
+                        decisions.push(match[1].trim());
+                }
+            }
+            // Extract key user messages
+            if (msg.role === "user" && content.length > 50) {
+                summaries.push(`- User: ${content.substring(0, 300)}`);
+            }
+            // Extract significant actions from assistant
+            if (msg.role === "assistant" && msg.metadata?.action) {
+                summaries.push(`- Action: ${msg.metadata.action}`);
+            }
+        }
+        // Build heuristic summary
+        const parts = [
+            `[Context compacted — ${messages.length} messages summarized by heuristic]`,
+            "",
+        ];
+        if (keyFiles.size > 0) {
+            parts.push("## Files Referenced");
+            for (const file of Array.from(keyFiles).slice(0, 10)) {
+                parts.push(`- ${file}`);
+            }
+            parts.push("");
+        }
+        if (decisions.length > 0) {
+            parts.push("## Key Points");
+            for (const decision of Array.from(new Set(decisions)).slice(0, 5)) {
+                parts.push(`- ${decision}`);
+            }
+            parts.push("");
+        }
+        if (summaries.length > 0) {
+            parts.push("## Summary");
+            for (const summary of summaries.slice(0, 10)) {
+                parts.push(summary);
+            }
+            parts.push("");
+        }
+        parts.push("Please continue from the recent messages provided.");
+        return parts.join("\n");
+    }
+}
+// --- Convenience Factory ----------------------------------------------------
+/**
+ * Create a ForkedSummarizer with default config
+ */
+export function createForkedSummarizer(model, invokeAgent, config) {
+    return new ForkedSummarizer({
+        model,
+        maxSummaryTokens: 4096,
+        ...config,
+    }, invokeAgent);
+}

package/harness/index.js

@@ -0,0 +1,54 @@
+/**
+ * Harness Runtime — Core Module Index
+ *
+ * Re-exports all harness components for easy importing.
+ */
+// State Machine
+export { JobStateMachine } from "./job-state-machine.js";
+// Loop Runtime
+export { LoopRuntime } from "./loop-runtime.js";
+// Repair Engine
+export { RepairEngine } from "./repair-engine.js";
+// Auto Quota Resume (5h window auto-resume)
+export { scheduleAutoResume, cancelAutoResume, getScheduledResume, } from "./auto-quota-resume.js";
+// Master Planner
+export { MasterPlanner } from "./master-planner.js";
+// Context Window Manager (RFC-0010 — enhanced with thresholds)
+export { ContextWindowManager, microcompactToolResults, parseTokenGapFromError, AUTOCOMPACT_BUFFER_TOKENS, WARNING_THRESHOLD_BUFFER_TOKENS, BLOCKING_THRESHOLD_BUFFER_TOKENS, MAX_CONSECUTIVE_COMPACT_FAILURES, } from "./context-window-manager.js";
+// Forked Summarizer (RFC-0028 Phase 2 — LLM-based compaction)
+export { ForkedSummarizer, createForkedSummarizer, } from "./forked-summarizer.js";
+// Continue Prompt Generator (RFC-0029 Phase 5 — auto-resume)
+export { ContinuePromptGenerator, continuePromptGenerator, } from "./continue-prompt.js";
+// BlackBoard
+export { createBlackboard, SharedBlackboard } from "./blackboard.js";
+// Agent Handoff
+export { AgentHandoffProtocol } from "./agent-handoff.js";
+// Auto Compact (RFC-0019)
+export { AutoCompactEngine } from "./auto-compact.js";
+// Context Compact Orchestrator (RFC-0028)
+export { CompactOrchestrator } from "./context-compact-orchestrator.js";
+// Output Limit Handler (RFC-0020)
+export { OutputLimitHandler } from "./output-limit-handler.js";
+// Partial Recovery (RFC-0021)
+export { PartialRecovery, createPartialRecovery } from "./partial-recovery.js";
+// Session Memory (RFC-0030)
+export { SessionMemoryManager, createSessionMemoryManager, } from "./session-memory.js";
+// Notification Events (RFC-0022)
+export { HarnessNotificationEvents, createNotificationConfigFromEnv, } from "./notification-events.js";
+// E2E Testing
+export { E2ETestEngine } from "./e2e/test-engine.js";
+export { PlaywrightE2ERunner } from "./e2e/playwright-runner.js";
+export { MiniMaxQuotaScraper, MiniMaxQuotaManager, } from "./e2e/minimax-quota-scraper.js";
+export { QuotaStatusManager, formatQuotaStatus, createQuotaStatusManagerFromEnv, } from "./e2e/quota-status.js";
+// Project Detector
+export { ProjectDetector } from "./project-detector/detector.js";
+// --- RFC-0056: Performance Optimizer ----------------------------------------
+export { PerformanceOptimizer, createPerformanceOptimizer, } from "../packages/performance-optimizer/src/index.js";
+// --- RFC-0057: Evaluation Engine -------------------------------------------
+export { EvaluationEngine, createEvaluationEngine, } from "../packages/evaluation-engine/src/index.js";
+// --- RFC-0058: Learning Engine ---------------------------------------------
+export { LearningEngine, createLearningEngine, } from "../packages/learning-engine/src/index.js";
+// --- RFC-0059: Experience Replay ------------------------------------------
+export { ExperienceReplay, createExperienceReplay, } from "../packages/experience-replay/src/index.js";
+// --- RFC-0060: Memory Engine -----------------------------------------------
+export { MemoryEngine, createMemoryEngine, } from "../packages/memory-engine/src/index.js";