pi-harness-runtime 0.10.13 → 0.10.15

package/packages/evaluation-engine/src/engine.js

@@ -0,0 +1,343 @@
+/**
+ * Evaluation Engine (RFC-0057)
+ *
+ * Determines whether an agent-produced result is acceptable before advancing.
+ */
+import { randomUUID } from "node:crypto";
+import { DEFAULT_SCORING_WEIGHTS } from "./types.js";
+// --- Evidence Collection -------------------------------------------------------
+function collectEvidence(outputs, testResults, policyFindings) {
+    const evidence = [];
+    // Add test results as evidence
+    for (const test of testResults) {
+        evidence.push({
+            kind: "test",
+            path: test.testFile,
+            description: `Test ${test.id}: ${test.status}`,
+            hash: test.id,
+        });
+    }
+    // Add output files as evidence
+    for (const output of outputs) {
+        if (output.path) {
+            evidence.push({
+                kind: "file",
+                path: output.path,
+                description: output.description,
+                hash: output.hash,
+            });
+        }
+    }
+    // Add policy findings as evidence
+    for (const finding of policyFindings) {
+        evidence.push({
+            kind: "policy",
+            path: finding.filePath,
+            description: `[${finding.severity.toUpperCase()}] ${finding.ruleId}: ${finding.message}`,
+        });
+    }
+    return evidence;
+}
+// --- Missing Output Detection --------------------------------------------------
+function findMissingOutputs(requiredArtifacts, outputs) {
+    const findings = [];
+    const outputPaths = new Set(outputs.filter((o) => o.path).map((o) => o.path));
+    for (const required of requiredArtifacts) {
+        const requiredLower = required.toLowerCase();
+        const found = outputPaths.has(required) ||
+            outputs.some((o) => o.description.toLowerCase().includes(requiredLower));
+        if (!found) {
+            findings.push({
+                dimension: "correctness",
+                severity: "critical",
+                message: `Required artifact '${required}' is missing from outputs`,
+                evidence: `Required: ${required}, Available: ${[...outputPaths].join(", ") || "none"}`,
+            });
+        }
+    }
+    return findings;
+}
+// --- Test Analysis ------------------------------------------------------------
+function analyzeTests(testResults) {
+    const findings = [];
+    let passedCount = 0;
+    let totalCount = 0;
+    let hasFailures = false;
+    for (const test of testResults) {
+        totalCount++;
+        if (test.status === "passed") {
+            passedCount++;
+        }
+        else if (test.status === "failed" || test.status === "error") {
+            hasFailures = true;
+            findings.push({
+                dimension: "test_completeness",
+                severity: "critical",
+                message: `Test '${test.testFile}' ${test.status}: ${test.errorMessage || "Unknown error"}`,
+                evidence: test.errorMessage,
+            });
+        }
+    }
+    if (testResults.length === 0) {
+        findings.push({
+            dimension: "test_completeness",
+            severity: "high",
+            message: "No test results provided",
+        });
+    }
+    return { findings, passedCount, totalCount, hasFailures };
+}
+// --- Policy Analysis ----------------------------------------------------------
+function analyzePolicies(policyFindings) {
+    const findings = [];
+    const criticalFindings = [];
+    for (const finding of policyFindings) {
+        findings.push({
+            dimension: "policy_compliance",
+            severity: finding.severity,
+            message: `${finding.ruleId}: ${finding.message}`,
+            evidence: finding.filePath
+                ? `${finding.filePath}:${finding.line || "?"}`
+                : undefined,
+        });
+        if (finding.severity === "critical") {
+            criticalFindings.push(finding);
+        }
+    }
+    return { findings, criticalFindings };
+}
+// --- Acceptance Criteria Coverage ---------------------------------------------
+function analyzeAcceptanceCoverage(acceptanceCriteria, outputs, testResults) {
+    const findings = [];
+    const coveredCriteria = [];
+    const unmetCriteria = [];
+    const outputText = outputs
+        .map((o) => `${o.description} ${o.path || ""}`.toLowerCase())
+        .join(" ");
+    const passingTests = testResults.filter((t) => t.status === "passed");
+    const passingTestText = passingTests
+        .map((t) => `${t.testFile} ${t.errorMessage || ""}`.toLowerCase())
+        .join(" ");
+    for (const criterion of acceptanceCriteria) {
+        const criterionLower = criterion.toLowerCase();
+        const keywords = criterionLower
+            .split(/[^a-z0-9]+/)
+            .filter((word) => word.length >= 4);
+        const outputKeywordMatches = keywords.filter((word) => outputText.includes(word)).length;
+        const coveredByOutput = keywords.length > 0 &&
+            outputKeywordMatches >= Math.ceil(keywords.length / 2);
+        const coveredByPassingTests = passingTests.length > 0 &&
+            (criterionLower.includes("test") ||
+                criterionLower.includes("pass") ||
+                keywords.some((word) => passingTestText.includes(word)));
+        if (coveredByOutput || coveredByPassingTests) {
+            coveredCriteria.push(criterion);
+        }
+        else {
+            unmetCriteria.push(criterion);
+            findings.push({
+                dimension: "acceptance_coverage",
+                severity: "medium",
+                message: `Acceptance criterion not covered: '${criterion}'`,
+                criterion,
+            });
+        }
+    }
+    const coverage = acceptanceCriteria.length > 0
+        ? (coveredCriteria.length / acceptanceCriteria.length) * 100
+        : 100;
+    return { findings, coverage, unmetCriteria };
+}
+// --- Scoring -----------------------------------------------------------------
+function calculateScore(weights, testAnalysis, acceptanceCoverage, criticalPolicies, securityFindings, regressionFindings) {
+    const breakdown = {};
+    // Correctness (30%) - based on missing outputs
+    const correctnessScore = weights.correctness;
+    breakdown.correctness = correctnessScore;
+    // Test completeness (20%)
+    let testScore = 0;
+    if (testAnalysis.totalCount > 0) {
+        testScore =
+            (testAnalysis.passedCount / testAnalysis.totalCount) * weights.test;
+    }
+    breakdown.tests = testScore;
+    // Acceptance coverage (20%)
+    const acceptanceScore = (acceptanceCoverage / 100) * weights.acceptance;
+    breakdown.acceptance = acceptanceScore;
+    // Code quality (15%) - assume passed if no major findings
+    const qualityScore = weights.quality;
+    breakdown.quality = qualityScore;
+    // Policy penalty (deducted)
+    let policyPenalty = 0;
+    for (const _ of criticalPolicies) {
+        policyPenalty += weights.policyPenalty;
+    }
+    breakdown.policyPenalty = -policyPenalty;
+    // Security penalty (deducted)
+    const securityPenalty = securityFindings.length * weights.securityPenalty;
+    breakdown.securityPenalty = -securityPenalty;
+    // Regression penalty (deducted)
+    const regressionPenalty = regressionFindings.length * weights.regressionPenalty;
+    breakdown.regressionPenalty = -regressionPenalty;
+    const totalScore = Math.max(0, Math.min(100, correctnessScore +
+        testScore +
+        acceptanceScore +
+        qualityScore -
+        policyPenalty -
+        securityPenalty -
+        regressionPenalty));
+    return { score: Math.round(totalScore), breakdown };
+}
+// --- Failure Classification ---------------------------------------------------
+function classifyFailure(findings) {
+    const classes = [];
+    for (const finding of findings) {
+        if (finding.dimension === "test_completeness" &&
+            finding.severity === "critical") {
+            if (!classes.includes("test_failure"))
+                classes.push("test_failure");
+        }
+        if (finding.dimension === "correctness" && finding.severity === "high") {
+            if (!classes.includes("missing_output"))
+                classes.push("missing_output");
+        }
+        if (finding.dimension === "acceptance_coverage" &&
+            finding.severity === "medium") {
+            if (!classes.includes("acceptance_gap"))
+                classes.push("acceptance_gap");
+        }
+        if (finding.dimension === "policy_compliance") {
+            if (finding.severity === "critical") {
+                if (!classes.includes("policy_violation"))
+                    classes.push("policy_violation");
+            }
+        }
+        if (finding.dimension === "security_risk") {
+            if (!classes.includes("security_issue"))
+                classes.push("security_issue");
+        }
+        if (finding.dimension === "regression_risk") {
+            if (!classes.includes("regression_risk"))
+                classes.push("regression_risk");
+        }
+    }
+    if (classes.length === 0 && findings.length > 0) {
+        classes.push("insufficient_evidence");
+    }
+    return classes;
+}
+// --- Main Engine ---------------------------------------------------------------
+export class EvaluationEngine {
+    weights;
+    state = "pending";
+    constructor(weights) {
+        this.weights = { ...DEFAULT_SCORING_WEIGHTS, ...weights };
+    }
+    /**
+     * Evaluate a task against its acceptance criteria
+     */
+    evaluate(request) {
+        this.state = "collecting_evidence";
+        const { jobId, task, outputs, testResults, policyFindings } = request;
+        const allFindings = [];
+        this.state = "evaluating";
+        // 1. Check for missing required outputs
+        const missingOutputs = findMissingOutputs(task.requiredArtifacts, outputs);
+        allFindings.push(...missingOutputs);
+        // 2. Analyze tests
+        const testAnalysis = analyzeTests(testResults);
+        allFindings.push(...testAnalysis.findings);
+        // 3. Analyze policies
+        const policyAnalysis = analyzePolicies(policyFindings);
+        allFindings.push(...policyAnalysis.findings);
+        // 4. Check acceptance criteria coverage
+        const acceptanceAnalysis = analyzeAcceptanceCoverage(task.acceptanceCriteria, outputs, testResults);
+        allFindings.push(...acceptanceAnalysis.findings);
+        // 5. Check for security issues
+        const securityFindings = allFindings.filter((f) => f.dimension === "security_risk");
+        // 6. Check for regression risks
+        const regressionFindings = allFindings.filter((f) => f.dimension === "regression_risk");
+        // 7. Calculate score
+        const { score, breakdown: _breakdown } = calculateScore(this.weights, testAnalysis, acceptanceAnalysis.coverage, policyAnalysis.criticalFindings, securityFindings, regressionFindings);
+        // 8. Collect evidence
+        const evidence = collectEvidence(outputs, testResults, policyFindings);
+        // 9. Determine status and recommended action
+        let status;
+        let recommendedAction;
+        // Critical policy violations force failure
+        if (policyAnalysis.criticalFindings.length > 0) {
+            status = "failed";
+            recommendedAction = "repair";
+            this.state = "failed";
+        }
+        // Test failures force failure
+        else if (testAnalysis.hasFailures) {
+            status = "failed";
+            recommendedAction = "repair";
+            this.state = "failed";
+        }
+        // Missing critical outputs
+        else if (missingOutputs.some((f) => f.severity === "critical")) {
+            status = "failed";
+            recommendedAction = "repair";
+            this.state = "failed";
+        }
+        // Low score with significant gaps
+        else if (score < 50) {
+            status = "failed";
+            recommendedAction = "repair";
+            this.state = "failed";
+        }
+        // Ambiguous criteria coverage
+        else if (acceptanceAnalysis.coverage < 80 &&
+            acceptanceAnalysis.unmetCriteria.length > 0) {
+            status = "needs_review";
+            recommendedAction = "human_review";
+            this.state = "needs_review";
+        }
+        // Good score
+        else {
+            status = "passed";
+            recommendedAction = "continue";
+            this.state = "passed";
+        }
+        // 10. Classify failure
+        const failureClasses = classifyFailure(allFindings);
+        // Add failure classification as a finding if failed
+        if (status === "failed" && failureClasses.length > 0) {
+            allFindings.push({
+                dimension: "correctness",
+                severity: "critical",
+                message: `Evaluation failed due to: ${failureClasses.join(", ")}`,
+                evidence: failureClasses.join(", "),
+            });
+        }
+        return {
+            status,
+            score,
+            findings: allFindings,
+            evidence,
+            recommendedAction,
+            evaluationId: randomUUID(),
+            jobId,
+            taskId: task.id,
+            evaluatedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Get current evaluation state
+     */
+    getState() {
+        return this.state;
+    }
+    /**
+     * Reset engine state
+     */
+    reset() {
+        this.state = "pending";
+    }
+}
+// --- Factory -----------------------------------------------------------------
+export function createEvaluationEngine(weights) {
+    return new EvaluationEngine(weights);
+}

package/packages/evaluation-engine/src/index.js

@@ -0,0 +1,7 @@
+/**
+ * Evaluation Engine (RFC-0057)
+ *
+ * Re-exports all public types and classes.
+ */
+export { EvaluationEngine, createEvaluationEngine } from "./engine.js";
+export { DEFAULT_SCORING_WEIGHTS } from "./types.js";

package/packages/evaluation-engine/src/types.js

@@ -0,0 +1,14 @@
+/**
+ * Evaluation Engine Types (RFC-0057)
+ *
+ * Interfaces for task evaluation and acceptance criteria checking.
+ */
+export const DEFAULT_SCORING_WEIGHTS = {
+    correctness: 30,
+    test: 20,
+    acceptance: 20,
+    quality: 15,
+    policyPenalty: 20,
+    securityPenalty: 30,
+    regressionPenalty: 15,
+};

package/packages/event-store/src/types.js

@@ -0,0 +1,5 @@
+/**
+ * Event Store — Types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/packages/experience-replay/src/index.js

@@ -0,0 +1,6 @@
+/**
+ * Experience Replay (RFC-0059)
+ *
+ * Re-exports all public types and classes.
+ */
+export { ExperienceReplay, createExperienceReplay } from "./replay.js";

package/packages/experience-replay/src/replay.js

@@ -0,0 +1,228 @@
+/**
+ * Experience Replay (RFC-0059)
+ *
+ * Reconstructs prior runtime execution from persisted events and checkpoints.
+ */
+// --- Event Timeline Reconstruction -------------------------------------------
+function reconstructTimeline(events, startSequence = 0) {
+    return events.map((event, index) => ({
+        sequence: startSequence + index,
+        timestamp: event.ts,
+        type: event.type,
+        data: event.data || {},
+    }));
+}
+// --- State Reconstruction -----------------------------------------------------
+function reconstructState(jobId, checkpoint, events, taskGraph) {
+    // Apply events to checkpoint to get final state
+    let currentTaskId = checkpoint?.currentTaskId;
+    let status = checkpoint?.status || "unknown";
+    // Process events to find latest state
+    for (const event of events) {
+        if (event.type === "task.started" && event.data?.taskId) {
+            currentTaskId = event.data.taskId;
+        }
+        if (event.type === "job.status") {
+            status = event.data?.status || status;
+        }
+    }
+    return {
+        jobId,
+        status,
+        currentTaskId,
+        checkpoint: checkpoint || {
+            version: 0,
+            jobId,
+            status: "unknown",
+            requirement: "",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+        },
+        taskGraph: taskGraph || { nodes: [], edges: [] },
+        events,
+        timestamp: new Date().toISOString(),
+    };
+}
+// --- Divergence Detection ----------------------------------------------------
+// --- Source Validation --------------------------------------------------------
+function validateSources(sources) {
+    const missing = [];
+    if (!sources.checkpoint && !sources.events) {
+        missing.push("checkpoint or events");
+    }
+    return { valid: missing.length === 0, missing };
+}
+// --- Mode Handlers ------------------------------------------------------------
+async function handleInspect(sources, jobId, fromSequence, toSequence) {
+    const events = sources.events || [];
+    const filteredEvents = fromSequence !== undefined || toSequence !== undefined
+        ? events.slice(fromSequence, toSequence)
+        : events;
+    const timeline = reconstructTimeline(filteredEvents, fromSequence ?? 0);
+    const state = reconstructState(jobId, sources.checkpoint, filteredEvents, sources.taskGraph);
+    return { state, timeline };
+}
+async function handleSimulate(sources, jobId, fromSequence, toSequence) {
+    const events = sources.events || [];
+    const filteredEvents = fromSequence !== undefined || toSequence !== undefined
+        ? events.slice(fromSequence, toSequence)
+        : events;
+    const timeline = reconstructTimeline(filteredEvents, fromSequence ?? 0);
+    const state = reconstructState(jobId, sources.checkpoint, filteredEvents, sources.taskGraph);
+    // Check for divergent events
+    const divergences = [];
+    for (let i = 0; i < timeline.length; i++) {
+        const event = timeline[i];
+        if (event.type.includes("error") || event.type.includes("failure")) {
+            divergences.push({
+                sequence: event.sequence,
+                reason: "Provider output differs",
+                severity: "warning",
+            });
+        }
+    }
+    return { state, timeline, divergences };
+}
+async function handleReexecute(sources, jobId, allowExternalCalls) {
+    // Reexecute requires explicit approval and valid sources
+    const validation = validateSources(sources);
+    if (!validation.valid) {
+        throw new Error(`Missing required sources: ${validation.missing.join(", ")}`);
+    }
+    if (!allowExternalCalls) {
+        throw new Error("Reexecution with external calls requires explicit approval");
+    }
+    // For reexecution, we simulate first then mark for actual reexecution
+    const result = await handleSimulate(sources, jobId);
+    // Add informational divergence for reexecution mode
+    result.divergences.push({
+        sequence: 0,
+        reason: "Runtime version differs",
+        severity: "info",
+    });
+    return result;
+}
+// --- Main Engine ---------------------------------------------------------------
+export class ExperienceReplay {
+    eventListeners = [];
+    /**
+     * Replay a prior job execution
+     */
+    async replay(request, sources) {
+        const { jobId, mode } = request;
+        // Emit start event
+        this.emit({ type: "replay.started", jobId, mode });
+        // Validate sources
+        const validation = validateSources(sources);
+        if (!validation.valid) {
+            return {
+                jobId,
+                reconstructedState: {
+                    jobId,
+                    status: "error",
+                    checkpoint: sources.checkpoint || {},
+                    taskGraph: sources.taskGraph || { nodes: [], edges: [] },
+                    events: [],
+                    timestamp: new Date().toISOString(),
+                },
+                timeline: [],
+                divergences: [
+                    {
+                        sequence: 0,
+                        reason: `Missing sources: ${validation.missing.join(", ")}`,
+                        severity: "error",
+                    },
+                ],
+                artifacts: [],
+                replayedAt: new Date().toISOString(),
+            };
+        }
+        try {
+            let result;
+            switch (mode) {
+                case "inspect":
+                    result = await handleInspect(sources, jobId, request.fromSequence, request.toSequence);
+                    break;
+                case "simulate":
+                    result = await handleSimulate(sources, jobId, request.fromSequence, request.toSequence);
+                    break;
+                case "reexecute":
+                    result = await handleReexecute(sources, jobId, request.allowExternalCalls);
+                    break;
+            }
+            // Emit divergence events
+            if (result.divergences) {
+                for (const div of result.divergences) {
+                    this.emit({
+                        type: "replay.divergence.detected",
+                        jobId,
+                        reason: div.reason,
+                    });
+                }
+            }
+            // Collect artifacts
+            const artifacts = [];
+            if (sources.prompts)
+                artifacts.push(...Object.keys(sources.prompts));
+            if (sources.outputs)
+                artifacts.push(...Object.keys(sources.outputs));
+            // Emit completion event
+            this.emit({ type: "replay.completed", jobId });
+            return {
+                jobId,
+                reconstructedState: result.state,
+                timeline: result.timeline,
+                divergences: result.divergences || [],
+                artifacts,
+                replayedAt: new Date().toISOString(),
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                jobId,
+                reconstructedState: {
+                    jobId,
+                    status: "error",
+                    checkpoint: sources.checkpoint || {},
+                    taskGraph: sources.taskGraph || { nodes: [], edges: [] },
+                    events: [],
+                    timestamp: new Date().toISOString(),
+                },
+                timeline: [],
+                divergences: [
+                    {
+                        sequence: 0,
+                        reason: errorMessage,
+                        severity: "error",
+                    },
+                ],
+                artifacts: [],
+                replayedAt: new Date().toISOString(),
+            };
+        }
+    }
+    /**
+     * Subscribe to replay events
+     */
+    onEvent(listener) {
+        this.eventListeners.push(listener);
+        return () => {
+            this.eventListeners = this.eventListeners.filter((l) => l !== listener);
+        };
+    }
+    emit(event) {
+        for (const listener of this.eventListeners) {
+            try {
+                listener(event);
+            }
+            catch {
+                // Best effort
+            }
+        }
+    }
+}
+// --- Factory -----------------------------------------------------------------
+export function createExperienceReplay() {
+    return new ExperienceReplay();
+}

package/packages/experience-replay/src/types.js

@@ -0,0 +1,6 @@
+/**
+ * Experience Replay Types (RFC-0059)
+ *
+ * Interfaces for replaying prior runtime execution.
+ */
+export {};