pi-harness-runtime 0.10.22 → 0.10.24

package/packages/evaluation-engine/src/engine.js

@@ -1,343 +0,0 @@
-/**
- * Evaluation Engine (RFC-0057)
- *
- * Determines whether an agent-produced result is acceptable before advancing.
- */
-import { randomUUID } from "node:crypto";
-import { DEFAULT_SCORING_WEIGHTS } from "./types.js";
-// --- Evidence Collection -------------------------------------------------------
-function collectEvidence(outputs, testResults, policyFindings) {
-    const evidence = [];
-    // Add test results as evidence
-    for (const test of testResults) {
-        evidence.push({
-            kind: "test",
-            path: test.testFile,
-            description: `Test ${test.id}: ${test.status}`,
-            hash: test.id,
-        });
-    }
-    // Add output files as evidence
-    for (const output of outputs) {
-        if (output.path) {
-            evidence.push({
-                kind: "file",
-                path: output.path,
-                description: output.description,
-                hash: output.hash,
-            });
-        }
-    }
-    // Add policy findings as evidence
-    for (const finding of policyFindings) {
-        evidence.push({
-            kind: "policy",
-            path: finding.filePath,
-            description: `[${finding.severity.toUpperCase()}] ${finding.ruleId}: ${finding.message}`,
-        });
-    }
-    return evidence;
-}
-// --- Missing Output Detection --------------------------------------------------
-function findMissingOutputs(requiredArtifacts, outputs) {
-    const findings = [];
-    const outputPaths = new Set(outputs.filter((o) => o.path).map((o) => o.path));
-    for (const required of requiredArtifacts) {
-        const requiredLower = required.toLowerCase();
-        const found = outputPaths.has(required) ||
-            outputs.some((o) => o.description.toLowerCase().includes(requiredLower));
-        if (!found) {
-            findings.push({
-                dimension: "correctness",
-                severity: "critical",
-                message: `Required artifact '${required}' is missing from outputs`,
-                evidence: `Required: ${required}, Available: ${[...outputPaths].join(", ") || "none"}`,
-            });
-        }
-    }
-    return findings;
-}
-// --- Test Analysis ------------------------------------------------------------
-function analyzeTests(testResults) {
-    const findings = [];
-    let passedCount = 0;
-    let totalCount = 0;
-    let hasFailures = false;
-    for (const test of testResults) {
-        totalCount++;
-        if (test.status === "passed") {
-            passedCount++;
-        }
-        else if (test.status === "failed" || test.status === "error") {
-            hasFailures = true;
-            findings.push({
-                dimension: "test_completeness",
-                severity: "critical",
-                message: `Test '${test.testFile}' ${test.status}: ${test.errorMessage || "Unknown error"}`,
-                evidence: test.errorMessage,
-            });
-        }
-    }
-    if (testResults.length === 0) {
-        findings.push({
-            dimension: "test_completeness",
-            severity: "high",
-            message: "No test results provided",
-        });
-    }
-    return { findings, passedCount, totalCount, hasFailures };
-}
-// --- Policy Analysis ----------------------------------------------------------
-function analyzePolicies(policyFindings) {
-    const findings = [];
-    const criticalFindings = [];
-    for (const finding of policyFindings) {
-        findings.push({
-            dimension: "policy_compliance",
-            severity: finding.severity,
-            message: `${finding.ruleId}: ${finding.message}`,
-            evidence: finding.filePath
-                ? `${finding.filePath}:${finding.line || "?"}`
-                : undefined,
-        });
-        if (finding.severity === "critical") {
-            criticalFindings.push(finding);
-        }
-    }
-    return { findings, criticalFindings };
-}
-// --- Acceptance Criteria Coverage ---------------------------------------------
-function analyzeAcceptanceCoverage(acceptanceCriteria, outputs, testResults) {
-    const findings = [];
-    const coveredCriteria = [];
-    const unmetCriteria = [];
-    const outputText = outputs
-        .map((o) => `${o.description} ${o.path || ""}`.toLowerCase())
-        .join(" ");
-    const passingTests = testResults.filter((t) => t.status === "passed");
-    const passingTestText = passingTests
-        .map((t) => `${t.testFile} ${t.errorMessage || ""}`.toLowerCase())
-        .join(" ");
-    for (const criterion of acceptanceCriteria) {
-        const criterionLower = criterion.toLowerCase();
-        const keywords = criterionLower
-            .split(/[^a-z0-9]+/)
-            .filter((word) => word.length >= 4);
-        const outputKeywordMatches = keywords.filter((word) => outputText.includes(word)).length;
-        const coveredByOutput = keywords.length > 0 &&
-            outputKeywordMatches >= Math.ceil(keywords.length / 2);
-        const coveredByPassingTests = passingTests.length > 0 &&
-            (criterionLower.includes("test") ||
-                criterionLower.includes("pass") ||
-                keywords.some((word) => passingTestText.includes(word)));
-        if (coveredByOutput || coveredByPassingTests) {
-            coveredCriteria.push(criterion);
-        }
-        else {
-            unmetCriteria.push(criterion);
-            findings.push({
-                dimension: "acceptance_coverage",
-                severity: "medium",
-                message: `Acceptance criterion not covered: '${criterion}'`,
-                criterion,
-            });
-        }
-    }
-    const coverage = acceptanceCriteria.length > 0
-        ? (coveredCriteria.length / acceptanceCriteria.length) * 100
-        : 100;
-    return { findings, coverage, unmetCriteria };
-}
-// --- Scoring -----------------------------------------------------------------
-function calculateScore(weights, testAnalysis, acceptanceCoverage, criticalPolicies, securityFindings, regressionFindings) {
-    const breakdown = {};
-    // Correctness (30%) - based on missing outputs
-    const correctnessScore = weights.correctness;
-    breakdown.correctness = correctnessScore;
-    // Test completeness (20%)
-    let testScore = 0;
-    if (testAnalysis.totalCount > 0) {
-        testScore =
-            (testAnalysis.passedCount / testAnalysis.totalCount) * weights.test;
-    }
-    breakdown.tests = testScore;
-    // Acceptance coverage (20%)
-    const acceptanceScore = (acceptanceCoverage / 100) * weights.acceptance;
-    breakdown.acceptance = acceptanceScore;
-    // Code quality (15%) - assume passed if no major findings
-    const qualityScore = weights.quality;
-    breakdown.quality = qualityScore;
-    // Policy penalty (deducted)
-    let policyPenalty = 0;
-    for (const _ of criticalPolicies) {
-        policyPenalty += weights.policyPenalty;
-    }
-    breakdown.policyPenalty = -policyPenalty;
-    // Security penalty (deducted)
-    const securityPenalty = securityFindings.length * weights.securityPenalty;
-    breakdown.securityPenalty = -securityPenalty;
-    // Regression penalty (deducted)
-    const regressionPenalty = regressionFindings.length * weights.regressionPenalty;
-    breakdown.regressionPenalty = -regressionPenalty;
-    const totalScore = Math.max(0, Math.min(100, correctnessScore +
-        testScore +
-        acceptanceScore +
-        qualityScore -
-        policyPenalty -
-        securityPenalty -
-        regressionPenalty));
-    return { score: Math.round(totalScore), breakdown };
-}
-// --- Failure Classification ---------------------------------------------------
-function classifyFailure(findings) {
-    const classes = [];
-    for (const finding of findings) {
-        if (finding.dimension === "test_completeness" &&
-            finding.severity === "critical") {
-            if (!classes.includes("test_failure"))
-                classes.push("test_failure");
-        }
-        if (finding.dimension === "correctness" && finding.severity === "high") {
-            if (!classes.includes("missing_output"))
-                classes.push("missing_output");
-        }
-        if (finding.dimension === "acceptance_coverage" &&
-            finding.severity === "medium") {
-            if (!classes.includes("acceptance_gap"))
-                classes.push("acceptance_gap");
-        }
-        if (finding.dimension === "policy_compliance") {
-            if (finding.severity === "critical") {
-                if (!classes.includes("policy_violation"))
-                    classes.push("policy_violation");
-            }
-        }
-        if (finding.dimension === "security_risk") {
-            if (!classes.includes("security_issue"))
-                classes.push("security_issue");
-        }
-        if (finding.dimension === "regression_risk") {
-            if (!classes.includes("regression_risk"))
-                classes.push("regression_risk");
-        }
-    }
-    if (classes.length === 0 && findings.length > 0) {
-        classes.push("insufficient_evidence");
-    }
-    return classes;
-}
-// --- Main Engine ---------------------------------------------------------------
-export class EvaluationEngine {
-    weights;
-    state = "pending";
-    constructor(weights) {
-        this.weights = { ...DEFAULT_SCORING_WEIGHTS, ...weights };
-    }
-    /**
-     * Evaluate a task against its acceptance criteria
-     */
-    evaluate(request) {
-        this.state = "collecting_evidence";
-        const { jobId, task, outputs, testResults, policyFindings } = request;
-        const allFindings = [];
-        this.state = "evaluating";
-        // 1. Check for missing required outputs
-        const missingOutputs = findMissingOutputs(task.requiredArtifacts, outputs);
-        allFindings.push(...missingOutputs);
-        // 2. Analyze tests
-        const testAnalysis = analyzeTests(testResults);
-        allFindings.push(...testAnalysis.findings);
-        // 3. Analyze policies
-        const policyAnalysis = analyzePolicies(policyFindings);
-        allFindings.push(...policyAnalysis.findings);
-        // 4. Check acceptance criteria coverage
-        const acceptanceAnalysis = analyzeAcceptanceCoverage(task.acceptanceCriteria, outputs, testResults);
-        allFindings.push(...acceptanceAnalysis.findings);
-        // 5. Check for security issues
-        const securityFindings = allFindings.filter((f) => f.dimension === "security_risk");
-        // 6. Check for regression risks
-        const regressionFindings = allFindings.filter((f) => f.dimension === "regression_risk");
-        // 7. Calculate score
-        const { score, breakdown: _breakdown } = calculateScore(this.weights, testAnalysis, acceptanceAnalysis.coverage, policyAnalysis.criticalFindings, securityFindings, regressionFindings);
-        // 8. Collect evidence
-        const evidence = collectEvidence(outputs, testResults, policyFindings);
-        // 9. Determine status and recommended action
-        let status;
-        let recommendedAction;
-        // Critical policy violations force failure
-        if (policyAnalysis.criticalFindings.length > 0) {
-            status = "failed";
-            recommendedAction = "repair";
-            this.state = "failed";
-        }
-        // Test failures force failure
-        else if (testAnalysis.hasFailures) {
-            status = "failed";
-            recommendedAction = "repair";
-            this.state = "failed";
-        }
-        // Missing critical outputs
-        else if (missingOutputs.some((f) => f.severity === "critical")) {
-            status = "failed";
-            recommendedAction = "repair";
-            this.state = "failed";
-        }
-        // Low score with significant gaps
-        else if (score < 50) {
-            status = "failed";
-            recommendedAction = "repair";
-            this.state = "failed";
-        }
-        // Ambiguous criteria coverage
-        else if (acceptanceAnalysis.coverage < 80 &&
-            acceptanceAnalysis.unmetCriteria.length > 0) {
-            status = "needs_review";
-            recommendedAction = "human_review";
-            this.state = "needs_review";
-        }
-        // Good score
-        else {
-            status = "passed";
-            recommendedAction = "continue";
-            this.state = "passed";
-        }
-        // 10. Classify failure
-        const failureClasses = classifyFailure(allFindings);
-        // Add failure classification as a finding if failed
-        if (status === "failed" && failureClasses.length > 0) {
-            allFindings.push({
-                dimension: "correctness",
-                severity: "critical",
-                message: `Evaluation failed due to: ${failureClasses.join(", ")}`,
-                evidence: failureClasses.join(", "),
-            });
-        }
-        return {
-            status,
-            score,
-            findings: allFindings,
-            evidence,
-            recommendedAction,
-            evaluationId: randomUUID(),
-            jobId,
-            taskId: task.id,
-            evaluatedAt: new Date().toISOString(),
-        };
-    }
-    /**
-     * Get current evaluation state
-     */
-    getState() {
-        return this.state;
-    }
-    /**
-     * Reset engine state
-     */
-    reset() {
-        this.state = "pending";
-    }
-}
-// --- Factory -----------------------------------------------------------------
-export function createEvaluationEngine(weights) {
-    return new EvaluationEngine(weights);
-}

package/packages/evaluation-engine/src/index.js

@@ -1,7 +0,0 @@
-/**
- * Evaluation Engine (RFC-0057)
- *
- * Re-exports all public types and classes.
- */
-export { EvaluationEngine, createEvaluationEngine } from "./engine.js";
-export { DEFAULT_SCORING_WEIGHTS } from "./types.js";

package/packages/evaluation-engine/src/types.js

@@ -1,14 +0,0 @@
-/**
- * Evaluation Engine Types (RFC-0057)
- *
- * Interfaces for task evaluation and acceptance criteria checking.
- */
-export const DEFAULT_SCORING_WEIGHTS = {
-    correctness: 30,
-    test: 20,
-    acceptance: 20,
-    quality: 15,
-    policyPenalty: 20,
-    securityPenalty: 30,
-    regressionPenalty: 15,
-};

package/packages/event-store/src/types.js

@@ -1,5 +0,0 @@
-/**
- * Event Store — Types
- */
-export {};
-//# sourceMappingURL=types.js.map

package/packages/experience-replay/src/index.js

@@ -1,6 +0,0 @@
-/**
- * Experience Replay (RFC-0059)
- *
- * Re-exports all public types and classes.
- */
-export { ExperienceReplay, createExperienceReplay } from "./replay.js";

package/packages/experience-replay/src/replay.js

@@ -1,228 +0,0 @@
-/**
- * Experience Replay (RFC-0059)
- *
- * Reconstructs prior runtime execution from persisted events and checkpoints.
- */
-// --- Event Timeline Reconstruction -------------------------------------------
-function reconstructTimeline(events, startSequence = 0) {
-    return events.map((event, index) => ({
-        sequence: startSequence + index,
-        timestamp: event.ts,
-        type: event.type,
-        data: event.data || {},
-    }));
-}
-// --- State Reconstruction -----------------------------------------------------
-function reconstructState(jobId, checkpoint, events, taskGraph) {
-    // Apply events to checkpoint to get final state
-    let currentTaskId = checkpoint?.currentTaskId;
-    let status = checkpoint?.status || "unknown";
-    // Process events to find latest state
-    for (const event of events) {
-        if (event.type === "task.started" && event.data?.taskId) {
-            currentTaskId = event.data.taskId;
-        }
-        if (event.type === "job.status") {
-            status = event.data?.status || status;
-        }
-    }
-    return {
-        jobId,
-        status,
-        currentTaskId,
-        checkpoint: checkpoint || {
-            version: 0,
-            jobId,
-            status: "unknown",
-            requirement: "",
-            createdAt: new Date().toISOString(),
-            updatedAt: new Date().toISOString(),
-        },
-        taskGraph: taskGraph || { nodes: [], edges: [] },
-        events,
-        timestamp: new Date().toISOString(),
-    };
-}
-// --- Divergence Detection ----------------------------------------------------
-// --- Source Validation --------------------------------------------------------
-function validateSources(sources) {
-    const missing = [];
-    if (!sources.checkpoint && !sources.events) {
-        missing.push("checkpoint or events");
-    }
-    return { valid: missing.length === 0, missing };
-}
-// --- Mode Handlers ------------------------------------------------------------
-async function handleInspect(sources, jobId, fromSequence, toSequence) {
-    const events = sources.events || [];
-    const filteredEvents = fromSequence !== undefined || toSequence !== undefined
-        ? events.slice(fromSequence, toSequence)
-        : events;
-    const timeline = reconstructTimeline(filteredEvents, fromSequence ?? 0);
-    const state = reconstructState(jobId, sources.checkpoint, filteredEvents, sources.taskGraph);
-    return { state, timeline };
-}
-async function handleSimulate(sources, jobId, fromSequence, toSequence) {
-    const events = sources.events || [];
-    const filteredEvents = fromSequence !== undefined || toSequence !== undefined
-        ? events.slice(fromSequence, toSequence)
-        : events;
-    const timeline = reconstructTimeline(filteredEvents, fromSequence ?? 0);
-    const state = reconstructState(jobId, sources.checkpoint, filteredEvents, sources.taskGraph);
-    // Check for divergent events
-    const divergences = [];
-    for (let i = 0; i < timeline.length; i++) {
-        const event = timeline[i];
-        if (event.type.includes("error") || event.type.includes("failure")) {
-            divergences.push({
-                sequence: event.sequence,
-                reason: "Provider output differs",
-                severity: "warning",
-            });
-        }
-    }
-    return { state, timeline, divergences };
-}
-async function handleReexecute(sources, jobId, allowExternalCalls) {
-    // Reexecute requires explicit approval and valid sources
-    const validation = validateSources(sources);
-    if (!validation.valid) {
-        throw new Error(`Missing required sources: ${validation.missing.join(", ")}`);
-    }
-    if (!allowExternalCalls) {
-        throw new Error("Reexecution with external calls requires explicit approval");
-    }
-    // For reexecution, we simulate first then mark for actual reexecution
-    const result = await handleSimulate(sources, jobId);
-    // Add informational divergence for reexecution mode
-    result.divergences.push({
-        sequence: 0,
-        reason: "Runtime version differs",
-        severity: "info",
-    });
-    return result;
-}
-// --- Main Engine ---------------------------------------------------------------
-export class ExperienceReplay {
-    eventListeners = [];
-    /**
-     * Replay a prior job execution
-     */
-    async replay(request, sources) {
-        const { jobId, mode } = request;
-        // Emit start event
-        this.emit({ type: "replay.started", jobId, mode });
-        // Validate sources
-        const validation = validateSources(sources);
-        if (!validation.valid) {
-            return {
-                jobId,
-                reconstructedState: {
-                    jobId,
-                    status: "error",
-                    checkpoint: sources.checkpoint || {},
-                    taskGraph: sources.taskGraph || { nodes: [], edges: [] },
-                    events: [],
-                    timestamp: new Date().toISOString(),
-                },
-                timeline: [],
-                divergences: [
-                    {
-                        sequence: 0,
-                        reason: `Missing sources: ${validation.missing.join(", ")}`,
-                        severity: "error",
-                    },
-                ],
-                artifacts: [],
-                replayedAt: new Date().toISOString(),
-            };
-        }
-        try {
-            let result;
-            switch (mode) {
-                case "inspect":
-                    result = await handleInspect(sources, jobId, request.fromSequence, request.toSequence);
-                    break;
-                case "simulate":
-                    result = await handleSimulate(sources, jobId, request.fromSequence, request.toSequence);
-                    break;
-                case "reexecute":
-                    result = await handleReexecute(sources, jobId, request.allowExternalCalls);
-                    break;
-            }
-            // Emit divergence events
-            if (result.divergences) {
-                for (const div of result.divergences) {
-                    this.emit({
-                        type: "replay.divergence.detected",
-                        jobId,
-                        reason: div.reason,
-                    });
-                }
-            }
-            // Collect artifacts
-            const artifacts = [];
-            if (sources.prompts)
-                artifacts.push(...Object.keys(sources.prompts));
-            if (sources.outputs)
-                artifacts.push(...Object.keys(sources.outputs));
-            // Emit completion event
-            this.emit({ type: "replay.completed", jobId });
-            return {
-                jobId,
-                reconstructedState: result.state,
-                timeline: result.timeline,
-                divergences: result.divergences || [],
-                artifacts,
-                replayedAt: new Date().toISOString(),
-            };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                jobId,
-                reconstructedState: {
-                    jobId,
-                    status: "error",
-                    checkpoint: sources.checkpoint || {},
-                    taskGraph: sources.taskGraph || { nodes: [], edges: [] },
-                    events: [],
-                    timestamp: new Date().toISOString(),
-                },
-                timeline: [],
-                divergences: [
-                    {
-                        sequence: 0,
-                        reason: errorMessage,
-                        severity: "error",
-                    },
-                ],
-                artifacts: [],
-                replayedAt: new Date().toISOString(),
-            };
-        }
-    }
-    /**
-     * Subscribe to replay events
-     */
-    onEvent(listener) {
-        this.eventListeners.push(listener);
-        return () => {
-            this.eventListeners = this.eventListeners.filter((l) => l !== listener);
-        };
-    }
-    emit(event) {
-        for (const listener of this.eventListeners) {
-            try {
-                listener(event);
-            }
-            catch {
-                // Best effort
-            }
-        }
-    }
-}
-// --- Factory -----------------------------------------------------------------
-export function createExperienceReplay() {
-    return new ExperienceReplay();
-}

package/packages/experience-replay/src/types.js

@@ -1,6 +0,0 @@
-/**
- * Experience Replay Types (RFC-0059)
- *
- * Interfaces for replaying prior runtime execution.
- */
-export {};