pi-harness-runtime 0.10.13 → 0.10.15

package/packages/learning-engine/src/engine.js

@@ -0,0 +1,294 @@
+/**
+ * Learning Engine (RFC-0058)
+ *
+ * Extracts reusable lessons from completed and failed jobs.
+ */
+import { randomUUID } from "node:crypto";
+import { containsSecret, calculateConfidence, } from "./types.js";
+// --- Pattern Extraction --------------------------------------------------------
+function extractProviderSuccessPatterns(metrics) {
+    const patterns = [];
+    // Group by task type
+    const byTaskType = new Map();
+    for (const metric of metrics) {
+        const existing = byTaskType.get(metric.taskType) || [];
+        existing.push(metric);
+        byTaskType.set(metric.taskType, existing);
+    }
+    // Find best provider per task type
+    for (const [taskType, taskMetrics] of byTaskType) {
+        if (taskMetrics.length < 2)
+            continue;
+        let bestProvider = taskMetrics[0];
+        let bestSuccessRate = 0;
+        for (const metric of taskMetrics) {
+            const total = metric.successCount + metric.failureCount;
+            if (total > 0) {
+                const successRate = metric.successCount / total;
+                if (successRate > bestSuccessRate) {
+                    bestSuccessRate = successRate;
+                    bestProvider = metric;
+                }
+            }
+        }
+        if (bestSuccessRate > 0.7) {
+            patterns.push({
+                type: "provider_success",
+                description: `Provider '${bestProvider.providerId}' performs best for '${taskType}' tasks (${(bestSuccessRate * 100).toFixed(0)}% success rate)`,
+                occurrenceCount: bestProvider.successCount + bestProvider.failureCount,
+                successIndicators: [
+                    `${bestSuccessRate > 0.8 ? "high" : "good"} success rate`,
+                ],
+                failureIndicators: [],
+            });
+        }
+    }
+    return patterns;
+}
+function extractRepairPatterns(repairs) {
+    const patterns = [];
+    // Group by repair type
+    const byType = new Map();
+    for (const repair of repairs) {
+        const existing = byType.get(repair.repairType) || [];
+        existing.push(repair);
+        byType.set(repair.repairType, existing);
+    }
+    for (const [repairType, typeRepairs] of byType) {
+        const successCount = typeRepairs.filter((r) => r.success).length;
+        const total = typeRepairs.length;
+        const successRate = total > 0 ? successCount / total : 0;
+        patterns.push({
+            type: "repair_strategy",
+            description: `Repair strategy '${repairType}' has ${(successRate * 100).toFixed(0)}% success rate`,
+            occurrenceCount: total,
+            successIndicators: successRate > 0.5 ? [`effective for ${successCount} tasks`] : [],
+            failureIndicators: successRate <= 0.5
+                ? [`ineffective for ${total - successCount} tasks`]
+                : [],
+        });
+    }
+    return patterns;
+}
+function extractFailurePatterns(events) {
+    const patterns = [];
+    const failureTypes = new Map();
+    // Count failure types
+    for (const event of events) {
+        if (event.type.includes("error") || event.type.includes("failure")) {
+            const existing = failureTypes.get(event.message) || 0;
+            failureTypes.set(event.message, existing + 1);
+        }
+    }
+    // Create patterns for common failures
+    for (const [message, count] of failureTypes) {
+        if (count >= 2) {
+            patterns.push({
+                type: "failure_cause",
+                description: message,
+                occurrenceCount: count,
+                successIndicators: [],
+                failureIndicators: [`occurred ${count} times`],
+            });
+        }
+    }
+    return patterns;
+}
+// --- Experience Creation ------------------------------------------------------
+function createExperience(pattern, scope, humanFeedback) {
+    // Calculate confidence
+    const factors = {
+        occurrenceCount: pattern.occurrenceCount,
+        consistentPositiveOutcomes: pattern.successIndicators.length * pattern.occurrenceCount,
+        humanApprovalCount: humanFeedback?.filter((f) => f.approved).length || 0,
+        frameworkCorroboration: 0,
+        contradictoryOutcomes: pattern.failureIndicators.length,
+        singleEventEvidence: pattern.occurrenceCount === 1,
+        staleRepository: false,
+        changedProjectRules: false,
+    };
+    const confidence = calculateConfidence(factors);
+    // Determine status
+    let status = "proposed";
+    if (humanFeedback?.some((f) => f.approved)) {
+        status = "approved";
+    }
+    return {
+        id: randomUUID(),
+        scope,
+        pattern: pattern.description,
+        recommendation: generateRecommendation(pattern),
+        confidence,
+        evidenceRefs: [],
+        status,
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+        tags: [pattern.type],
+        taskType: scope === "task_type" ? undefined : undefined,
+    };
+}
+function generateRecommendation(pattern) {
+    switch (pattern.type) {
+        case "provider_success":
+            return `Use the recommended provider for this task type to improve success rate.`;
+        case "repair_strategy":
+            return `Consider using this repair strategy for similar failures.`;
+        case "failure_cause":
+            return `Avoid or handle this failure condition proactively.`;
+        case "prompt_structure":
+            return `Use this prompt structure for better outcomes.`;
+        case "framework_command":
+            return `Use this command pattern for framework-specific tasks.`;
+        case "test_scenario":
+            return `Add this test scenario to catch regressions.`;
+        case "context_source":
+            return `Prioritize this context source for similar tasks.`;
+        default:
+            return pattern.description;
+    }
+}
+// --- Secret Filtering ---------------------------------------------------------
+function filterSecrets(data) {
+    const filtered = {};
+    for (const [key, value] of Object.entries(data)) {
+        // Skip secret-like keys
+        if (containsSecret(key)) {
+            filtered[key] = "[REDACTED]";
+            continue;
+        }
+        // Recursively filter nested objects
+        if (typeof value === "object" && value !== null) {
+            filtered[key] = filterSecrets(value);
+        }
+        else {
+            filtered[key] = value;
+        }
+    }
+    return filtered;
+}
+// --- Main Engine -------------------------------------------------------------
+export class LearningEngine {
+    framework;
+    repository;
+    constructor(options) {
+        this.framework = options?.framework;
+        this.repository = options?.repository;
+    }
+    /**
+     * Extract learned experiences from completed job
+     */
+    learn(request) {
+        const experiences = [];
+        // 1. Extract provider success patterns
+        const providerPatterns = extractProviderSuccessPatterns(request.providerMetrics);
+        for (const pattern of providerPatterns) {
+            const experience = createExperience(pattern, "task_type", request.humanFeedback);
+            experiences.push(experience);
+        }
+        // 2. Extract repair patterns
+        const repairPatterns = extractRepairPatterns(request.repairHistory);
+        for (const pattern of repairPatterns) {
+            const experience = createExperience(pattern, "task_type", request.humanFeedback);
+            experiences.push(experience);
+        }
+        // 3. Extract failure patterns
+        const filteredEvents = request.events
+            .map((e) => {
+            if (e.data) {
+                return { ...e, data: filterSecrets(e.data) };
+            }
+            return e;
+        })
+            .map((e) => ({
+            type: e.type,
+            message: e.message,
+            data: e.data,
+        }));
+        const failurePatterns = extractFailurePatterns(filteredEvents);
+        for (const pattern of failurePatterns) {
+            const experience = createExperience(pattern, "global", request.humanFeedback);
+            experiences.push(experience);
+        }
+        // 4. Process human feedback
+        if (request.humanFeedback) {
+            for (const feedback of request.humanFeedback) {
+                if (feedback.approved) {
+                    // Find matching experience or create new one
+                    const existing = experiences.find((e) => e.pattern.toLowerCase().includes(feedback.taskId.toLowerCase()));
+                    if (existing) {
+                        existing.status = "approved";
+                        existing.confidence = Math.min(100, existing.confidence + 20);
+                        existing.updatedAt = new Date().toISOString();
+                    }
+                }
+            }
+        }
+        // 5. Calculate summary
+        const summary = {
+            totalExtracted: experiences.length,
+            approved: experiences.filter((e) => e.status === "approved").length,
+            proposed: experiences.filter((e) => e.status === "proposed").length,
+            rejected: experiences.filter((e) => e.status === "rejected").length,
+        };
+        return {
+            jobId: request.jobId,
+            experiences,
+            summary,
+            extractedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Approve a learned experience
+     */
+    approve(experience) {
+        return {
+            ...experience,
+            status: "approved",
+            confidence: Math.min(100, experience.confidence + 10),
+            updatedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Reject a learned experience
+     */
+    reject(experience) {
+        return {
+            ...experience,
+            status: "rejected",
+            updatedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Export experience to OKF format
+     */
+    toOkfFormat(experience) {
+        const timestamp = new Date().toISOString();
+        const tags = experience.tags.join(", ");
+        return `---
+type: Engineering Lesson
+title: ${experience.pattern.substring(0, 60)}
+tags: [${tags}]
+timestamp: ${timestamp}
+authority: ${experience.status}
+scope: ${experience.scope}
+confidence: ${experience.confidence}
+---
+
+${experience.recommendation}
+
+## Evidence
+
+${experience.evidenceRefs.length > 0
+            ? experience.evidenceRefs.map((ref) => `- ${ref}`).join("\n")
+            : "No evidence references available."}
+
+## Pattern
+
+${experience.pattern}
+`;
+    }
+}
+// --- Factory -----------------------------------------------------------------
+export function createLearningEngine(options) {
+    return new LearningEngine(options);
+}

package/packages/learning-engine/src/index.js

@@ -0,0 +1,7 @@
+/**
+ * Learning Engine (RFC-0058)
+ *
+ * Re-exports all public types and classes.
+ */
+export { LearningEngine, createLearningEngine } from "./engine.js";
+export { containsSecret, redactSecrets, calculateConfidence, SECRET_PATTERNS, } from "./types.js";

package/packages/learning-engine/src/types.js

@@ -0,0 +1,63 @@
+/**
+ * Learning Engine Types (RFC-0058)
+ *
+ * Interfaces for learning from runtime execution.
+ */
+// --- Secret Detection ---------------------------------------------------------
+export const SECRET_PATTERNS = [
+    /api[_-]?key/i,
+    /password/i,
+    /secret/i,
+    /token/i,
+    /bearer/i,
+    /auth/i,
+    /credential/i,
+    /private[_-]?key/i,
+    /access[_-]?token/i,
+    /refresh[_-]?token/i,
+];
+export function containsSecret(value) {
+    return SECRET_PATTERNS.some((pattern) => pattern.test(value));
+}
+export function redactSecrets(data) {
+    const redacted = { ...data };
+    for (const [key, value] of Object.entries(redacted)) {
+        if (typeof value === "string" && containsSecret(key)) {
+            redacted[key] = "[REDACTED]";
+        }
+        else if (typeof value === "object" && value !== null) {
+            redacted[key] = redactSecrets(value);
+        }
+    }
+    return redacted;
+}
+export function calculateConfidence(factors) {
+    let confidence = 0;
+    // Occurrence boost (up to +30)
+    confidence += Math.min(30, factors.occurrenceCount * 5);
+    // Consistent positive outcomes (up to +25)
+    if (factors.occurrenceCount > 0) {
+        const consistencyRatio = factors.consistentPositiveOutcomes / factors.occurrenceCount;
+        confidence += consistencyRatio * 25;
+    }
+    // Human approval (up to +20)
+    confidence += Math.min(20, factors.humanApprovalCount * 10);
+    // Framework corroboration (up to +10)
+    confidence += Math.min(10, factors.frameworkCorroboration * 5);
+    // Penalties
+    // Contradictory outcomes (-15 per)
+    confidence -= Math.min(40, factors.contradictoryOutcomes * 15);
+    // Single event evidence (-10)
+    if (factors.singleEventEvidence) {
+        confidence -= 10;
+    }
+    // Stale repository (-5)
+    if (factors.staleRepository) {
+        confidence -= 5;
+    }
+    // Changed project rules (-20)
+    if (factors.changedProjectRules) {
+        confidence -= 20;
+    }
+    return Math.max(0, Math.min(100, Math.round(confidence)));
+}