progmune-runtime 2.1.6 → 3.2.0

package/dist/cve-benchmark.js

@@ -0,0 +1,180 @@
+"use strict";
+/**
+ * P9.2: Real CVE Validation — Protocol Violation → Security Bug
+ *
+ * THE decisive question: can state-machine invariant violations
+ * detect REAL security vulnerabilities, not just synthetic damage?
+ *
+ * Uses the 20 real-world defect cases (mapped to CVEs) from
+ * realworld-benchmark.ts as a ground-truth dataset.
+ *
+ * For each CVE case:
+ *   1. Build template SM from the expected (correct) sequence
+ *   2. Build test SM from the broken (vulnerable) sequence
+ *   3. Run detectStructuralViolations(testSM, templateSM)
+ *   4. Check if the detected violation type matches the CVE category
+ *
+ * Success criteria:
+ *   Recall > 70%   (most real vulnerabilities detected)
+ *   Precision > 50% (few false positives)
+ *   Explainability > 90% (violation description maps to CVE type)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAnyCVEBenchmark = runAnyCVEBenchmark;
+exports.runCVEBenchmark = runCVEBenchmark;
+exports.printCVEReport = printCVEReport;
+const realworld_benchmark_1 = require("./realworld-benchmark");
+const state_inference_1 = require("./experimental/state-inference");
+const protocol_invariants_1 = require("./protocol-invariants");
+// ═══════════════════════════════════════════════════════════════
+// CVE Category → Expected Violation Subtype mapping
+// ═══════════════════════════════════════════════════════════════
+const CWE_TO_VIOLATION = {
+    resource_leak: "missing_release",
+    auth_bypass: "missing_prerequisite",
+    data_corruption: "missing_commit",
+    use_after_free: "missing_release",
+    race_condition: "missing_prerequisite",
+};
+/**
+ * Run the CVE benchmark against ANY array of CVE cases.
+ * The benchmark is named after the data source: "20-case" or "100-case".
+ */
+function runAnyCVEBenchmark(cases) {
+    return runCVEBenchmarkInternal(cases);
+}
+/** Run the benchmark against the curated 20-case set. */
+function runCVEBenchmark() {
+    const curated = realworld_benchmark_1.REAL_WORLD_DEFECTS.map((d) => ({
+        id: d.id,
+        cve: d.source?.replace(" pattern", "") || "",
+        title: d.title,
+        description: d.description,
+        severity: d.severity,
+        cwe: "",
+        category: d.category,
+        broken: d.broken,
+        expected: d.expected,
+        project: "curated",
+        affectedVersions: [],
+        source: "curated",
+    }));
+    return runCVEBenchmarkInternal(curated);
+}
+function runCVEBenchmarkInternal(defects) {
+    const results = {
+        total: defects.length,
+        detected: 0,
+        categoryMatched: 0,
+        recall: 0,
+        precision: 0,
+        bySeverity: {},
+        byCategory: {},
+        results: [],
+    };
+    for (const defect of defects) {
+        // Build template SM from the expected (correct) sequence
+        const templateSM = (0, state_inference_1.inferStateMachine)([defect.expected]);
+        // Build test SM from the broken (vulnerable) sequence
+        const brokenSM = (0, state_inference_1.inferStateMachine)([defect.broken]);
+        // Run structural violation detection
+        const violations = (0, protocol_invariants_1.detectStructuralViolations)(brokenSM, templateSM);
+        // Map violations to CVE categories
+        const violationTypes = violations.map(v => v.violationSubtype);
+        const expectedViolation = CWE_TO_VIOLATION[defect.category || "other"];
+        const categoryMatch = expectedViolation
+            ? violationTypes.includes(expectedViolation)
+            : false;
+        const detected = violations.length > 0;
+        // Track by severity
+        const sev = defect.severity || "medium";
+        if (!results.bySeverity[sev]) {
+            results.bySeverity[sev] = { total: 0, detected: 0 };
+        }
+        results.bySeverity[sev].total++;
+        if (detected)
+            results.bySeverity[sev].detected++;
+        // Track by category
+        const cat = defect.category || "other";
+        if (!results.byCategory[cat]) {
+            results.byCategory[cat] = { total: 0, detected: 0 };
+        }
+        results.byCategory[cat].total++;
+        if (detected)
+            results.byCategory[cat].detected++;
+        if (detected)
+            results.detected++;
+        if (categoryMatch)
+            results.categoryMatched++;
+        results.results.push({
+            defectId: defect.id,
+            title: defect.title,
+            severity: defect.severity,
+            cweCategory: defect.category || "other",
+            detected,
+            violationCount: violations.length,
+            violationTypes,
+            categoryMatch,
+            templateStates: templateSM.stateCount,
+            brokenStates: brokenSM.stateCount,
+            details: violations.map(v => v.description),
+        });
+    }
+    // Compute metrics
+    results.recall = results.total > 0 ? results.detected / results.total : 0;
+    // Precision: of detected, how many have correct category?
+    results.precision = results.detected > 0
+        ? results.categoryMatched / results.detected
+        : 0;
+    return results;
+}
+function printCVEReport(report) {
+    console.log("\n╔════════════════════════════════════════════════════╗");
+    console.log("║   P9.2 Real CVE Validation                          ║");
+    console.log("║   Protocol Violation → Security Bug?                ║");
+    console.log("╚════════════════════════════════════════════════════╝\n");
+    console.log(`  Total CVEs:          ${report.total}`);
+    console.log(`  Detected:            ${report.detected} (${(report.recall * 100).toFixed(0)}%)`);
+    console.log(`  Category matched:    ${report.categoryMatched} (${(report.precision * 100).toFixed(0)}%)`);
+    console.log();
+    console.log(`  ── By Severity ──`);
+    for (const [sev, stats] of Object.entries(report.bySeverity)) {
+        const rate = stats.total > 0 ? (stats.detected / stats.total * 100).toFixed(0) : "N/A";
+        const icon = rate === "100" ? "✅" : rate === "0" ? "❌" : "⚠️";
+        console.log(`    ${sev.padEnd(10)} ${stats.detected}/${stats.total} (${rate}%) ${icon}`);
+    }
+    console.log();
+    console.log(`  ── By Category ──`);
+    for (const [cat, stats] of Object.entries(report.byCategory)) {
+        const rate = stats.total > 0 ? (stats.detected / stats.total * 100).toFixed(0) : "N/A";
+        const icon = rate === "100" ? "✅" : rate === "0" ? "❌" : "⚠️";
+        console.log(`    ${cat.padEnd(18)} ${stats.detected}/${stats.total} (${rate}%) ${icon}`);
+    }
+    console.log();
+    console.log(`  ── Per-CVE Results ──`);
+    for (const r of report.results) {
+        const icon = r.detected
+            ? (r.categoryMatch ? "✅" : "⚠️")
+            : "❌";
+        console.log(`    ${icon} ${r.defectId} ${r.severity.padEnd(8)} tpl=${r.templateStates} broken=${r.brokenStates} violations=${r.violationCount} types=${r.violationTypes.join(",") || "none"}`);
+        if (r.details.length > 0) {
+            for (const d of r.details.slice(0, 2)) {
+                console.log(`       ${d.slice(0, 100)}`);
+            }
+        }
+    }
+    console.log();
+    console.log(`  ── Verdict ──`);
+    const recallOk = report.recall > 0.7;
+    const precisionOk = report.precision > 0.5;
+    if (recallOk && precisionOk) {
+        console.log(`  ✅ COMMERCIAL VIABILITY: Protocol violations → Real CVE detection works.`);
+    }
+    else if (recallOk || precisionOk) {
+        console.log(`  ⚠️  PARTIAL: One metric passes, one fails. Need investigation.`);
+    }
+    else {
+        console.log(`  ❌ GAP EXISTS: Protocol structure alone insufficient for real CVE detection.`);
+    }
+    console.log();
+}

package/dist/cve-benchmark.test.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const cve_benchmark_1 = require("./cve-benchmark");
+(0, vitest_1.describe)("P9.2 Real CVE Validation", () => {
+    (0, vitest_1.it)("detects protocol violations in real-world CVE cases", () => {
+        const report = (0, cve_benchmark_1.runCVEBenchmark)();
+        (0, cve_benchmark_1.printCVEReport)(report);
+        (0, vitest_1.expect)(report.total).toBe(20);
+        (0, vitest_1.expect)(report.detected).toBeGreaterThan(0);
+        (0, vitest_1.expect)(report.recall).toBeGreaterThanOrEqual(0);
+        (0, vitest_1.expect)(report.precision).toBeGreaterThanOrEqual(0);
+    });
+    (0, vitest_1.it)("resource_leak CVEs detected as missing_release", () => {
+        const report = (0, cve_benchmark_1.runCVEBenchmark)();
+        const rl = report.byCategory["resource_leak"];
+        (0, vitest_1.expect)(rl).toBeTruthy();
+        // Most resource leaks should be detected
+        (0, vitest_1.expect)(rl.detected).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)("auth_bypass CVEs detected as missing_prerequisite", () => {
+        const report = (0, cve_benchmark_1.runCVEBenchmark)();
+        const ab = report.byCategory["auth_bypass"];
+        (0, vitest_1.expect)(ab).toBeTruthy();
+        // At least some auth bypasses should be detected
+        (0, vitest_1.expect)(ab.detected).toBeGreaterThanOrEqual(0);
+    });
+});

package/dist/cve-collector.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchCVEsFromNVD = fetchCVEsFromNVD;
+const NVD_API_BASE = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
+async function fetchCVEsFromNVD(options = {}) {
+    const { limit = 1000, cwe, severity } = options;
+    const params = new URLSearchParams({
+        resultsPerPage: Math.min(limit, 2000).toString(),
+    });
+    if (cwe)
+        params.append('cweId', cwe);
+    if (severity)
+        params.append('cvssV3Severity', severity);
+    const url = `${NVD_API_BASE}?${params.toString()}`;
+    const response = await fetch(url);
+    if (!response.ok)
+        throw new Error(`NVD API error: ${response.status} ${response.statusText}`);
+    const data = (await response.json());
+    const cves = [];
+    for (const vuln of data.vulnerabilities || []) {
+        const cve = vuln.cve;
+        const id = cve.id;
+        const description = cve.descriptions?.find((d) => d.lang === 'en')?.value || '';
+        const severityScore = cve.metrics?.cvssMetricV3?.[0]?.cvssData?.baseSeverity || 'medium';
+        const cweId = cve.weaknesses?.[0]?.description?.[0]?.value || '';
+        const project = extractProjectName(cve);
+        const affectedVersions = cve.affects?.vendor?.vendor_data?.[0]?.product?.product_data?.[0]?.version?.version_data
+            ?.map((v) => v.version_value) || [];
+        const { broken, expected } = inferFromDescription(description);
+        cves.push({
+            id,
+            cve: id,
+            title: cve.descriptions?.find((d) => d.lang === 'en')?.value?.slice(0, 80) || id,
+            description,
+            severity: severityScore.toLowerCase(),
+            cwe: cweId,
+            project,
+            affectedVersions,
+            broken,
+            expected,
+            source: 'nvd',
+        });
+    }
+    return cves.slice(0, limit);
+}
+function inferFromDescription(desc) {
+    const verbs = ['open', 'close', 'read', 'write', 'connect', 'disconnect', 'auth', 'verify', 'commit', 'rollback', 'malloc', 'free'];
+    const found = verbs.filter(v => desc.toLowerCase().includes(v));
+    if (found.length > 0) {
+        return { broken: found.slice(0, -1), expected: found };
+    }
+    return { broken: ['unknown'], expected: ['unknown'] };
+}
+function extractProjectName(cve) {
+    const vendorData = cve.affects?.vendor?.vendor_data;
+    if (vendorData && vendorData.length > 0) {
+        const product = vendorData[0]?.product?.product_data?.[0]?.product_name;
+        if (product)
+            return product;
+    }
+    const refs = cve.references?.reference_data || [];
+    for (const ref of refs) {
+        const url = ref.url;
+        if (url.includes('github.com')) {
+            const parts = url.split('/');
+            const idx = parts.indexOf('github.com');
+            if (idx !== -1 && parts.length > idx + 2) {
+                return parts[idx + 1] + '/' + parts[idx + 2];
+            }
+        }
+    }
+    return 'unknown';
+}

package/dist/data-quality.js

@@ -0,0 +1,141 @@
+"use strict";
+/**
+ * P3.1: Data Quality Layer
+ *
+ * Separates raw acceptance from verified correctness.
+ *
+ * Risk: accepted ≠ correct, rejected ≠ wrong.
+ * A user accepting a fast-but-leaky repair produces toxic training data
+ * for the Reward Model. A user rejecting a correct-but-slow repair
+ * deprives the system of a positive signal.
+ *
+ * RepairOutcome adds three independent verification signals:
+ *   1. executionSucceeded  — did it actually run without errors?
+ *   2. postValidationPassed — did the SSG validator accept the final state?
+ *   3. regressionTestsPassed — did existing tests still pass?
+ *
+ * Future P4 Reward Model trains on verified outcomes, not raw acceptance.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeQualityScore = computeQualityScore;
+exports.computeRewardSignal = computeRewardSignal;
+exports.generateQualityReport = generateQualityReport;
+exports.printQualityReport = printQualityReport;
+// ═══════════════════════════════════════════════════════════════
+// Quality Scoring
+// ═══════════════════════════════════════════════════════════════
+/** Default weights for quality score computation. */
+const DEFAULT_QUALITY_WEIGHTS = {
+    execution: 0.4, // did it run?
+    validation: 0.4, // did the SSG accept the result?
+    regression: 0.2, // did existing tests pass?
+};
+/**
+ * Compute a quality score from a RepairOutcome.
+ *
+ * If verification signals are missing, the score degrades gracefully:
+ *   - No execution data   → weight redistributed to validation
+ *   - No validation data  → weight redistributed to execution
+ *   - Neither             → 0.5 (neutral prior)
+ */
+function computeQualityScore(outcome, weights) {
+    const w = { ...DEFAULT_QUALITY_WEIGHTS, ...weights };
+    let score = 0;
+    let totalWeight = 0;
+    if (outcome.executionSucceeded !== undefined) {
+        score += (outcome.executionSucceeded ? 1 : 0) * w.execution;
+        totalWeight += w.execution;
+    }
+    if (outcome.postValidationPassed !== undefined) {
+        score += (outcome.postValidationPassed ? 1 : 0) * w.validation;
+        totalWeight += w.validation;
+    }
+    if (outcome.regressionTestsPassed !== undefined) {
+        score += (outcome.regressionTestsPassed ? 1 : 0) * w.regression;
+        totalWeight += w.regression;
+    }
+    return totalWeight > 0 ? score / totalWeight : 0.5;
+}
+// ═══════════════════════════════════════════════════════════════
+// Quality-aware reward signal
+// ═══════════════════════════════════════════════════════════════
+/**
+ * Compute a quality-aware reward for a repair.
+ *
+ *   reward = accepted * 0.4 + validationPassed * 0.4 + executionSucceeded * 0.2
+ *
+ * This is the training signal for P4 Reward Model.
+ * Raw acceptance alone is NOT sufficient — a fast-but-leaky repair
+ * that users love should NOT get a high reward.
+ */
+function computeRewardSignal(outcome) {
+    const accepted = outcome.accepted ? 1 : 0;
+    const validation = outcome.postValidationPassed ? 1 : 0;
+    const execution = outcome.executionSucceeded ? 1 : 0;
+    const regression = outcome.regressionTestsPassed ? 1 : 0;
+    // If validation or regression data is available, it dominates
+    if (outcome.postValidationPassed !== undefined || outcome.regressionTestsPassed !== undefined) {
+        return (accepted * 0.4 +
+            validation * 0.4 +
+            (execution * 0.1 + regression * 0.1));
+    }
+    // Fallback: execution + acceptance only
+    if (outcome.executionSucceeded !== undefined) {
+        return accepted * 0.5 + execution * 0.5;
+    }
+    // Minimal: no verification data → neutral prior (avoid overfitting to raw acceptance)
+    return 0.5;
+}
+/**
+ * Generate a quality report from telemetry data.
+ * Identifies contradictory outcomes where acceptance disagrees with execution.
+ */
+function generateQualityReport(outcomes) {
+    const total = outcomes.length;
+    if (total === 0) {
+        return {
+            totalOutcomes: 0, rawAcceptanceRate: 0, executionSuccessRate: 0,
+            validationPassRate: 0, regressionPassRate: 0, qualityScoreAvg: 0,
+            contradictoryOutcomes: 0,
+        };
+    }
+    const accepted = outcomes.filter(o => o.accepted).length;
+    const execOk = outcomes.filter(o => o.executionSucceeded === true).length;
+    const execTotal = outcomes.filter(o => o.executionSucceeded !== undefined).length;
+    const valOk = outcomes.filter(o => o.postValidationPassed === true).length;
+    const valTotal = outcomes.filter(o => o.postValidationPassed !== undefined).length;
+    const regOk = outcomes.filter(o => o.regressionTestsPassed === true).length;
+    const regTotal = outcomes.filter(o => o.regressionTestsPassed !== undefined).length;
+    const qualityScores = outcomes.map(o => computeQualityScore(o));
+    const qualityAvg = qualityScores.reduce((s, v) => s + v, 0) / total;
+    // Contradictory: accepted but execution failed, OR rejected but execution succeeded
+    const contradictory = outcomes.filter(o => o.executionSucceeded !== undefined &&
+        o.accepted !== o.executionSucceeded).length;
+    return {
+        totalOutcomes: total,
+        rawAcceptanceRate: total > 0 ? accepted / total : 0,
+        executionSuccessRate: execTotal > 0 ? execOk / execTotal : 0,
+        validationPassRate: valTotal > 0 ? valOk / valTotal : 0,
+        regressionPassRate: regTotal > 0 ? regOk / regTotal : 0,
+        qualityScoreAvg: qualityAvg,
+        contradictoryOutcomes: contradictory,
+    };
+}
+function printQualityReport(report) {
+    console.log("\n╔══════════════════════════════════════════╗");
+    console.log("║   Data Quality Report                    ║");
+    console.log("╚══════════════════════════════════════════╝\n");
+    console.log(`Total Outcomes:       ${report.totalOutcomes}`);
+    console.log(`Raw Acceptance:       ${(report.rawAcceptanceRate * 100).toFixed(1)}%`);
+    console.log(`Execution Success:    ${(report.executionSuccessRate * 100).toFixed(1)}%`);
+    console.log(`Validation Pass:      ${(report.validationPassRate * 100).toFixed(1)}%`);
+    console.log(`Regression Pass:      ${(report.regressionPassRate * 100).toFixed(1)}%`);
+    console.log(`Avg Quality Score:    ${(report.qualityScoreAvg * 100).toFixed(1)}%`);
+    console.log();
+    if (report.contradictoryOutcomes > 0) {
+        const pct = (report.contradictoryOutcomes / report.totalOutcomes * 100).toFixed(1);
+        console.log(`⚠️  Contradictory:      ${report.contradictoryOutcomes} (${pct}%)`);
+        console.log("   (accepted ≠ execution result — potential data poison)");
+    }
+    console.log();
+}