progmune-runtime 2.1.5 → 3.2.0

package/dist/protocol-security-benchmark.js

@@ -0,0 +1,222 @@
+"use strict";
+/**
+ * P9.2b++: Protocol Security Benchmark — defining a new security category
+ *
+ * This is NOT just another CVE benchmark. It's the FIRST dataset
+ * specifically designed for protocol lifecycle vulnerabilities —
+ * missing states, missing edges, and illegal transitions that
+ * traditional SAST tools cannot see because they look at code
+ * patterns, not state machines.
+ *
+ * The benchmark measures one thing: can a system detect that a
+ * protocol lifecycle has been violated, regardless of how the
+ * code is written?
+ *
+ * Categories:
+ *   RESOURCE_LEAK:       Acquire→Use...no Release
+ *   AUTH_BYPASS:         Action without prior Authentication
+ *   TRANSACTION_VIOLATION: Begin→...no Commit/Rollback
+ *   USE_AFTER_FREE:      Free→Use (illegal edge)
+ *   SESSION_VIOLATION:   Logout→Access (illegal edge)
+ *   DOUBLE_FREE:         Free→Free (illegal self-loop)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildProtocolSecurityBenchmark = buildProtocolSecurityBenchmark;
+exports.runProtocolSecurityBenchmark = runProtocolSecurityBenchmark;
+exports.printProtocolSecurityReport = printProtocolSecurityReport;
+const gold_cve_1 = require("./gold-cve");
+// ═══════════════════════════════════════════════════════════════
+// Build the benchmark from all available verified cases
+// ═══════════════════════════════════════════════════════════════
+function buildProtocolSecurityBenchmark() {
+    const cases = [];
+    // Source 1: Curated 20 cases (manually verified against real CVE patterns)
+    const curated = (0, gold_cve_1.loadGoldDataset)().cases;
+    for (const c of curated) {
+        cases.push({
+            id: c.id,
+            category: c.category,
+            severity: c.severity,
+            broken: c.broken,
+            expected: c.expected,
+            source: "curated",
+            verified: true,
+            notes: c.notes,
+        });
+    }
+    // Source 2: Diff-based gold cases (from real git diffs)
+    try {
+        const fs = require("fs");
+        const path = require("path");
+        const seedPath = path.resolve(__dirname, "..", "benchmarks", "gold-seed.json");
+        if (fs.existsSync(seedPath)) {
+            const seed = JSON.parse(fs.readFileSync(seedPath, "utf-8"));
+            for (const c of seed) {
+                cases.push({
+                    id: `GOLD-${c.cve}`,
+                    category: c.category,
+                    severity: c.severity || "high",
+                    broken: c.before,
+                    expected: c.after,
+                    source: "git_diff",
+                    verified: true,
+                    notes: c.notes,
+                });
+            }
+        }
+    }
+    catch { }
+    // Source 3: Synthetic cases from topology factory (for stress testing)
+    // These are NOT verified — marked accordingly
+    const generateSyntheticCases = () => {
+        const { createProtocolForTopology, ALL_TOPOLOGIES } = require("./topology-factory");
+        const synthetic = [];
+        let id = 200;
+        for (const topo of ALL_TOPOLOGIES) {
+            const rules = createProtocolForTopology(topo);
+            if (rules.size < 2)
+                continue;
+            const entries = [...rules.entries()];
+            for (let v = 0; v < 3 && synthetic.length < 30; v++) {
+                const path = [];
+                const ss = new Set(["INIT", "IDLE"]);
+                const s = entries[Math.floor(Math.random() * entries.length)];
+                path.push(s[0]);
+                const r = s[1];
+                if (r.invalidate)
+                    r.invalidate.forEach((x) => ss.delete(x));
+                for (const x of r.post_states)
+                    ss.add(x);
+                for (let t = 0; t < 4; t++) {
+                    const cands = entries.filter(([, rr]) => rr.pre_states.every((x) => ss.has(x)));
+                    if (!cands.length)
+                        break;
+                    const [fn, nr] = cands[Math.floor(Math.random() * cands.length)];
+                    path.push(fn);
+                    if (nr.invalidate)
+                        nr.invalidate.forEach((x) => ss.delete(x));
+                    for (const x of nr.post_states)
+                        ss.add(x);
+                }
+                if (path.length < 3)
+                    continue;
+                synthetic.push({
+                    id: `SYN-${id++}`,
+                    category: "resource_leak",
+                    severity: "medium",
+                    broken: path.slice(0, -1),
+                    expected: path,
+                    source: "synthetic",
+                    verified: false,
+                });
+            }
+        }
+        return synthetic;
+    };
+    try {
+        cases.push(...generateSyntheticCases());
+    }
+    catch { }
+    // Build metadata
+    const byCategory = {};
+    const bySource = {};
+    let verified = 0;
+    for (const c of cases) {
+        byCategory[c.category] = (byCategory[c.category] || 0) + 1;
+        bySource[c.source] = (bySource[c.source] || 0) + 1;
+        if (c.verified)
+            verified++;
+    }
+    return {
+        name: "Protocol Lifecycle Security Benchmark",
+        version: "1.0.0",
+        cases,
+        metadata: { total: cases.length, verified, byCategory, bySource },
+    };
+}
+function runProtocolSecurityBenchmark() {
+    const bench = buildProtocolSecurityBenchmark();
+    const { inferStateMachine } = require("./experimental/state-inference");
+    const { detectStructuralViolations } = require("./protocol-invariants");
+    const CWE_TO_VIOLATION = {
+        resource_leak: "missing_release",
+        auth_bypass: "missing_prerequisite",
+        data_corruption: "missing_commit",
+        use_after_free: "illegal_transition",
+        race_condition: "missing_prerequisite",
+    };
+    const byCat = {};
+    let totalDetected = 0, totalMatched = 0;
+    let verifiedDetected = 0, verifiedTotal = 0;
+    for (const c of bench.cases) {
+        if (!byCat[c.category])
+            byCat[c.category] = { total: 0, verified: 0, detected: 0, matched: 0 };
+        byCat[c.category].total++;
+        if (c.verified) {
+            byCat[c.category].verified++;
+            verifiedTotal++;
+        }
+        const templateSM = inferStateMachine([c.expected]);
+        const brokenSM = inferStateMachine([c.broken]);
+        const violations = detectStructuralViolations(brokenSM, templateSM);
+        const violationTypes = violations.map((v) => v.violationSubtype);
+        const detected = violations.length > 0;
+        const expectedV = CWE_TO_VIOLATION[c.category] || "";
+        const matched = expectedV ? violationTypes.includes(expectedV) : false;
+        if (detected) {
+            totalDetected++;
+            byCat[c.category].detected++;
+            if (c.verified)
+                verifiedDetected++;
+        }
+        if (matched) {
+            totalMatched++;
+            byCat[c.category].matched++;
+        }
+    }
+    const total = bench.cases.length;
+    const overallRecall = total > 0 ? totalDetected / total : 0;
+    const overallPrecision = totalDetected > 0 ? totalMatched / totalDetected : 0;
+    const verifiedRecall = verifiedTotal > 0 ? verifiedDetected / verifiedTotal : 0;
+    const verifiedPrecision = verifiedTotal > 0 ? totalMatched / Math.max(1, totalDetected) : 0;
+    const parserGap = verifiedRecall - overallRecall;
+    const categoryBreakdown = {};
+    for (const [cat, s] of Object.entries(byCat)) {
+        categoryBreakdown[cat] = {
+            total: s.total, verified: s.verified,
+            detected: s.detected, matched: s.matched,
+            recall: s.total > 0 ? s.detected / s.total : 0,
+            precision: s.detected > 0 ? s.matched / s.detected : 0,
+        };
+    }
+    return {
+        benchmark: bench.name,
+        totalCases: total,
+        verifiedCases: verifiedTotal,
+        overallRecall, overallPrecision,
+        byCategory: categoryBreakdown,
+        verifiedOnly: { total: verifiedTotal, detected: verifiedDetected, recall: verifiedRecall, precision: verifiedPrecision },
+        parserNoiseGap: parserGap,
+        verdict: verifiedRecall > 0.85
+            ? "✅ PROTOCOL LIFECYCLE SECURITY — verified performance exceeds 85%"
+            : verifiedRecall > 0.7
+                ? "⚠️ PROMISING — verified recall >70%, needs more cases"
+                : "❌ INSUFFICIENT — core detector performance below threshold",
+    };
+}
+function printProtocolSecurityReport(report) {
+    console.log(`\n╔════════════════════════════════════════════════════╗`);
+    console.log(`║   ${report.benchmark}`);
+    console.log(`║   ${report.totalCases} cases (${report.verifiedCases} manually verified)`);
+    console.log(`╚════════════════════════════════════════════════════╝\n`);
+    console.log(`  Overall:     recall ${(report.overallRecall * 100).toFixed(0)}%  precision ${(report.overallPrecision * 100).toFixed(0)}%`);
+    console.log(`  Verified:    recall ${(report.verifiedOnly.recall * 100).toFixed(0)}%  precision ${(report.verifiedOnly.precision * 100).toFixed(0)}%`);
+    console.log(`  Parser gap:  ${(report.parserNoiseGap * 100).toFixed(0)}% (noise in CVE→sequence conversion)`);
+    console.log();
+    console.log(`  ── Per Category ──`);
+    for (const [cat, s] of Object.entries(report.byCategory)) {
+        const vMark = s.verified > 0 ? ` (${s.verified} verified)` : "";
+        console.log(`  ${cat.padEnd(22)} ${String(s.total).padStart(3)} cases${vMark.padEnd(14)} recall ${(s.recall * 100).toFixed(0).padStart(3)}%  precision ${(s.precision * 100).toFixed(0)}%`);
+    }
+    console.log(`\n  Verdict: ${report.verdict}\n`);
+}

package/dist/protocol-vulnerability.js

@@ -0,0 +1,257 @@
+"use strict";
+/**
+ * P9.0: Protocol Structure → Vulnerability Prediction
+ *
+ * THE decisive question: does protocol TOPOLOGY carry enough information
+ * to predict defects WITHOUT seeing function names, code, or comments?
+ *
+ * If yes: protocol structure contains security information.
+ * If no: structure is real but semantically empty.
+ *
+ * Experiment design:
+ *   1. For each of 10 topologies, generate CLEAN (complete) and BROKEN
+ *      (missing-step, wrong-order) sequences
+ *   2. Extract WL fingerprints from both
+ *   3. Train a simple nearest-centroid classifier on 7 topologies
+ *   4. Test zero-shot on 3 held-out topologies
+ *   5. Measure: precision, recall, F1 on held-out set
+ *
+ * Baseline: random guess = 50% (binary classification: clean/broken)
+ * Target: >70% F1 on held-out topologies
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildWLDefectDataset = buildWLDefectDataset;
+exports.runZeroShotDefectPrediction = runZeroShotDefectPrediction;
+exports.printDefectReport = printDefectReport;
+const topology_factory_1 = require("./topology-factory");
+const state_inference_1 = require("./experimental/state-inference");
+const wl_fingerprint_1 = require("./wl-fingerprint");
+/**
+ * Generate clean sequences (valid random walks through the state machine).
+ */
+function generateCleanSequences(rules, count) {
+    const entries = [...rules.entries()];
+    const sequences = [];
+    for (let i = 0; i < count; i++) {
+        const targetLen = 2 + Math.floor(Math.random() * 4); // 2-5 steps
+        const path = [];
+        const visited = new Set();
+        // Start from any rule
+        const startIdx = Math.floor(Math.random() * entries.length);
+        path.push(entries[startIdx][0]);
+        const stateSet = new Set(["INIT", "IDLE"]);
+        const rule = entries[startIdx][1];
+        if (rule.invalidate)
+            rule.invalidate.forEach((s) => stateSet.delete(s));
+        for (const s of rule.post_states)
+            stateSet.add(s);
+        let currentFn = path[0];
+        while (path.length < targetLen) {
+            const candidates = entries.filter(([fn, r]) => !visited.has(fn) && r.pre_states.every((s) => stateSet.has(s)));
+            if (candidates.length === 0)
+                break;
+            const [nextFn, nextRule] = candidates[Math.floor(Math.random() * candidates.length)];
+            path.push(nextFn);
+            visited.add(nextFn);
+            if (nextRule.invalidate)
+                nextRule.invalidate.forEach((s) => stateSet.delete(s));
+            for (const s of nextRule.post_states)
+                stateSet.add(s);
+            currentFn = nextFn;
+        }
+        if (path.length >= 2)
+            sequences.push(path);
+    }
+    return sequences;
+}
+/**
+ * Generate broken sequences by removing a step or inserting a wrong step.
+ *
+ * Broken variants:
+ *   1. REMOVE: drop the last step (incomplete lifecycle)
+ *   2. SKIP: skip a middle step (jump over a state)
+ */
+function generateBrokenSequences(rules, count) {
+    const clean = generateCleanSequences(rules, Math.ceil(count * 1.5));
+    const broken = [];
+    for (const seq of clean) {
+        if (broken.length >= count)
+            break;
+        // REMOVE last step (resource leak / missing close / missing commit)
+        if (seq.length >= 3) {
+            broken.push(seq.slice(0, -1));
+        }
+        // SKIP: remove a middle step if long enough
+        if (seq.length >= 4 && broken.length < count) {
+            const mid = 1 + Math.floor(Math.random() * (seq.length - 2));
+            broken.push([...seq.slice(0, mid), ...seq.slice(mid + 1)]);
+        }
+        // WRONG ORDER: swap two adjacent steps
+        if (seq.length >= 3 && broken.length < count) {
+            const swapIdx = Math.floor(Math.random() * (seq.length - 1));
+            const swapped = [...seq];
+            [swapped[swapIdx], swapped[swapIdx + 1]] = [swapped[swapIdx + 1], swapped[swapIdx]];
+            broken.push(swapped);
+        }
+    }
+    return broken.slice(0, count);
+}
+// ═══════════════════════════════════════════════════════════════
+// Dataset construction
+// ═══════════════════════════════════════════════════════════════
+/**
+ * Build a labeled dataset of WL fingerprints from protocol topologies.
+ *
+ * @param topologies  Which topologies to include
+ * @param perClass    Samples per class per topology
+ */
+function buildWLDefectDataset(topologies, perClass = 10) {
+    const samples = [];
+    for (const topo of topologies) {
+        const rules = (0, topology_factory_1.createProtocolForTopology)(topo);
+        if (rules.size === 0)
+            continue;
+        const cleanSeqs = generateCleanSequences(rules, perClass);
+        const brokenSeqs = generateBrokenSequences(rules, perClass);
+        for (const seq of cleanSeqs) {
+            const sm = (0, state_inference_1.inferStateMachine)(seq.map(fn => [fn]));
+            const wl = (0, wl_fingerprint_1.extractWLFingerprint)(sm, 3);
+            samples.push({ topology: topo, label: "clean", wl });
+        }
+        for (const seq of brokenSeqs) {
+            const sm = (0, state_inference_1.inferStateMachine)(seq.map(fn => [fn]));
+            const wl = (0, wl_fingerprint_1.extractWLFingerprint)(sm, 3);
+            samples.push({ topology: topo, label: "broken", wl });
+        }
+    }
+    return samples;
+}
+// ═══════════════════════════════════════════════════════════════
+// Nearest-centroid classifier (simplest possible, no overfitting risk)
+// ═══════════════════════════════════════════════════════════════
+function meanVector(vectors) {
+    const dims = vectors[0].length;
+    const mean = new Array(dims).fill(0);
+    for (const v of vectors) {
+        for (let i = 0; i < dims; i++)
+            mean[i] += v[i];
+    }
+    for (let i = 0; i < dims; i++)
+        mean[i] /= vectors.length;
+    return mean;
+}
+function cosineSim(a, b) {
+    let dot = 0, normA = 0, normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    if (normA === 0 && normB === 0)
+        return 1;
+    if (normA === 0 || normB === 0)
+        return 0;
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+/**
+ * Train on some topologies, test zero-shot on held-out ones.
+ *
+ * Uses nearest-centroid: compute the mean WL vector for "clean" and
+ * "broken" classes from training topologies, then classify test
+ * samples by which centroid is closer.
+ */
+function runZeroShotDefectPrediction(trainTopos, testTopos, samplesPerClass = 10) {
+    // Build training set
+    const trainSamples = buildWLDefectDataset(trainTopos, samplesPerClass);
+    const cleanVectors = trainSamples.filter(s => s.label === "clean").map(s => s.wl.vector);
+    const brokenVectors = trainSamples.filter(s => s.label === "broken").map(s => s.wl.vector);
+    if (cleanVectors.length === 0 || brokenVectors.length === 0) {
+        return {
+            trainTopos, testTopos, samples: samplesPerClass,
+            accuracy: 0, precision: 0, recall: 0, f1: 0,
+            perTopology: {}, confusion: { tp: 0, fp: 0, tn: 0, fn: 0 },
+        };
+    }
+    // Compute centroids
+    const cleanCentroid = meanVector(cleanVectors);
+    const brokenCentroid = meanVector(brokenVectors);
+    // Test on held-out topologies
+    const testSamples = buildWLDefectDataset(testTopos, samplesPerClass);
+    let tp = 0, fp = 0, tn = 0, fn = 0;
+    const perTopology = {};
+    for (const sample of testSamples) {
+        const distClean = 1 - cosineSim(sample.wl.vector, cleanCentroid);
+        const distBroken = 1 - cosineSim(sample.wl.vector, brokenCentroid);
+        const predicted = distClean < distBroken ? "clean" : "broken";
+        if (!perTopology[sample.topology]) {
+            perTopology[sample.topology] = { correct: 0, total: 0, tp: 0, fp: 0, tn: 0, fn: 0 };
+        }
+        perTopology[sample.topology].total++;
+        if (predicted === sample.label)
+            perTopology[sample.topology].correct++;
+        if (sample.label === "broken" && predicted === "broken") {
+            tp++;
+            perTopology[sample.topology].tp++;
+        }
+        else if (sample.label === "clean" && predicted === "broken") {
+            fp++;
+            perTopology[sample.topology].fp++;
+        }
+        else if (sample.label === "clean" && predicted === "clean") {
+            tn++;
+            perTopology[sample.topology].tn++;
+        }
+        else if (sample.label === "broken" && predicted === "clean") {
+            fn++;
+            perTopology[sample.topology].fn++;
+        }
+    }
+    const total = tp + fp + tn + fn;
+    const accuracy = total > 0 ? (tp + tn) / total : 0;
+    const precision = (tp + fp) > 0 ? tp / (tp + fp) : 0;
+    const recall = (tp + fn) > 0 ? tp / (tp + fn) : 0;
+    const f1 = (precision + recall) > 0 ? 2 * precision * recall / (precision + recall) : 0;
+    return {
+        trainTopos, testTopos, samples: samplesPerClass,
+        accuracy, precision, recall, f1,
+        perTopology: Object.fromEntries(Object.entries(perTopology).map(([k, v]) => [
+            k,
+            {
+                accuracy: v.total > 0 ? v.correct / v.total : 0,
+                f1: (v.tp + v.fp + v.fn) > 0
+                    ? 2 * (v.tp / Math.max(1, v.tp + v.fp)) * (v.tp / Math.max(1, v.tp + v.fn))
+                        / ((v.tp / Math.max(1, v.tp + v.fp)) + (v.tp / Math.max(1, v.tp + v.fn)))
+                    : 0,
+            },
+        ])),
+        confusion: { tp, fp, tn, fn },
+    };
+}
+function printDefectReport(report) {
+    console.log("\n╔════════════════════════════════════════════════════╗");
+    console.log("║   P9.0 Protocol → Defect Prediction (Zero-Shot)    ║");
+    console.log("╚════════════════════════════════════════════════════╝\n");
+    console.log(`  Train: ${report.trainTopos.join(", ")}`);
+    console.log(`  Test:  ${report.testTopos.join(", ")}`);
+    console.log(`  Samples per topology: ${report.samples}`);
+    console.log();
+    console.log(`  Accuracy:  ${(report.accuracy * 100).toFixed(0)}%`);
+    console.log(`  Precision: ${(report.precision * 100).toFixed(0)}% (broken class)`);
+    console.log(`  Recall:    ${(report.recall * 100).toFixed(0)}% (broken class)`);
+    console.log(`  F1:        ${(report.f1 * 100).toFixed(0)}%`);
+    console.log();
+    console.log(`  Confusion: TP=${report.confusion.tp} FP=${report.confusion.fp} TN=${report.confusion.tn} FN=${report.confusion.fn}`);
+    console.log();
+    console.log(`  Per-topology:`);
+    for (const [topo, stats] of Object.entries(report.perTopology)) {
+        console.log(`    ${topo.padEnd(18)} acc=${(stats.accuracy * 100).toFixed(0)}%  f1=${(stats.f1 * 100).toFixed(0)}%`);
+    }
+    const baseline = 0.5; // random chance
+    const improvement = report.f1 - baseline;
+    const verdict = report.f1 > 0.7 ? "✅ PROTOCOL STRUCTURE PREDICTS DEFECTS"
+        : report.f1 > baseline + 0.1 ? "⚠️  WEAK SIGNAL — above random, below useful"
+            : "❌ No predictive power — structure alone insufficient";
+    console.log(`\n  Baseline (random): 50%`);
+    console.log(`  Improvement:       ${improvement > 0 ? '+' : ''}${(improvement * 100).toFixed(0)}%`);
+    console.log(`  Verdict:           ${verdict}\n`);
+}

package/dist/protocol-vulnerability.test.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * P9.0: Protocol Structure → Defect Prediction Test
+ *
+ * THE decisive question: can WL fingerprints of protocol state machines
+ * predict whether a sequence has a defect (missing step, wrong order)
+ * WITHOUT seeing function names, code, or comments?
+ */
+const vitest_1 = require("vitest");
+const protocol_vulnerability_1 = require("./protocol-vulnerability");
+const topology_factory_1 = require("./topology-factory");
+(0, vitest_1.describe)("P9.0 Protocol → Vulnerability Prediction", () => {
+    (0, vitest_1.it)("builds WL defect dataset from protocol topologies", () => {
+        const samples = (0, protocol_vulnerability_1.buildWLDefectDataset)(["linear", "star", "loop"], 5);
+        (0, vitest_1.expect)(samples.length).toBeGreaterThanOrEqual(20);
+        const clean = samples.filter(s => s.label === "clean").length;
+        const broken = samples.filter(s => s.label === "broken").length;
+        (0, vitest_1.expect)(clean).toBeGreaterThan(0);
+        (0, vitest_1.expect)(broken).toBeGreaterThan(0);
+        // All samples should have valid WL fingerprints
+        for (const s of samples) {
+            (0, vitest_1.expect)(s.wl.vector.length).toBe(256);
+            (0, vitest_1.expect)(s.wl.uniqueLabels).toBeGreaterThan(0);
+        }
+    });
+    (0, vitest_1.it)("ZERO-SHOT: train on 7 topologies, test on 3 held-out", () => {
+        // Leave out 3 topologies for zero-shot testing
+        const trainTopos = topology_factory_1.ALL_TOPOLOGIES.slice(0, 7);
+        const testTopos = topology_factory_1.ALL_TOPOLOGIES.slice(7);
+        const report = (0, protocol_vulnerability_1.runZeroShotDefectPrediction)(trainTopos, testTopos, 10);
+        (0, protocol_vulnerability_1.printDefectReport)(report);
+        // Must be better than random (50%)
+        (0, vitest_1.expect)(report.accuracy).toBeGreaterThan(0.5);
+    });
+    (0, vitest_1.it)("FULL CROSS-VALIDATION: average over 3 random splits", () => {
+        const shuffled = [...topology_factory_1.ALL_TOPOLOGIES].sort(() => Math.random() - 0.5);
+        const splits = [
+            { train: shuffled.slice(0, 7), test: shuffled.slice(7) },
+            { train: shuffled.slice(3), test: shuffled.slice(0, 3) },
+            { train: [...shuffled.slice(0, 4), ...shuffled.slice(7)], test: shuffled.slice(4, 7) },
+        ];
+        let totalF1 = 0;
+        const results = [];
+        for (const { train, test } of splits) {
+            const report = (0, protocol_vulnerability_1.runZeroShotDefectPrediction)(train, test, 8);
+            totalF1 += report.f1;
+            results.push({ testTopos: test, accuracy: report.accuracy, f1: report.f1 });
+        }
+        const avgF1 = totalF1 / splits.length;
+        console.log(`\n  ═══ 3-FOLD CROSS-VALIDATION ═══`);
+        for (const r of results) {
+            console.log(`  Test: ${r.testTopos.join(",").padEnd(35)} acc=${(r.accuracy * 100).toFixed(0)}%  f1=${(r.f1 * 100).toFixed(0)}%`);
+        }
+        console.log(`  Average F1: ${(avgF1 * 100).toFixed(0)}%`);
+        console.log(`  Baseline (random): 50%`);
+        // Average F1 should exceed random baseline
+        (0, vitest_1.expect)(avgF1).toBeGreaterThan(0.5);
+    });
+});

package/dist/python-benchmark.js

@@ -0,0 +1,120 @@
+"use strict";
+/**
+ * Phase 2: Python Detection Benchmark Runner
+ *
+ * Runs protocol detection + safeguard detection on Python projects,
+ * compares against labeled data, reports Precision/Recall/F1.
+ *
+ * Usage: npx ts-node src/python-benchmark.ts [projectPath]
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runPythonBenchmark = runPythonBenchmark;
+exports.loadLabels = loadLabels;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const sequence_extractor_1 = require("./sequence-extractor");
+const protocol_detector_1 = require("./protocol-detector");
+const protocol_detector_2 = require("./protocol-detector");
+const resource_detector_1 = require("./resource-detector");
+function loadLabels(labelsPath) {
+    if (!fs.existsSync(labelsPath))
+        return [];
+    return JSON.parse(fs.readFileSync(labelsPath, "utf-8"));
+}
+function runPythonBenchmark(projectPath, labelsPath) {
+    const sequences = (0, sequence_extractor_1.extractSequences)(projectPath, { maxBodyLines: 200 });
+    const labels = loadLabels(labelsPath);
+    // Build label lookup
+    const labelMap = new Map();
+    for (const l of labels) {
+        labelMap.set(`${l.filePath}:${l.functionName}`, l);
+    }
+    let tp = 0, fp = 0, fn = 0, tn = 0;
+    for (const seq of sequences) {
+        const key = `${seq.filePath}:${seq.functionName}`;
+        const label = labelMap.get(key);
+        const hasLabelViolation = label?.hasViolation ?? false;
+        // Run detection
+        const protoViolations = (0, protocol_detector_1.detectProtocolViolations)(seq.calls);
+        const safeViolations = (0, protocol_detector_2.detectSafeguardViolations)(seq.calls, seq.functionName, "python");
+        const resResult = (0, resource_detector_1.validateResourceLifecycle)(seq.calls);
+        const resViolations = resResult.violations || [];
+        const totalViolations = protoViolations.length + safeViolations.length + resViolations.length;
+        const detected = totalViolations > 0;
+        if (detected && hasLabelViolation)
+            tp++;
+        else if (detected && !hasLabelViolation)
+            fp++;
+        else if (!detected && hasLabelViolation)
+            fn++;
+        else
+            tn++;
+    }
+    const precision = tp + fp > 0 ? tp / (tp + fp) : 0;
+    const recall = tp + fn > 0 ? tp / (tp + fn) : 0;
+    const f1 = precision + recall > 0 ? 2 * (precision * recall) / (precision + recall) : 0;
+    // Check for unlabeled functions → treat as true negatives
+    const unlabeledCount = sequences.length - labels.length;
+    return {
+        precision: Math.round(precision * 1000) / 10,
+        recall: Math.round(recall * 1000) / 10,
+        f1: Math.round(f1 * 1000) / 10,
+        tp, fp, fn,
+        totalFunctions: sequences.length,
+        totalViolations: tp + fp,
+    };
+}
+// ── Main ──
+if (require.main === module) {
+    const projectPath = process.argv[2] || path.join(__dirname, "..", "test-python-protocol");
+    const labelsPath = process.argv[3] || path.join(__dirname, "..", "benchmarks", "python-labels.json");
+    console.log(`Python Benchmark Runner`);
+    console.log(`Project: ${projectPath}`);
+    console.log(`Labels:  ${labelsPath}`);
+    console.log("");
+    if (!fs.existsSync(projectPath)) {
+        console.error(`Project path not found: ${projectPath}`);
+        process.exit(1);
+    }
+    const result = runPythonBenchmark(projectPath, labelsPath);
+    console.log(`Functions:       ${result.totalFunctions}`);
+    console.log(`Detected:        ${result.totalViolations}`);
+    console.log(`TP: ${result.tp}  FP: ${result.fp}  FN: ${result.fn}`);
+    console.log(`Precision:       ${result.precision}%`);
+    console.log(`Recall:          ${result.recall}%`);
+    console.log(`F1:              ${result.f1}%`);
+    process.exit(0);
+}