veto-sdk 1.0.0

package/dist/benchmark/runner.js

@@ -0,0 +1,279 @@
+/**
+ * Benchmark runner for evaluating Veto kernel performance.
+ *
+ * @module benchmark/runner
+ */
+import { writeFileSync } from 'node:fs';
+import { KernelClient } from '../kernel/client.js';
+import { createLogger } from '../utils/logger.js';
+import { loadBenchmarkSamples } from './loader.js';
+import { calculateConfusionMatrix, calculateClassificationMetrics, calculateLatencyStats, calculateCategoryMetrics, } from './metrics.js';
+/**
+ * Run a benchmark evaluation.
+ */
+export async function runBenchmark(options) {
+    const { config, onProgress } = options;
+    const logger = options.logger ?? createLogger('info');
+    const startTime = new Date();
+    logger.info('Starting benchmark', {
+        datasetPath: config.datasetPath,
+        maxSamples: config.maxSamples || 'all',
+        model: config.kernel.model,
+    });
+    // Load samples
+    logger.info('Loading benchmark samples...');
+    const samples = await loadBenchmarkSamples(config.datasetPath, config.maxSamples, config.shuffle, config.seed);
+    logger.info(`Loaded ${samples.length} samples`);
+    // Create kernel client
+    const kernelConfig = {
+        baseUrl: config.kernel.baseUrl,
+        model: config.kernel.model,
+        temperature: config.kernel.temperature,
+        maxTokens: config.kernel.maxTokens,
+        timeout: config.kernel.timeout,
+    };
+    const kernelClient = new KernelClient({
+        config: kernelConfig,
+        logger,
+    });
+    // Run evaluations
+    const results = [];
+    const startTimes = [];
+    let errorCount = 0;
+    for (let i = 0; i < samples.length; i++) {
+        const sample = samples[i];
+        const sampleStart = Date.now();
+        startTimes.push(sampleStart);
+        let result;
+        try {
+            const toolCall = {
+                tool: sample.tool,
+                arguments: sample.arguments,
+            };
+            const response = await kernelClient.evaluate(toolCall, sample.rules);
+            result = {
+                sample,
+                actualDecision: response.decision,
+                actualPassWeight: response.pass_weight,
+                actualBlockWeight: response.block_weight,
+                reasoning: response.reasoning,
+                matchedRules: response.matched_rules,
+                correct: response.decision === sample.expectedDecision,
+                latencyMs: Date.now() - sampleStart,
+            };
+        }
+        catch (error) {
+            errorCount++;
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            // Log first few errors for debugging
+            if (errorCount <= 3) {
+                logger.error(`Evaluation error (${errorCount}):`, {
+                    tool: sample.tool,
+                    error: errorMessage,
+                    stack: error instanceof Error ? error.stack : undefined,
+                });
+            }
+            result = {
+                sample,
+                actualDecision: 'pass', // Default on error
+                actualPassWeight: 0,
+                actualBlockWeight: 0,
+                reasoning: '',
+                correct: false,
+                latencyMs: Date.now() - sampleStart,
+                error: errorMessage,
+            };
+        }
+        results.push(result);
+        // Calculate ETA
+        const elapsed = Date.now() - startTime.getTime();
+        const avgTimePerSample = elapsed / (i + 1);
+        const remaining = samples.length - (i + 1);
+        const eta = avgTimePerSample * remaining;
+        // Report progress
+        if (onProgress) {
+            onProgress(i + 1, samples.length, result, eta);
+        }
+        // Log every 10% or every 100 samples
+        if ((i + 1) % Math.max(Math.floor(samples.length / 10), 100) === 0) {
+            const progress = ((i + 1) / samples.length * 100).toFixed(1);
+            const correctSoFar = results.filter(r => r.correct).length;
+            const accuracySoFar = (correctSoFar / (i + 1) * 100).toFixed(2);
+            logger.info(`Progress: ${progress}% (${i + 1}/${samples.length}) - Accuracy: ${accuracySoFar}%`);
+        }
+    }
+    const endTime = new Date();
+    // Calculate metrics
+    const confusionMatrix = calculateConfusionMatrix(results);
+    const metrics = calculateClassificationMetrics(confusionMatrix);
+    const latency = calculateLatencyStats(results);
+    const categories = calculateCategoryMetrics(results);
+    // Get incorrect predictions for analysis
+    const incorrectPredictions = results.filter(r => !r.correct && !r.error);
+    // Build report
+    const report = {
+        startTime: startTime.toISOString(),
+        endTime: endTime.toISOString(),
+        durationMs: endTime.getTime() - startTime.getTime(),
+        model: config.kernel.model,
+        totalSamples: samples.length,
+        correctCount: results.filter(r => r.correct).length,
+        incorrectCount: results.filter(r => !r.correct && !r.error).length,
+        errorCount,
+        confusionMatrix,
+        metrics,
+        latency,
+        categories,
+        incorrectPredictions: incorrectPredictions.slice(0, 100), // Limit to 100 for report size
+        config,
+    };
+    if (config.includeResults) {
+        report.results = results;
+    }
+    logger.info('Benchmark complete', {
+        totalSamples: report.totalSamples,
+        accuracy: (metrics.accuracy * 100).toFixed(2) + '%',
+        f1Score: metrics.f1Score.toFixed(4),
+        meanLatency: latency.mean.toFixed(2) + 'ms',
+        errors: errorCount,
+    });
+    return report;
+}
+/**
+ * Format a benchmark report for console output.
+ */
+export function formatReportConsole(report) {
+    const lines = [];
+    lines.push('');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    lines.push('                    VETO BENCHMARK REPORT                       ');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    lines.push('');
+    // Overview
+    lines.push('┌─────────────────────────────────────────────────────────────┐');
+    lines.push('│ OVERVIEW                                                    │');
+    lines.push('├─────────────────────────────────────────────────────────────┤');
+    lines.push(`│ Model:          ${report.model.padEnd(43)}│`);
+    lines.push(`│ Total Samples:  ${report.totalSamples.toString().padEnd(43)}│`);
+    lines.push(`│ Duration:       ${formatDuration(report.durationMs).padEnd(43)}│`);
+    lines.push(`│ Throughput:     ${(report.totalSamples / (report.durationMs / 1000)).toFixed(2).padEnd(40)} samples/sec │`);
+    lines.push('└─────────────────────────────────────────────────────────────┘');
+    lines.push('');
+    // Classification Metrics
+    lines.push('┌─────────────────────────────────────────────────────────────┐');
+    lines.push('│ CLASSIFICATION METRICS                                      │');
+    lines.push('├─────────────────────────────────────────────────────────────┤');
+    lines.push(`│ Accuracy:       ${(report.metrics.accuracy * 100).toFixed(2)}%`.padEnd(62) + '│');
+    lines.push(`│ Precision:      ${(report.metrics.precision * 100).toFixed(2)}%`.padEnd(62) + '│');
+    lines.push(`│ Recall:         ${(report.metrics.recall * 100).toFixed(2)}%`.padEnd(62) + '│');
+    lines.push(`│ F1 Score:       ${report.metrics.f1Score.toFixed(4)}`.padEnd(62) + '│');
+    lines.push(`│ MCC:            ${report.metrics.mcc.toFixed(4)}`.padEnd(62) + '│');
+    lines.push(`│ FP Rate:        ${(report.metrics.falsePositiveRate * 100).toFixed(2)}%`.padEnd(62) + '│');
+    lines.push(`│ FN Rate:        ${(report.metrics.falseNegativeRate * 100).toFixed(2)}%`.padEnd(62) + '│');
+    lines.push('└─────────────────────────────────────────────────────────────┘');
+    lines.push('');
+    // Confusion Matrix
+    const cm = report.confusionMatrix;
+    lines.push('┌─────────────────────────────────────────────────────────────┐');
+    lines.push('│ CONFUSION MATRIX                                            │');
+    lines.push('├─────────────────────────────────────────────────────────────┤');
+    lines.push('│                    Predicted                                │');
+    lines.push('│                    PASS        BLOCK                        │');
+    lines.push(`│ Actual PASS        ${cm.trueNegative.toString().padStart(6)}      ${cm.falsePositive.toString().padStart(6)}   (TN / FP)            │`);
+    lines.push(`│ Actual BLOCK       ${cm.falseNegative.toString().padStart(6)}      ${cm.truePositive.toString().padStart(6)}   (FN / TP)            │`);
+    lines.push('└─────────────────────────────────────────────────────────────┘');
+    lines.push('');
+    // Latency Statistics
+    lines.push('┌─────────────────────────────────────────────────────────────┐');
+    lines.push('│ LATENCY STATISTICS                                          │');
+    lines.push('├─────────────────────────────────────────────────────────────┤');
+    lines.push(`│ Min:            ${report.latency.min.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ Max:            ${report.latency.max.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ Mean:           ${report.latency.mean.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ Median:         ${report.latency.median.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ P95:            ${report.latency.p95.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ P99:            ${report.latency.p99.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push(`│ Std Dev:        ${report.latency.stdDev.toFixed(2)} ms`.padEnd(62) + '│');
+    lines.push('└─────────────────────────────────────────────────────────────┘');
+    lines.push('');
+    // Per-Category Results
+    if (report.categories.length > 0) {
+        lines.push('┌─────────────────────────────────────────────────────────────┐');
+        lines.push('│ PER-CATEGORY ACCURACY                                       │');
+        lines.push('├─────────────────────────────────────────────────────────────┤');
+        for (const cat of report.categories) {
+            const catName = cat.category.length > 30
+                ? cat.category.slice(0, 27) + '...'
+                : cat.category;
+            const accuracy = (cat.accuracy * 100).toFixed(1) + '%';
+            lines.push(`│ ${catName.padEnd(32)} ${accuracy.padStart(7)} (n=${cat.sampleCount})`.padEnd(62) + '│');
+        }
+        lines.push('└─────────────────────────────────────────────────────────────┘');
+        lines.push('');
+    }
+    // Error Summary
+    if (report.errorCount > 0) {
+        lines.push('┌─────────────────────────────────────────────────────────────┐');
+        lines.push('│ ERRORS                                                      │');
+        lines.push('├─────────────────────────────────────────────────────────────┤');
+        lines.push(`│ Error Count:    ${report.errorCount.toString().padEnd(43)}│`);
+        lines.push(`│ Error Rate:     ${((report.errorCount / report.totalSamples) * 100).toFixed(2)}%`.padEnd(62) + '│');
+        lines.push('└─────────────────────────────────────────────────────────────┘');
+        lines.push('');
+    }
+    // Sample Incorrect Predictions
+    if (report.incorrectPredictions.length > 0) {
+        lines.push('┌─────────────────────────────────────────────────────────────┐');
+        lines.push('│ SAMPLE INCORRECT PREDICTIONS (first 5)                      │');
+        lines.push('├─────────────────────────────────────────────────────────────┤');
+        for (const pred of report.incorrectPredictions.slice(0, 5)) {
+            lines.push(`│ ID: ${pred.sample.id}`.padEnd(62) + '│');
+            lines.push(`│   Tool: ${pred.sample.tool}`.padEnd(62) + '│');
+            lines.push(`│   Expected: ${pred.sample.expectedDecision}, Actual: ${pred.actualDecision}`.padEnd(62) + '│');
+            lines.push(`│   Reasoning: ${pred.reasoning.slice(0, 45)}...`.padEnd(62) + '│');
+            lines.push('│'.padEnd(62) + '│');
+        }
+        lines.push('└─────────────────────────────────────────────────────────────┘');
+    }
+    lines.push('');
+    lines.push('═══════════════════════════════════════════════════════════════');
+    return lines.join('\n');
+}
+/**
+ * Format duration in human-readable format.
+ */
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const minutes = Math.floor(ms / 60000);
+    const seconds = ((ms % 60000) / 1000).toFixed(0);
+    return `${minutes}m ${seconds}s`;
+}
+/**
+ * Save benchmark report to JSON file.
+ */
+export function saveReportJson(report, path) {
+    writeFileSync(path, JSON.stringify(report, null, 2), 'utf-8');
+}
+/**
+ * Create a default progress logger.
+ */
+export function createProgressLogger() {
+    let lastPercent = -1;
+    return (completed, total, _current, eta) => {
+        const percent = Math.floor((completed / total) * 100);
+        // Only log on percentage change
+        if (percent !== lastPercent) {
+            lastPercent = percent;
+            const bar = '█'.repeat(Math.floor(percent / 2)) + '░'.repeat(50 - Math.floor(percent / 2));
+            const etaStr = formatDuration(eta);
+            process.stdout.write(`\r[${bar}] ${percent}% (${completed}/${total}) ETA: ${etaStr}  `);
+            if (completed === total) {
+                process.stdout.write('\n');
+            }
+        }
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACL,wBAAwB,EACxB,8BAA8B,EAC9B,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,cAAc,CAAC;AA6BtB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA+B;IAChE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;QAChC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK;QACtC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;KAC3B,CAAC,CAAC;IAEH,eAAe;IACf,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,IAAI,CACZ,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAEhD,uBAAuB;IACvB,MAAM,YAAY,GAAiB;QACjC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;QAC9B,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;QACtC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;QAClC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,CAAC;IAEF,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC;QACpC,MAAM,EAAE,YAAY;QACpB,MAAM;KACP,CAAC,CAAC;IAEH,kBAAkB;IAClB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE7B,IAAI,MAAuB,CAAC;QAE5B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAErE,MAAM,GAAG;gBACP,MAAM;gBACN,cAAc,EAAE,QAAQ,CAAC,QAAQ;gBACjC,gBAAgB,EAAE,QAAQ,CAAC,WAAW;gBACtC,iBAAiB,EAAE,QAAQ,CAAC,YAAY;gBACxC,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,YAAY,EAAE,QAAQ,CAAC,aAAa;gBACpC,OAAO,EAAE,QAAQ,CAAC,QAAQ,KAAK,MAAM,CAAC,gBAAgB;gBACtD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;aACpC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,UAAU,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAE5E,qCAAqC;YACrC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,KAAK,CAAC,qBAAqB,UAAU,IAAI,EAAE;oBAChD,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,YAAY;oBACnB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG;gBACP,MAAM;gBACN,cAAc,EAAE,MAAM,EAAE,mBAAmB;gBAC3C,gBAAgB,EAAE,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,SAAS,EAAE,EAAE;gBACb,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;gBACnC,KAAK,EAAE,YAAY;aACpB,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,gBAAgB;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,gBAAgB,GAAG,SAAS,CAAC;QAEzC,kBAAkB;QAClB,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAED,qCAAqC;QACrC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnE,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YAC3D,MAAM,aAAa,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,aAAa,QAAQ,MAAM,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,iBAAiB,aAAa,GAAG,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAE3B,oBAAoB;IACpB,MAAM,eAAe,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,8BAA8B,CAAC,eAAe,CAAC,CAAC;IAChE,MAAM,OAAO,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAErD,yCAAyC;IACzC,MAAM,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzE,eAAe;IACf,MAAM,MAAM,GAAoB;QAC9B,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE;QAC9B,UAAU,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;QACnD,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM;QACnD,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;QAClE,UAAU;QACV,eAAe;QACf,OAAO;QACP,OAAO;QACP,UAAU;QACV,oBAAoB,EAAE,oBAAoB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,+BAA+B;QACzF,MAAM;KACP,CAAC;IAEF,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,QAAQ,EAAE,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG;QACnD,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACnC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI;QAC3C,MAAM,EAAE,UAAU;KACnB,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAuB;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,WAAW;IACX,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IAC5D,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,qBAAqB,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,YAAY,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;IAC1H,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,yBAAyB;IACzB,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IAChG,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IAC9F,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACtF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACzG,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACzG,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,mBAAmB;IACnB,MAAM,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;IACtJ,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;IACtJ,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,qBAAqB;IACrB,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACtF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,uBAAuB;IACvB,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAE9E,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE;gBACtC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;gBACnC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;YACjB,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,WAAW,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;QACxG,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,gBAAgB;IAChB,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;QAClH,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,+BAA+B;IAC/B,IAAI,MAAM,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAE9E,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC3D,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,gBAAgB,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;YAC7G,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;YAChF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;QACnC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAE9E,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,KAAK;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACjD,OAAO,GAAG,OAAO,KAAK,OAAO,GAAG,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAuB,EAAE,IAAY;IAClE,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC,CAAC;IAErB,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAEtD,gCAAgC;QAChC,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;YAC5B,WAAW,GAAG,OAAO,CAAC;YACtB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,OAAO,MAAM,SAAS,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC,CAAC;YAExF,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/benchmark/types.d.ts

@@ -0,0 +1,188 @@
+/**
+ * Types for the Veto benchmark suite.
+ *
+ * @module benchmark/types
+ */
+/**
+ * A single benchmark sample from the dataset.
+ */
+export interface BenchmarkSample {
+    /** Unique identifier for this sample */
+    id: string;
+    /** Tool name being called */
+    tool: string;
+    /** Tool arguments */
+    arguments: Record<string, unknown>;
+    /** Rules to evaluate against */
+    rules: import('../rules/types.js').Rule[];
+    /** Expected decision */
+    expectedDecision: 'pass' | 'block';
+    /** Expected pass weight (approximate) */
+    expectedPassWeight: number;
+    /** Expected block weight (approximate) */
+    expectedBlockWeight: number;
+    /** Source file this sample came from */
+    sourceFile?: string;
+    /** Category/domain of this sample */
+    category?: string;
+}
+/**
+ * Result of running a single benchmark sample.
+ */
+export interface BenchmarkResult {
+    /** Sample that was evaluated */
+    sample: BenchmarkSample;
+    /** Actual decision from the kernel */
+    actualDecision: 'pass' | 'block';
+    /** Actual pass weight */
+    actualPassWeight: number;
+    /** Actual block weight */
+    actualBlockWeight: number;
+    /** Reasoning from the kernel */
+    reasoning: string;
+    /** Matched rules (if block) */
+    matchedRules?: string[];
+    /** Whether the decision was correct */
+    correct: boolean;
+    /** Latency in milliseconds */
+    latencyMs: number;
+    /** Error if evaluation failed */
+    error?: string;
+}
+/**
+ * Confusion matrix for binary classification.
+ */
+export interface ConfusionMatrix {
+    /** True Positives: expected block, actual block */
+    truePositive: number;
+    /** True Negatives: expected pass, actual pass */
+    trueNegative: number;
+    /** False Positives: expected pass, actual block */
+    falsePositive: number;
+    /** False Negatives: expected block, actual pass */
+    falseNegative: number;
+}
+/**
+ * Latency statistics.
+ */
+export interface LatencyStats {
+    /** Minimum latency in ms */
+    min: number;
+    /** Maximum latency in ms */
+    max: number;
+    /** Mean latency in ms */
+    mean: number;
+    /** Median latency in ms */
+    median: number;
+    /** 95th percentile latency in ms */
+    p95: number;
+    /** 99th percentile latency in ms */
+    p99: number;
+    /** Standard deviation in ms */
+    stdDev: number;
+    /** Total time in ms */
+    totalMs: number;
+}
+/**
+ * Classification metrics derived from confusion matrix.
+ */
+export interface ClassificationMetrics {
+    /** Overall accuracy */
+    accuracy: number;
+    /** Precision for block class (TP / (TP + FP)) */
+    precision: number;
+    /** Recall for block class (TP / (TP + FN)) */
+    recall: number;
+    /** F1 score (harmonic mean of precision and recall) */
+    f1Score: number;
+    /** False positive rate (FP / (FP + TN)) */
+    falsePositiveRate: number;
+    /** False negative rate (FN / (FN + TP)) */
+    falseNegativeRate: number;
+    /** Matthews Correlation Coefficient */
+    mcc: number;
+}
+/**
+ * Per-category breakdown of metrics.
+ */
+export interface CategoryMetrics {
+    /** Category name */
+    category: string;
+    /** Number of samples in this category */
+    sampleCount: number;
+    /** Accuracy for this category */
+    accuracy: number;
+    /** Confusion matrix for this category */
+    confusionMatrix: ConfusionMatrix;
+    /** Latency stats for this category */
+    latency: LatencyStats;
+}
+/**
+ * Complete benchmark report.
+ */
+export interface BenchmarkReport {
+    /** Timestamp when benchmark started */
+    startTime: string;
+    /** Timestamp when benchmark ended */
+    endTime: string;
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** Model used for inference */
+    model: string;
+    /** Total samples evaluated */
+    totalSamples: number;
+    /** Samples that passed (correct) */
+    correctCount: number;
+    /** Samples that failed (incorrect) */
+    incorrectCount: number;
+    /** Samples that errored */
+    errorCount: number;
+    /** Overall confusion matrix */
+    confusionMatrix: ConfusionMatrix;
+    /** Classification metrics */
+    metrics: ClassificationMetrics;
+    /** Latency statistics */
+    latency: LatencyStats;
+    /** Per-category breakdown */
+    categories: CategoryMetrics[];
+    /** Individual results (optional, can be large) */
+    results?: BenchmarkResult[];
+    /** Incorrect predictions for analysis */
+    incorrectPredictions: BenchmarkResult[];
+    /** Configuration used */
+    config: BenchmarkConfig;
+}
+/**
+ * Configuration for running benchmarks.
+ */
+export interface BenchmarkConfig {
+    /** Path to dataset files (glob pattern) */
+    datasetPath: string;
+    /** Maximum samples to evaluate (0 = all) */
+    maxSamples: number;
+    /** Whether to shuffle samples */
+    shuffle: boolean;
+    /** Random seed for shuffling */
+    seed?: number;
+    /** Whether to include individual results in report */
+    includeResults: boolean;
+    /** Kernel configuration */
+    kernel: {
+        baseUrl: string;
+        model: string;
+        temperature: number;
+        maxTokens: number;
+        timeout: number;
+    };
+    /** Concurrency (parallel requests) */
+    concurrency: number;
+    /** Output format */
+    outputFormat: 'console' | 'json' | 'both';
+    /** Output file path (for JSON) */
+    outputPath?: string;
+}
+/**
+ * Default benchmark configuration.
+ */
+export declare const DEFAULT_BENCHMARK_CONFIG: BenchmarkConfig;
+//# sourceMappingURL=types.d.ts.map

package/dist/benchmark/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAC;IACX,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,gCAAgC;IAChC,KAAK,EAAE,OAAO,mBAAmB,EAAE,IAAI,EAAE,CAAC;IAC1C,wBAAwB;IACxB,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,yCAAyC;IACzC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0CAA0C;IAC1C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,eAAe,CAAC;IACxB,sCAAsC;IACtC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC;IACjC,yBAAyB;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,0BAA0B;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gCAAgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,sCAAsC;IACtC,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,eAAe,EAAE,eAAe,CAAC;IACjC,6BAA6B;IAC7B,OAAO,EAAE,qBAAqB,CAAC;IAC/B,yBAAyB;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,6BAA6B;IAC7B,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,kDAAkD;IAClD,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,yCAAyC;IACzC,oBAAoB,EAAE,eAAe,EAAE,CAAC;IACxC,yBAAyB;IACzB,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B;IAC3B,MAAM,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB;IACpB,YAAY,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAC1C,kCAAkC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eActC,CAAC"}

package/dist/benchmark/types.js

@@ -0,0 +1,24 @@
+/**
+ * Types for the Veto benchmark suite.
+ *
+ * @module benchmark/types
+ */
+/**
+ * Default benchmark configuration.
+ */
+export const DEFAULT_BENCHMARK_CONFIG = {
+    datasetPath: 'data/batches/**/*.jsonl',
+    maxSamples: 0,
+    shuffle: true,
+    includeResults: false,
+    kernel: {
+        baseUrl: 'http://localhost:11434/v1',
+        model: 'hf.co/ycaleb/veto-warden-4b-GGUF:Q4_K_M',
+        temperature: 0.1,
+        maxTokens: 256,
+        timeout: 30000,
+    },
+    concurrency: 1,
+    outputFormat: 'both',
+};
+//# sourceMappingURL=types.js.map

package/dist/benchmark/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA4LH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,WAAW,EAAE,yBAAyB;IACtC,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,IAAI;IACb,cAAc,EAAE,KAAK;IACrB,MAAM,EAAE;QACN,OAAO,EAAE,2BAA2B;QACpC,KAAK,EAAE,yCAAyC;QAChD,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,GAAG;QACd,OAAO,EAAE,KAAK;KACf;IACD,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,MAAM;CACrB,CAAC"}

package/dist/cli/bin.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+/**
+ * Veto CLI entry point.
+ *
+ * @module cli/bin
+ */
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/dist/cli/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../../src/cli/bin.ts"],"names":[],"mappings":";AAEA;;;;GAIG"}

package/dist/cli/bin.js

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+/**
+ * Veto CLI entry point.
+ *
+ * @module cli/bin
+ */
+import { init } from './init.js';
+const VERSION = '0.1.0';
+/**
+ * Print help message.
+ */
+function printHelp() {
+    console.log(`
+Veto - AI Agent Tool Call Guardrail
+
+Usage:
+  veto <command> [options]
+
+Commands:
+  init          Initialize Veto in the current directory
+  version       Show version information
+  help          Show this help message
+
+Options:
+  --force, -f   Force overwrite existing files (init)
+  --quiet, -q   Suppress output
+  --help, -h    Show help
+
+Examples:
+  veto init           Initialize Veto in current directory
+  veto init --force   Reinitialize, overwriting existing files
+`);
+}
+/**
+ * Print version.
+ */
+function printVersion() {
+    console.log(`veto v${VERSION}`);
+}
+/**
+ * Parse command line arguments.
+ */
+function parseArgs(args) {
+    const flags = {};
+    let command = '';
+    for (const arg of args) {
+        if (arg.startsWith('--')) {
+            const flag = arg.slice(2);
+            flags[flag] = true;
+        }
+        else if (arg.startsWith('-')) {
+            const shortFlags = arg.slice(1).split('');
+            for (const f of shortFlags) {
+                switch (f) {
+                    case 'f':
+                        flags['force'] = true;
+                        break;
+                    case 'q':
+                        flags['quiet'] = true;
+                        break;
+                    case 'h':
+                        flags['help'] = true;
+                        break;
+                }
+            }
+        }
+        else if (!command) {
+            command = arg;
+        }
+    }
+    return { command, flags };
+}
+/**
+ * Main CLI entry point.
+ */
+async function main() {
+    const args = process.argv.slice(2);
+    const { command, flags } = parseArgs(args);
+    // Handle help flag
+    if (flags['help'] || command === 'help') {
+        printHelp();
+        process.exit(0);
+    }
+    // Handle version flag or command
+    if (flags['version'] || command === 'version') {
+        printVersion();
+        process.exit(0);
+    }
+    // Handle commands
+    switch (command) {
+        case 'init': {
+            const result = await init({
+                force: flags['force'],
+                quiet: flags['quiet'],
+            });
+            process.exit(result.success ? 0 : 1);
+            break;
+        }
+        case '': {
+            // No command provided
+            console.log('Veto - AI Agent Tool Call Guardrail');
+            console.log('');
+            console.log('Run "veto help" for usage information.');
+            console.log('Run "veto init" to initialize Veto in your project.');
+            process.exit(0);
+            break;
+        }
+        default: {
+            console.error(`Unknown command: ${command}`);
+            console.error('Run "veto help" for usage information.');
+            process.exit(1);
+        }
+    }
+}
+// Run the CLI
+main().catch((error) => {
+    console.error('Error:', error.message);
+    process.exit(1);
+});
+//# sourceMappingURL=bin.js.map