veto-sdk 1.0.0

package/dist/benchmark/loader.js

@@ -0,0 +1,321 @@
+/**
+ * Dataset loader for benchmark samples.
+ *
+ * Parses JSONL training data files and extracts benchmark samples
+ * with tool calls, rules, and expected decisions.
+ *
+ * @module benchmark/loader
+ */
+import { readFileSync, existsSync } from 'node:fs';
+import { basename, dirname } from 'node:path';
+import { glob } from '../utils/glob.js';
+/**
+ * Load benchmark samples from a glob pattern.
+ *
+ * @param pattern - Glob pattern for JSONL files
+ * @param maxSamples - Maximum samples to load (0 = all)
+ * @param shuffle - Whether to shuffle samples
+ * @param seed - Random seed for shuffling
+ */
+export async function loadBenchmarkSamples(pattern, maxSamples = 0, shuffle = false, seed) {
+    const files = await glob(pattern);
+    if (files.length === 0) {
+        throw new Error(`No files found matching pattern: ${pattern}`);
+    }
+    const samples = [];
+    let sampleId = 0;
+    for (const file of files) {
+        if (!existsSync(file))
+            continue;
+        const category = extractCategory(file);
+        const content = readFileSync(file, 'utf-8');
+        const lines = content.trim().split('\n').filter(line => line.trim());
+        for (const line of lines) {
+            try {
+                const example = JSON.parse(line);
+                const sample = parseTrainingExample(example, sampleId++, file, category);
+                if (sample) {
+                    samples.push(sample);
+                }
+            }
+            catch (error) {
+                // Skip malformed lines
+                continue;
+            }
+        }
+    }
+    if (samples.length === 0) {
+        throw new Error('No valid samples found in dataset');
+    }
+    // Shuffle if requested
+    let result = shuffle ? shuffleArray(samples, seed) : samples;
+    // Limit samples if requested
+    if (maxSamples > 0 && result.length > maxSamples) {
+        result = result.slice(0, maxSamples);
+    }
+    return result;
+}
+/**
+ * Parse a training example into a benchmark sample.
+ */
+function parseTrainingExample(example, id, sourceFile, category) {
+    if (!example.messages || example.messages.length < 3) {
+        return null;
+    }
+    const userMessage = example.messages.find(m => m.role === 'user');
+    const assistantMessage = example.messages.find(m => m.role === 'assistant');
+    if (!userMessage || !assistantMessage) {
+        return null;
+    }
+    // Parse user content to extract tool call and rules
+    const { tool, arguments: args, rules } = parseUserContent(userMessage.content);
+    if (!tool || !rules) {
+        return null;
+    }
+    // Parse expected response
+    const expected = parseExpectedResponse(assistantMessage.content);
+    if (!expected) {
+        return null;
+    }
+    return {
+        id: `sample-${id}`,
+        tool,
+        arguments: args,
+        rules,
+        expectedDecision: expected.decision,
+        expectedPassWeight: expected.pass_weight,
+        expectedBlockWeight: expected.block_weight,
+        sourceFile,
+        category,
+    };
+}
+/**
+ * Parse user content to extract tool call and rules.
+ */
+function parseUserContent(content) {
+    const result = {
+        tool: null,
+        arguments: {},
+        rules: null,
+    };
+    // Split into TOOL CALL and RULES sections
+    const toolCallMatch = content.match(/TOOL CALL:\s*\n([\s\S]*?)(?=\nRULES:|$)/);
+    const rulesMatch = content.match(/RULES:\s*\n([\s\S]*?)$/);
+    if (toolCallMatch) {
+        const toolSection = toolCallMatch[1];
+        // Extract tool name
+        const toolNameMatch = toolSection.match(/tool:\s*(\S+)/);
+        if (toolNameMatch) {
+            result.tool = toolNameMatch[1];
+        }
+        // Extract arguments (YAML-like format)
+        const argsMatch = toolSection.match(/arguments:\s*\n([\s\S]*)/);
+        if (argsMatch) {
+            result.arguments = parseYamlLikeArgs(argsMatch[1]);
+        }
+    }
+    if (rulesMatch) {
+        result.rules = parseRulesSection(rulesMatch[1]);
+    }
+    return result;
+}
+/**
+ * Parse YAML-like arguments.
+ */
+function parseYamlLikeArgs(content) {
+    const args = {};
+    const lines = content.split('\n');
+    for (const line of lines) {
+        const match = line.match(/^\s{2}(\w+):\s*(.*)$/);
+        if (match) {
+            const [, key, rawValue] = match;
+            args[key] = parseValue(rawValue.trim());
+        }
+    }
+    return args;
+}
+/**
+ * Parse a YAML-like value.
+ */
+function parseValue(value) {
+    // Empty string
+    if (value === '""' || value === "''")
+        return '';
+    // Quoted string
+    if ((value.startsWith('"') && value.endsWith('"')) ||
+        (value.startsWith("'") && value.endsWith("'"))) {
+        return value.slice(1, -1);
+    }
+    // Boolean
+    if (value === 'true')
+        return true;
+    if (value === 'false')
+        return false;
+    // Null
+    if (value === 'null' || value === '~')
+        return null;
+    // Number
+    const num = Number(value);
+    if (!isNaN(num) && value !== '')
+        return num;
+    // Array (simple)
+    if (value.startsWith('[') && value.endsWith(']')) {
+        const inner = value.slice(1, -1);
+        if (inner === '')
+            return [];
+        return inner.split(',').map(v => parseValue(v.trim()));
+    }
+    // Default to string
+    return value;
+}
+/**
+ * Parse RULES section into Rule objects.
+ */
+function parseRulesSection(content) {
+    const rules = [];
+    const ruleBlocks = content.split(/^- id:/m).filter(Boolean);
+    for (const block of ruleBlocks) {
+        const rule = parseRuleBlock('- id:' + block);
+        if (rule) {
+            rules.push(rule);
+        }
+    }
+    return rules;
+}
+/**
+ * Parse a single rule block.
+ */
+function parseRuleBlock(block) {
+    const lines = block.split('\n');
+    const rule = {};
+    let currentSection = null;
+    let conditions = [];
+    let currentCondition = {};
+    for (const line of lines) {
+        // Rule ID
+        const idMatch = line.match(/^- id:\s*(.+)$/);
+        if (idMatch) {
+            rule.id = idMatch[1].trim();
+            continue;
+        }
+        // Simple fields
+        const fieldMatch = line.match(/^\s{2}(\w+):\s*(.+)$/);
+        if (fieldMatch) {
+            const [, key, value] = fieldMatch;
+            switch (key) {
+                case 'name':
+                    rule.name = value;
+                    break;
+                case 'enabled':
+                    rule.enabled = value === 'true';
+                    break;
+                case 'severity':
+                    rule.severity = value;
+                    break;
+                case 'action':
+                    rule.action = value;
+                    break;
+                case 'tools':
+                    // Parse inline array: [tool1, tool2]
+                    if (value.startsWith('[')) {
+                        rule.tools = value.slice(1, -1).split(',').map(t => t.trim());
+                    }
+                    currentSection = 'tools';
+                    break;
+                case 'conditions':
+                    currentSection = 'conditions';
+                    conditions = [];
+                    break;
+                case 'condition_groups':
+                    currentSection = 'condition_groups';
+                    break;
+            }
+            continue;
+        }
+        // Condition fields
+        if (currentSection === 'conditions') {
+            const condFieldMatch = line.match(/^\s{4,6}- field:\s*(.+)$/);
+            if (condFieldMatch) {
+                if (Object.keys(currentCondition).length > 0) {
+                    conditions.push(currentCondition);
+                }
+                currentCondition = { field: condFieldMatch[1].trim() };
+                continue;
+            }
+            const operatorMatch = line.match(/^\s{6,8}operator:\s*(.+)$/);
+            if (operatorMatch) {
+                currentCondition.operator = operatorMatch[1].trim();
+                continue;
+            }
+            const valueMatch = line.match(/^\s{6,8}value:\s*(.+)$/);
+            if (valueMatch) {
+                currentCondition.value = parseValue(valueMatch[1].trim());
+                continue;
+            }
+        }
+    }
+    // Add last condition
+    if (Object.keys(currentCondition).length > 0) {
+        conditions.push(currentCondition);
+    }
+    if (conditions.length > 0) {
+        rule.conditions = conditions;
+    }
+    // Validate required fields
+    if (!rule.id || !rule.name || rule.enabled === undefined || !rule.severity || !rule.action) {
+        return null;
+    }
+    return rule;
+}
+/**
+ * Parse expected response from assistant content.
+ */
+function parseExpectedResponse(content) {
+    try {
+        // Extract JSON from content (might have extra text)
+        const jsonMatch = content.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return null;
+        const parsed = JSON.parse(jsonMatch[0]);
+        if (typeof parsed.pass_weight !== 'number' ||
+            typeof parsed.block_weight !== 'number' ||
+            (parsed.decision !== 'pass' && parsed.decision !== 'block')) {
+            return null;
+        }
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Extract category from file path.
+ */
+function extractCategory(filePath) {
+    const dir = basename(dirname(filePath));
+    const file = basename(filePath, '.jsonl');
+    return `${dir}/${file}`;
+}
+/**
+ * Shuffle array using Fisher-Yates algorithm with optional seed.
+ */
+function shuffleArray(array, seed) {
+    const result = [...array];
+    let random = seed !== undefined ? seededRandom(seed) : Math.random;
+    for (let i = result.length - 1; i > 0; i--) {
+        const j = Math.floor(random() * (i + 1));
+        [result[i], result[j]] = [result[j], result[i]];
+    }
+    return result;
+}
+/**
+ * Create a seeded random number generator.
+ */
+function seededRandom(seed) {
+    let state = seed;
+    return () => {
+        state = (state * 1103515245 + 12345) & 0x7fffffff;
+        return state / 0x7fffffff;
+    };
+}
+//# sourceMappingURL=loader.js.map

package/dist/benchmark/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/benchmark/loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AA8BxC;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAe,EACf,aAAqB,CAAC,EACtB,UAAmB,KAAK,EACxB,IAAa;IAEb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IAElC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAEhC,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoB,CAAC;gBACpD,MAAM,MAAM,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAEzE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,uBAAuB;gBACvB,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,uBAAuB;IACvB,IAAI,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAE7D,6BAA6B;IAC7B,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;QACjD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAC3B,OAAwB,EACxB,EAAU,EACV,UAAkB,EAClB,QAAgB;IAEhB,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAClE,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;IAE5E,IAAI,CAAC,WAAW,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oDAAoD;IACpD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE/E,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0BAA0B;IAC1B,MAAM,QAAQ,GAAG,qBAAqB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAEjE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,EAAE,EAAE,UAAU,EAAE,EAAE;QAClB,IAAI;QACJ,SAAS,EAAE,IAAI;QACf,KAAK;QACL,gBAAgB,EAAE,QAAQ,CAAC,QAAQ;QACnC,kBAAkB,EAAE,QAAQ,CAAC,WAAW;QACxC,mBAAmB,EAAE,QAAQ,CAAC,YAAY;QAC1C,UAAU;QACV,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IAKvC,MAAM,MAAM,GAIR;QACF,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,EAAE;QACb,KAAK,EAAE,IAAI;KACZ,CAAC;IAEF,0CAA0C;IAC1C,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAE3D,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAErC,oBAAoB;QACpB,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACzD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,uCAAuC;QACvC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAChE,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,CAAC,SAAS,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,KAAa;IAC/B,eAAe;IACf,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAEhD,gBAAgB;IAChB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,UAAU;IACV,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IAEpC,OAAO;IACP,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAEnD,SAAS;IACT,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC;IAE5C,iBAAiB;IACjB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,oBAAoB;IACpB,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC;QAC7C,IAAI,IAAI,EAAE,CAAC;YACT,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAuD,IAAI,CAAC;IAC9E,IAAI,UAAU,GAAoB,EAAE,CAAC;IACrC,IAAI,gBAAgB,GAA2B,EAAE,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,UAAU;QACV,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC7C,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,UAAU,CAAC;YAClC,QAAQ,GAAG,EAAE,CAAC;gBACZ,KAAK,MAAM;oBACT,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;oBAClB,MAAM;gBACR,KAAK,SAAS;oBACZ,IAAI,CAAC,OAAO,GAAG,KAAK,KAAK,MAAM,CAAC;oBAChC,MAAM;gBACR,KAAK,UAAU;oBACb,IAAI,CAAC,QAAQ,GAAG,KAAyB,CAAC;oBAC1C,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,CAAC,MAAM,GAAG,KAAuB,CAAC;oBACtC,MAAM;gBACR,KAAK,OAAO;oBACV,qCAAqC;oBACrC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;oBAChE,CAAC;oBACD,cAAc,GAAG,OAAO,CAAC;oBACzB,MAAM;gBACR,KAAK,YAAY;oBACf,cAAc,GAAG,YAAY,CAAC;oBAC9B,UAAU,GAAG,EAAE,CAAC;oBAChB,MAAM;gBACR,KAAK,kBAAkB;oBACrB,cAAc,GAAG,kBAAkB,CAAC;oBACpC,MAAM;YACV,CAAC;YACD,SAAS;QACX,CAAC;QAED,mBAAmB;QACnB,IAAI,cAAc,KAAK,YAAY,EAAE,CAAC;YACpC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9D,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,UAAU,CAAC,IAAI,CAAC,gBAAiC,CAAC,CAAC;gBACrD,CAAC;gBACD,gBAAgB,GAAG,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvD,SAAS;YACX,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC9D,IAAI,aAAa,EAAE,CAAC;gBAClB,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAA+B,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;YACxD,IAAI,UAAU,EAAE,CAAC;gBACf,gBAAgB,CAAC,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1D,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,UAAU,CAAC,IAAI,CAAC,gBAAiC,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3F,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,oDAAoD;QACpD,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAExC,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ;YACtC,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ;YACvC,CAAC,MAAM,CAAC,QAAQ,KAAK,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAA0B,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,QAAgB;IACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1C,OAAO,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAI,KAAU,EAAE,IAAa;IAChD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IAC1B,IAAI,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAEnE,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,GAAG,KAAK,CAAC,GAAG,UAAU,CAAC;QAClD,OAAO,KAAK,GAAG,UAAU,CAAC;IAC5B,CAAC,CAAC;AACJ,CAAC"}

package/dist/benchmark/metrics.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Metrics calculation for benchmark results.
+ *
+ * @module benchmark/metrics
+ */
+import type { BenchmarkResult, ConfusionMatrix, LatencyStats, ClassificationMetrics, CategoryMetrics } from './types.js';
+/**
+ * Calculate confusion matrix from benchmark results.
+ * Treats "block" as the positive class.
+ */
+export declare function calculateConfusionMatrix(results: BenchmarkResult[]): ConfusionMatrix;
+/**
+ * Calculate classification metrics from confusion matrix.
+ */
+export declare function calculateClassificationMetrics(cm: ConfusionMatrix): ClassificationMetrics;
+/**
+ * Calculate latency statistics from benchmark results.
+ */
+export declare function calculateLatencyStats(results: BenchmarkResult[]): LatencyStats;
+/**
+ * Calculate per-category metrics.
+ */
+export declare function calculateCategoryMetrics(results: BenchmarkResult[]): CategoryMetrics[];
+/**
+ * Calculate weight calibration metrics.
+ * Measures how well the model's confidence weights match decisions.
+ */
+export declare function calculateWeightCalibration(results: BenchmarkResult[]): {
+    meanPassWeightForPass: number;
+    meanPassWeightForBlock: number;
+    meanBlockWeightForPass: number;
+    meanBlockWeightForBlock: number;
+    weightAccuracy: number;
+};
+//# sourceMappingURL=metrics.d.ts.map

package/dist/benchmark/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/benchmark/metrics.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EACf,YAAY,EACZ,qBAAqB,EACrB,eAAe,EAChB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,CA0BpF;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,EAAE,EAAE,eAAe,GAAG,qBAAqB,CAsCzF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,YAAY,CA0C9E;AAqBD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CA8BtF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG;IACtE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,uBAAuB,EAAE,MAAM,CAAC;IAChC,cAAc,EAAE,MAAM,CAAC;CACxB,CAsCA"}

package/dist/benchmark/metrics.js

@@ -0,0 +1,195 @@
+/**
+ * Metrics calculation for benchmark results.
+ *
+ * @module benchmark/metrics
+ */
+/**
+ * Calculate confusion matrix from benchmark results.
+ * Treats "block" as the positive class.
+ */
+export function calculateConfusionMatrix(results) {
+    let tp = 0, tn = 0, fp = 0, fn = 0;
+    for (const result of results) {
+        if (result.error)
+            continue;
+        const expected = result.sample.expectedDecision;
+        const actual = result.actualDecision;
+        if (expected === 'block' && actual === 'block') {
+            tp++;
+        }
+        else if (expected === 'pass' && actual === 'pass') {
+            tn++;
+        }
+        else if (expected === 'pass' && actual === 'block') {
+            fp++;
+        }
+        else if (expected === 'block' && actual === 'pass') {
+            fn++;
+        }
+    }
+    return {
+        truePositive: tp,
+        trueNegative: tn,
+        falsePositive: fp,
+        falseNegative: fn,
+    };
+}
+/**
+ * Calculate classification metrics from confusion matrix.
+ */
+export function calculateClassificationMetrics(cm) {
+    const { truePositive: tp, trueNegative: tn, falsePositive: fp, falseNegative: fn } = cm;
+    const total = tp + tn + fp + fn;
+    // Accuracy
+    const accuracy = total > 0 ? (tp + tn) / total : 0;
+    // Precision (for block class)
+    const precision = (tp + fp) > 0 ? tp / (tp + fp) : 0;
+    // Recall (for block class)
+    const recall = (tp + fn) > 0 ? tp / (tp + fn) : 0;
+    // F1 Score
+    const f1Score = (precision + recall) > 0
+        ? 2 * (precision * recall) / (precision + recall)
+        : 0;
+    // False Positive Rate
+    const falsePositiveRate = (fp + tn) > 0 ? fp / (fp + tn) : 0;
+    // False Negative Rate
+    const falseNegativeRate = (fn + tp) > 0 ? fn / (fn + tp) : 0;
+    // Matthews Correlation Coefficient
+    const mccNumerator = (tp * tn) - (fp * fn);
+    const mccDenominator = Math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn));
+    const mcc = mccDenominator > 0 ? mccNumerator / mccDenominator : 0;
+    return {
+        accuracy,
+        precision,
+        recall,
+        f1Score,
+        falsePositiveRate,
+        falseNegativeRate,
+        mcc,
+    };
+}
+/**
+ * Calculate latency statistics from benchmark results.
+ */
+export function calculateLatencyStats(results) {
+    const latencies = results
+        .filter(r => !r.error)
+        .map(r => r.latencyMs)
+        .sort((a, b) => a - b);
+    if (latencies.length === 0) {
+        return {
+            min: 0,
+            max: 0,
+            mean: 0,
+            median: 0,
+            p95: 0,
+            p99: 0,
+            stdDev: 0,
+            totalMs: 0,
+        };
+    }
+    const min = latencies[0];
+    const max = latencies[latencies.length - 1];
+    const sum = latencies.reduce((a, b) => a + b, 0);
+    const mean = sum / latencies.length;
+    const median = percentile(latencies, 50);
+    const p95 = percentile(latencies, 95);
+    const p99 = percentile(latencies, 99);
+    // Standard deviation
+    const squaredDiffs = latencies.map(l => Math.pow(l - mean, 2));
+    const avgSquaredDiff = squaredDiffs.reduce((a, b) => a + b, 0) / latencies.length;
+    const stdDev = Math.sqrt(avgSquaredDiff);
+    return {
+        min,
+        max,
+        mean,
+        median,
+        p95,
+        p99,
+        stdDev,
+        totalMs: sum,
+    };
+}
+/**
+ * Calculate percentile value from sorted array.
+ */
+function percentile(sortedArray, p) {
+    if (sortedArray.length === 0)
+        return 0;
+    if (sortedArray.length === 1)
+        return sortedArray[0];
+    const index = (p / 100) * (sortedArray.length - 1);
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    const fraction = index - lower;
+    if (lower === upper) {
+        return sortedArray[lower];
+    }
+    return sortedArray[lower] * (1 - fraction) + sortedArray[upper] * fraction;
+}
+/**
+ * Calculate per-category metrics.
+ */
+export function calculateCategoryMetrics(results) {
+    // Group results by category
+    const byCategory = new Map();
+    for (const result of results) {
+        const category = result.sample.category ?? 'unknown';
+        const existing = byCategory.get(category) ?? [];
+        existing.push(result);
+        byCategory.set(category, existing);
+    }
+    // Calculate metrics for each category
+    const categoryMetrics = [];
+    for (const [category, categoryResults] of byCategory) {
+        const confusionMatrix = calculateConfusionMatrix(categoryResults);
+        const metrics = calculateClassificationMetrics(confusionMatrix);
+        const latency = calculateLatencyStats(categoryResults);
+        categoryMetrics.push({
+            category,
+            sampleCount: categoryResults.length,
+            accuracy: metrics.accuracy,
+            confusionMatrix,
+            latency,
+        });
+    }
+    // Sort by category name
+    return categoryMetrics.sort((a, b) => a.category.localeCompare(b.category));
+}
+/**
+ * Calculate weight calibration metrics.
+ * Measures how well the model's confidence weights match decisions.
+ */
+export function calculateWeightCalibration(results) {
+    const passDecisions = results.filter(r => !r.error && r.actualDecision === 'pass');
+    const blockDecisions = results.filter(r => !r.error && r.actualDecision === 'block');
+    const meanPassWeightForPass = passDecisions.length > 0
+        ? passDecisions.reduce((sum, r) => sum + r.actualPassWeight, 0) / passDecisions.length
+        : 0;
+    const meanBlockWeightForPass = passDecisions.length > 0
+        ? passDecisions.reduce((sum, r) => sum + r.actualBlockWeight, 0) / passDecisions.length
+        : 0;
+    const meanPassWeightForBlock = blockDecisions.length > 0
+        ? blockDecisions.reduce((sum, r) => sum + r.actualPassWeight, 0) / blockDecisions.length
+        : 0;
+    const meanBlockWeightForBlock = blockDecisions.length > 0
+        ? blockDecisions.reduce((sum, r) => sum + r.actualBlockWeight, 0) / blockDecisions.length
+        : 0;
+    // Weight accuracy: how often does max weight match decision
+    const validResults = results.filter(r => !r.error);
+    const weightMatchCount = validResults.filter(r => {
+        const maxWeight = r.actualPassWeight > r.actualBlockWeight ? 'pass' : 'block';
+        return maxWeight === r.actualDecision;
+    }).length;
+    const weightAccuracy = validResults.length > 0
+        ? weightMatchCount / validResults.length
+        : 0;
+    return {
+        meanPassWeightForPass,
+        meanPassWeightForBlock,
+        meanBlockWeightForPass,
+        meanBlockWeightForBlock,
+        weightAccuracy,
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/benchmark/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/benchmark/metrics.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAA0B;IACjE,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,KAAK;YAAE,SAAS;QAE3B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;QAErC,IAAI,QAAQ,KAAK,OAAO,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC/C,EAAE,EAAE,CAAC;QACP,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACpD,EAAE,EAAE,CAAC;QACP,CAAC;aAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACrD,EAAE,EAAE,CAAC;QACP,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACrD,EAAE,EAAE,CAAC;QACP,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY,EAAE,EAAE;QAChB,YAAY,EAAE,EAAE;QAChB,aAAa,EAAE,EAAE;QACjB,aAAa,EAAE,EAAE;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,EAAmB;IAChE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IAEhC,WAAW;IACX,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,8BAA8B;IAC9B,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErD,2BAA2B;IAC3B,MAAM,MAAM,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAElD,WAAW;IACX,MAAM,OAAO,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;QACtC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IAEN,sBAAsB;IACtB,MAAM,iBAAiB,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,sBAAsB;IACtB,MAAM,iBAAiB,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,mCAAmC;IACnC,MAAM,YAAY,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAChF,MAAM,GAAG,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,QAAQ;QACR,SAAS;QACT,MAAM;QACN,OAAO;QACP,iBAAiB;QACjB,iBAAiB;QACjB,GAAG;KACJ,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAA0B;IAC9D,MAAM,SAAS,GAAG,OAAO;SACtB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;SACrB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,CAAC;YACT,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,CAAC;SACX,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;IACpC,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAEtC,qBAAqB;IACrB,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAClF,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAEzC,OAAO;QACL,GAAG;QACH,GAAG;QACH,IAAI;QACJ,MAAM;QACN,GAAG;QACH,GAAG;QACH,MAAM;QACN,OAAO,EAAE,GAAG;KACb,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,WAAqB,EAAE,CAAS;IAClD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACvC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC;IAE/B,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;QACpB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;AAC7E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAA0B;IACjE,4BAA4B;IAC5B,MAAM,UAAU,GAAG,IAAI,GAAG,EAA6B,CAAC;IAExD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC;QACrD,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChD,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,sCAAsC;IACtC,MAAM,eAAe,GAAsB,EAAE,CAAC;IAE9C,KAAK,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,UAAU,EAAE,CAAC;QACrD,MAAM,eAAe,GAAG,wBAAwB,CAAC,eAAe,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,8BAA8B,CAAC,eAAe,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,qBAAqB,CAAC,eAAe,CAAC,CAAC;QAEvD,eAAe,CAAC,IAAI,CAAC;YACnB,QAAQ;YACR,WAAW,EAAE,eAAe,CAAC,MAAM;YACnC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,eAAe;YACf,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAA0B;IAOnE,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,cAAc,KAAK,MAAM,CAAC,CAAC;IACnF,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAErF,MAAM,qBAAqB,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;QACpD,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QACtF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,sBAAsB,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;QACrD,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QACvF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,sBAAsB,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC;QACtD,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM;QACxF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,uBAAuB,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC;QACvD,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM;QACzF,CAAC,CAAC,CAAC,CAAC;IAEN,4DAA4D;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAC/C,MAAM,SAAS,GAAG,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAC9E,OAAO,SAAS,KAAK,CAAC,CAAC,cAAc,CAAC;IACxC,CAAC,CAAC,CAAC,MAAM,CAAC;IAEV,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC;QAC5C,CAAC,CAAC,gBAAgB,GAAG,YAAY,CAAC,MAAM;QACxC,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,qBAAqB;QACrB,sBAAsB;QACtB,sBAAsB;QACtB,uBAAuB;QACvB,cAAc;KACf,CAAC;AACJ,CAAC"}

package/dist/benchmark/runner.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Benchmark runner for evaluating Veto kernel performance.
+ *
+ * @module benchmark/runner
+ */
+import { type Logger } from '../utils/logger.js';
+import type { BenchmarkResult, BenchmarkReport, BenchmarkConfig } from './types.js';
+/**
+ * Progress callback for benchmark updates.
+ */
+export type ProgressCallback = (completed: number, total: number, current: BenchmarkResult | null, eta: number) => void;
+/**
+ * Options for running a benchmark.
+ */
+export interface BenchmarkRunnerOptions {
+    /** Benchmark configuration */
+    config: BenchmarkConfig;
+    /** Progress callback */
+    onProgress?: ProgressCallback;
+    /** Logger instance */
+    logger?: Logger;
+}
+/**
+ * Run a benchmark evaluation.
+ */
+export declare function runBenchmark(options: BenchmarkRunnerOptions): Promise<BenchmarkReport>;
+/**
+ * Format a benchmark report for console output.
+ */
+export declare function formatReportConsole(report: BenchmarkReport): string;
+/**
+ * Save benchmark report to JSON file.
+ */
+export declare function saveReportJson(report: BenchmarkReport, path: string): void;
+/**
+ * Create a default progress logger.
+ */
+export declare function createProgressLogger(): ProgressCallback;
+//# sourceMappingURL=runner.d.ts.map

package/dist/benchmark/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAgB,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAQ/D,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EACf,eAAe,EAChB,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,GAAG,IAAI,EAC/B,GAAG,EAAE,MAAM,KACR,IAAI,CAAC;AAEV;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,8BAA8B;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,wBAAwB;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,sBAAsB;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CA0J5F;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA8GnE;AAaD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,gBAAgB,CAkBvD"}