theorum 0.1.7 → 0.1.10

package/esm/src/cli/commands/bench.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Synthetic performance benchmark for the THEORUM kernel pipeline.
+ *
+ * Measures overhead added by profile resolution, sanitization, canary
+ * binding, stream processing, and event dispatch vs. a bare provider call.
+ *
+ * Metrics:
+ *   TTFE  — time to first event (profile resolve + provider setup)
+ *   TTFT  — time to first text delta
+ *   T/s   — text tokens per second throughput
+ *   Overhead — total wall-clock delta vs. raw provider consumption
+ *
+ * @module
+ */
+export interface BenchOptions {
+    chunks?: number;
+    iterations?: number;
+    warmup?: number;
+}
+export declare function benchCommand(options?: BenchOptions): Promise<void>;

package/esm/src/cli/commands/bench.js

@@ -0,0 +1,454 @@
+/**
+ * Synthetic performance benchmark for the THEORUM kernel pipeline.
+ *
+ * Measures overhead added by profile resolution, sanitization, canary
+ * binding, stream processing, and event dispatch vs. a bare provider call.
+ *
+ * Metrics:
+ *   TTFE  — time to first event (profile resolve + provider setup)
+ *   TTFT  — time to first text delta
+ *   T/s   — text tokens per second throughput
+ *   Overhead — total wall-clock delta vs. raw provider consumption
+ *
+ * @module
+ */
+import { sanitizeTurnRequest } from '../../guardrails/sanitize.js';
+import { bindCanary, eventHasCanary, mintCanary } from '../../kernel/engine/boundary.js';
+import { runTurn } from '../../kernel/engine/runner.js';
+import { clearProfiles, registerProfile, } from '../../kernel/registry/profiles.js';
+import { pickSystemRole, resolveTurn } from '../../kernel/registry/resolve.js';
+import { buildRecord } from '../../observability/trace-record.js';
+const DEFAULT_CHUNKS = 200;
+const DEFAULT_ITERATIONS = 50;
+const DEFAULT_WARMUP = 5;
+const MS_PER_SEC = 1000;
+const BENCH_PROFILE_ID = '__bench__';
+function registerBenchProfile() {
+    registerProfile({
+        id: BENCH_PROFILE_ID,
+        identity: {
+            handle: 'bench',
+            system: 'You are a benchmark stub.',
+        },
+        model: {
+            protocol: 'openAi',
+            provider: 'openrouter',
+            allow: ['bench-model'],
+            config: {
+                'bench-model': {
+                    apiId: 'bench-model',
+                    thinking: { on: 'none', off: 'none' },
+                    thinkingLevels: ['none'],
+                    summaries: { on: 'none', off: 'none' },
+                    maxOutputTokens: 4096,
+                    temperature: 0,
+                    keyBuiltins: [],
+                },
+            },
+            thinking: 'none',
+        },
+        guardrails: {
+            canary: true,
+            sanitizeInput: true,
+            redactSensitive: true,
+        },
+    });
+}
+function generateChunks(count) {
+    const events = [];
+    for (let i = 0; i < count; i++) {
+        events.push({ type: 'text', text: `token_${i} ` });
+    }
+    events.push({
+        type: 'tokens',
+        tokens: { input: 10, output: count, total: 10 + count },
+    });
+    events.push({ type: 'done' });
+    return events;
+}
+function createMockProvider(chunks) {
+    return {
+        async *complete(_req) {
+            for (const chunk of chunks) {
+                yield chunk;
+            }
+        },
+    };
+}
+function buildBenchRequest() {
+    return {
+        profile: BENCH_PROFILE_ID,
+        input: { text: 'Benchmark prompt.' },
+    };
+}
+async function measureRawProvider(provider) {
+    const req = buildBenchRequest();
+    const start = performance.now();
+    let firstEvent = 0;
+    let firstText = 0;
+    let textEvents = 0;
+    let gotFirst = false;
+    let gotFirstText = false;
+    // Simulate what a bare consumer does — no kernel overhead
+    const fakeReq = {
+        model: 'bench-model',
+        apiId: 'bench-model',
+        thinking: 'none',
+        summaries: 'none',
+        maxOutputTokens: 4096,
+        temperature: 0,
+        builtins: [],
+        system: '',
+        input: [{ type: 'text', text: req.input?.text ?? '' }],
+        structured: null,
+        image: null,
+    };
+    for await (const event of provider.complete(fakeReq)) {
+        if (!gotFirst) {
+            firstEvent = performance.now() - start;
+            gotFirst = true;
+        }
+        if (!gotFirstText && event.type === 'text') {
+            firstText = performance.now() - start;
+            gotFirstText = true;
+        }
+        if (event.type === 'text') {
+            textEvents++;
+        }
+    }
+    return {
+        ttfe: firstEvent,
+        ttft: firstText,
+        totalMs: performance.now() - start,
+        textEvents,
+    };
+}
+async function measureKernelPipeline(provider) {
+    const req = buildBenchRequest();
+    const start = performance.now();
+    let firstEvent = 0;
+    let firstText = 0;
+    let textEvents = 0;
+    let gotFirst = false;
+    let gotFirstText = false;
+    for await (const event of runTurn(req, provider)) {
+        if (!gotFirst) {
+            firstEvent = performance.now() - start;
+            gotFirst = true;
+        }
+        if (!gotFirstText && event.type === 'text') {
+            firstText = performance.now() - start;
+            gotFirstText = true;
+        }
+        if (event.type === 'text') {
+            textEvents++;
+        }
+    }
+    return {
+        ttfe: firstEvent,
+        ttft: firstText,
+        totalMs: performance.now() - start,
+        textEvents,
+    };
+}
+function percentile(sorted, p) {
+    const idx = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, idx)];
+}
+function aggregate(results) {
+    const ttfe = results.map((r) => r.ttfe).sort((a, b) => a - b);
+    const ttft = results.map((r) => r.ttft).sort((a, b) => a - b);
+    const total = results.map((r) => r.totalMs).sort((a, b) => a - b);
+    const tps = results
+        .map((r) => (r.textEvents / r.totalMs) * MS_PER_SEC)
+        .sort((a, b) => a - b);
+    const mean = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length;
+    return {
+        ttfeMs: {
+            median: percentile(ttfe, 50),
+            p95: percentile(ttfe, 95),
+            mean: mean(ttfe),
+        },
+        ttftMs: {
+            median: percentile(ttft, 50),
+            p95: percentile(ttft, 95),
+            mean: mean(ttft),
+        },
+        tokensPerSec: {
+            median: percentile(tps, 50),
+            p95: percentile(tps, 5), // lower is worse for throughput
+            mean: mean(tps),
+        },
+        totalMs: {
+            median: percentile(total, 50),
+            p95: percentile(total, 95),
+            mean: mean(total),
+        },
+    };
+}
+function fmtMs(ms) {
+    if (ms < 1) {
+        return `${(ms * MS_PER_SEC).toFixed(0)}µs`;
+    }
+    return `${ms.toFixed(2)}ms`;
+}
+function fmtRate(tps) {
+    if (tps >= MS_PER_SEC * MS_PER_SEC) {
+        return `${(tps / (MS_PER_SEC * MS_PER_SEC)).toFixed(1)}M/s`;
+    }
+    if (tps >= MS_PER_SEC) {
+        return `${(tps / MS_PER_SEC).toFixed(1)}K/s`;
+    }
+    return `${tps.toFixed(0)}/s`;
+}
+function printMetricRow(label, raw, fmt) {
+    console.log(`  ${label.padEnd(14)} median ${fmt(raw.median).padStart(10)}   p95 ${fmt(raw.p95).padStart(10)}   mean ${fmt(raw.mean).padStart(10)}`);
+}
+function printResults(label, metrics) {
+    console.log(`\n${label}`);
+    console.log('─'.repeat(72));
+    printMetricRow('TTFE', metrics.ttfeMs, fmtMs);
+    printMetricRow('TTFT', metrics.ttftMs, fmtMs);
+    printMetricRow('Throughput', metrics.tokensPerSec, fmtRate);
+    printMetricRow('Total', metrics.totalMs, fmtMs);
+}
+function printOverhead(raw, kernel) {
+    console.log('\nOverhead (kernel − raw)');
+    console.log('─'.repeat(72));
+    const ttfeDelta = kernel.ttfeMs.median - raw.ttfeMs.median;
+    const ttftDelta = kernel.ttftMs.median - raw.ttftMs.median;
+    const totalDelta = kernel.totalMs.median - raw.totalMs.median;
+    const tpsDelta = kernel.tokensPerSec.median - raw.tokensPerSec.median;
+    console.log(`  TTFE           ${ttfeDelta >= 0 ? '+' : ''}${fmtMs(ttfeDelta)}`);
+    console.log(`  TTFT           ${ttftDelta >= 0 ? '+' : ''}${fmtMs(ttftDelta)}`);
+    console.log(`  Throughput     ${tpsDelta >= 0 ? '+' : ''}${fmtRate(tpsDelta)}`);
+    console.log(`  Total          ${totalDelta >= 0 ? '+' : ''}${fmtMs(totalDelta)}`);
+    if (raw.totalMs.median > 0) {
+        const pct = ((totalDelta / raw.totalMs.median) * 100).toFixed(1);
+        console.log(`  Relative       ${totalDelta >= 0 ? '+' : ''}${pct}%`);
+    }
+}
+function measureSetupPhases() {
+    const req = buildBenchRequest();
+    const t0 = performance.now();
+    const safe = sanitizeTurnRequest(req);
+    const t1 = performance.now();
+    const { profile, generation } = resolveTurn(safe);
+    const t2 = performance.now();
+    pickSystemRole(profile, safe.input?.role);
+    const sys = profile.identity.system ?? '';
+    const t3 = performance.now();
+    bindCanary(sys, generation.canary);
+    const t4 = performance.now();
+    return {
+        sanitizeMs: t1 - t0,
+        resolveMs: t2 - t1,
+        systemMs: t3 - t2,
+        canaryMs: t4 - t3,
+        totalSetupMs: t4 - t0,
+    };
+}
+function printPhaseBreakdown(iterations) {
+    const results = [];
+    for (let i = 0; i < iterations; i++) {
+        results.push(measureSetupPhases());
+    }
+    const median = (arr) => {
+        const s = arr.slice().sort((a, b) => a - b);
+        return percentile(s, 50);
+    };
+    const sanitize = median(results.map((r) => r.sanitizeMs));
+    const resolve = median(results.map((r) => r.resolveMs));
+    const system = median(results.map((r) => r.systemMs));
+    const canary = median(results.map((r) => r.canaryMs));
+    const total = median(results.map((r) => r.totalSetupMs));
+    console.log('\nSetup Phase Breakdown (median)');
+    console.log('─'.repeat(72));
+    console.log(`  sanitizeTurnRequest    ${fmtMs(sanitize).padStart(10)}`);
+    console.log(`  resolveTurn            ${fmtMs(resolve).padStart(10)}`);
+    console.log(`  pickSystemRole         ${fmtMs(system).padStart(10)}`);
+    console.log(`  bindCanary             ${fmtMs(canary).padStart(10)}`);
+    console.log(`  ─────────────────────────────`);
+    console.log(`  Total setup            ${fmtMs(total).padStart(10)}`);
+}
+function microCanaryCheck(chunks, iterations) {
+    const canary = mintCanary();
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for (const chunk of chunks) {
+            eventHasCanary(chunk, canary);
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'eventHasCanary',
+        totalMs: elapsed,
+        perEventNs: (elapsed / total) * 1e6,
+    };
+}
+function microArrayPush(chunks, iterations) {
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        const seen = [];
+        const allEmitted = [];
+        const attempt = [];
+        for (const chunk of chunks) {
+            seen.push(chunk);
+            allEmitted.push(chunk);
+            attempt.push(chunk);
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'Array.push ×3',
+        totalMs: elapsed,
+        perEventNs: (elapsed / total) * 1e6,
+    };
+}
+async function microAsyncGenOverhead(chunks, iterations) {
+    async function* layer1(events) {
+        for (const e of events)
+            yield e;
+    }
+    async function* layer2(events) {
+        for await (const e of layer1(events))
+            yield e;
+    }
+    async function* layer3(events) {
+        for await (const e of layer2(events))
+            yield e;
+    }
+    async function* layer4(events) {
+        for await (const e of layer3(events))
+            yield e;
+    }
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for await (const _e of layer4(chunks)) {
+            // consume
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'AsyncGen ×4 layers',
+        totalMs: elapsed,
+        perEventNs: (elapsed / total) * 1e6,
+    };
+}
+function microAbortCheck(chunks, iterations) {
+    const controller = new AbortController();
+    const { signal } = controller;
+    const total = chunks.length * iterations;
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        for (const _chunk of chunks) {
+            if (signal.aborted)
+                throw signal.reason;
+        }
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'signal.aborted check',
+        totalMs: elapsed,
+        perEventNs: (elapsed / total) * 1e6,
+    };
+}
+async function microTraceRecord(chunks, iterations) {
+    const req = buildBenchRequest();
+    const start = performance.now();
+    for (let i = 0; i < iterations; i++) {
+        await buildRecord({
+            req,
+            events: chunks,
+            started: Date.now(),
+            model: 'bench-model',
+        });
+    }
+    const elapsed = performance.now() - start;
+    return {
+        label: 'buildRecord (trace)',
+        totalMs: elapsed,
+        perEventNs: (elapsed / iterations) * 1e6,
+    };
+}
+function microSanitizeScaling() {
+    const sizes = [50, 200, 1000, 5000];
+    console.log('\nSanitize Scaling (input text length)');
+    console.log('─'.repeat(72));
+    for (const size of sizes) {
+        const text = 'A'.repeat(size);
+        const req = {
+            profile: BENCH_PROFILE_ID,
+            input: { text },
+        };
+        const runs = 500;
+        const start = performance.now();
+        for (let i = 0; i < runs; i++) {
+            sanitizeTurnRequest(req);
+        }
+        const elapsed = performance.now() - start;
+        const perCall = elapsed / runs;
+        console.log(`  ${String(size).padStart(5)} chars          ${fmtMs(perCall).padStart(10)}/call`);
+    }
+}
+function fmtNs(ns) {
+    if (ns < 1000) {
+        return `${ns.toFixed(0)}ns`;
+    }
+    return `${(ns / 1000).toFixed(1)}µs`;
+}
+async function printMicroBenchmarks(chunks, iterations) {
+    const results = [];
+    results.push(microCanaryCheck(chunks, iterations));
+    results.push(microArrayPush(chunks, iterations));
+    results.push(await microAsyncGenOverhead(chunks, iterations));
+    results.push(microAbortCheck(chunks, iterations));
+    results.push(await microTraceRecord(chunks, Math.min(iterations, 20)));
+    console.log('\nPer-Event Micro-Benchmarks');
+    console.log('─'.repeat(72));
+    for (const r of results) {
+        const perEvent = r.label === 'buildRecord (trace)'
+            ? `${fmtMs(r.totalMs / Math.min(iterations, 20)).padStart(10)}/turn`
+            : `${fmtNs(r.perEventNs).padStart(8)}/event`;
+        console.log(`  ${r.label.padEnd(24)} ${perEvent}    (${fmtMs(r.totalMs)} total)`);
+    }
+    microSanitizeScaling();
+}
+export async function benchCommand(options = {}) {
+    const chunkCount = options.chunks ?? DEFAULT_CHUNKS;
+    const iterations = options.iterations ?? DEFAULT_ITERATIONS;
+    const warmup = options.warmup ?? DEFAULT_WARMUP;
+    console.log(`\n⏱  Theorum Kernel Benchmark`);
+    console.log(`   ${chunkCount} chunks × ${iterations} iterations (${warmup} warmup)\n`);
+    const chunks = generateChunks(chunkCount);
+    const provider = createMockProvider(chunks);
+    registerBenchProfile();
+    // Warmup
+    for (let i = 0; i < warmup; i++) {
+        await measureRawProvider(provider);
+        await measureKernelPipeline(provider);
+    }
+    // Collect raw baseline
+    const rawResults = [];
+    for (let i = 0; i < iterations; i++) {
+        rawResults.push(await measureRawProvider(provider));
+    }
+    // Collect kernel pipeline
+    const kernelResults = [];
+    for (let i = 0; i < iterations; i++) {
+        kernelResults.push(await measureKernelPipeline(provider));
+    }
+    const rawMetrics = aggregate(rawResults);
+    const kernelMetrics = aggregate(kernelResults);
+    printResults('Raw Provider (baseline)', rawMetrics);
+    printResults('Kernel Pipeline (runTurn)', kernelMetrics);
+    printOverhead(rawMetrics, kernelMetrics);
+    printPhaseBreakdown(iterations);
+    await printMicroBenchmarks(chunks, iterations);
+    // Cleanup
+    clearProfiles();
+    console.log('\n');
+}

package/esm/src/cli/commands/fuzz-guardrails.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Adversarial guardrail fuzzer.
+ *
+ * Throws injection payloads at every sanitization ingress and reports
+ * what gets through unchanged — meaning the guardrails missed it.
+ *
+ * @module
+ */
+export declare function fuzzGuardrailsCommand(): void;