theorum 0.1.15 → 1.0.0

package/esm/src/cli/commands/run.js

@@ -1,27 +1,6 @@
-import * as dntShim from "../../../_dnt.shims.js";
 import { runTurn } from '../../kernel/engine/runner.js';
 import { getProfile } from '../../kernel/registry/profiles.js';
-function handleRunEvent(event) {
-    if (event.type === 'thought' && event.text) {
-        dntShim.Deno.stdout.write(new TextEncoder().encode(`\x1b[2m${event.text}\x1b[0m`));
-    }
-    else if (event.type === 'text' && event.text) {
-        dntShim.Deno.stdout.write(new TextEncoder().encode(event.text));
-    }
-    else if (event.type === 'tool' && event.tool) {
-        console.log(`\n\x1b[33m⚡ [Tool Call] ${event.tool.name}\x1b[0m:`, event.tool.arguments);
-    }
-    else if (event.type === 'structured' && event.structured) {
-        console.log('\n\x1b[32m✓ [Structured Output]\x1b[0m:');
-        console.log(JSON.stringify(event.structured, null, 2));
-    }
-    else if (event.type === 'media' && event.media) {
-        console.log(`\n\x1b[34m🖼 [Media Output]\x1b[0m (${event.media.mimeType})`);
-    }
-    else if (event.type === 'error' && event.error) {
-        console.error(`\n\x1b[31m✗ Error\x1b[0m: ${event.error}`);
-    }
-}
+import { createCliTraceCapture, printRunEvent, printTraceRecord } from '../event-log.js';
 export async function runCommand(options) {
     getProfile(options.profile);
     const provider = options.provider;
@@ -30,24 +9,37 @@ export async function runCommand(options) {
         return;
     }
     const prompt = options.prompt || 'Hello! Please introduce your capabilities.';
-    const tools = {};
-    if (options.search)
-        tools.googleSearch = true;
-    if (options.map)
-        tools.googleMaps = true;
+    if (options.search || options.map) {
+        const profile = getProfile(options.profile);
+        const selected = options.mode
+            ? (profile.model.select?.[options.mode] ?? profile.model.allow[0])
+            : profile.model.allow[0];
+        const builtins = new Set(profile.model.config[selected]?.builtInTools ?? []);
+        if (options.search && !builtins.has('googleSearch')) {
+            console.error('\n\x1b[31mExecution Failed\x1b[0m: --search requires googleSearch on model.config.*.builtInTools for the selected model.\n');
+            return;
+        }
+        if (options.map && !builtins.has('googleMaps')) {
+            console.error('\n\x1b[31mExecution Failed\x1b[0m: --map requires googleMaps on model.config.*.builtInTools for the selected model.\n');
+            return;
+        }
+    }
     const req = {
         profile: options.profile,
         select: options.mode,
-        tools: Object.keys(tools).length > 0 ? tools : undefined,
         input: { text: prompt },
     };
     console.log(`\n▶ [RUNNING TURN] Profile: ${options.profile} (${options.mode ?? 'default'})`);
     console.log(`Prompt: "${prompt}"\n`);
+    const traceCapture = options.trace ? createCliTraceCapture(options.traceDir) : undefined;
     try {
-        for await (const event of runTurn(req, provider)) {
-            handleRunEvent(event);
+        for await (const event of runTurn(req, provider, traceCapture?.sink)) {
+            printRunEvent(event, { verbose: options.verbose });
         }
         console.log('\n');
+        if (options.trace) {
+            printTraceRecord(traceCapture?.records.at(-1), options.verbose === true);
+        }
     }
     catch (err) {
         console.error(`\n\x1b[31mExecution Failed\x1b[0m: ${err instanceof Error ? err.message : String(err)}\n`);

package/esm/src/cli/commands/test.d.ts

@@ -7,9 +7,18 @@ export interface TestRunResult {
     tokensTotal?: number;
     error?: string;
 }
-export declare function executeSingleTest(req: TurnRequest, testName: string, provider?: ModelProvider): Promise<TestRunResult>;
+interface CliTestOptions {
+    verbose?: boolean;
+    trace?: boolean;
+    traceDir?: string;
+}
+export declare function executeSingleTest(req: TurnRequest, testName: string, provider?: ModelProvider, cliOptions?: CliTestOptions): Promise<TestRunResult>;
 export declare function testProfileCommand(profileId?: string, options?: MatrixOptions & {
     all?: boolean;
     matrix?: boolean;
     provider?: ModelProvider;
+    verbose?: boolean;
+    trace?: boolean;
+    traceDir?: string;
 }): Promise<boolean>;
+export {};

package/esm/src/cli/commands/test.js

@@ -1,39 +1,30 @@
-import * as dntShim from "../../../_dnt.shims.js";
 import { runTurn } from '../../kernel/engine/runner.js';
 import { getProfile, listProfiles } from '../../kernel/registry/profiles.js';
+import { createCliTraceCapture, printTestEvent, printTraceRecord } from '../event-log.js';
 import { buildCustomTurnRequest, synthesizeMatrixCombos, } from '../matrix/synthesizer.js';
 function printTestHeader(req, testName) {
-    const activeTools = Object.entries(req.tools ?? {})
-        .filter(([, v]) => Boolean(v))
-        .map(([k]) => k)
-        .join(', ');
+    const profile = getProfile(req.profile);
+    const modelId = req.select && profile.model.select?.[req.select]
+        ? profile.model.select[req.select]
+        : profile.model.allow[0];
+    const customs = profile.type === 'speech' ? 'none' : profile.tools.allow.join(', ') || 'none';
+    const builtins = (profile.model.config[modelId]?.builtInTools ?? []).join(', ') || 'none';
     console.log(`\n▶ [THEORUM TEST] ${testName}`);
     console.log(`  Profile:     ${req.profile} (Mode: ${req.select ?? 'default'})`);
-    console.log(`  Tools:       ${activeTools || 'none'}`);
+    console.log(`  Custom:      ${customs}`);
+    console.log(`  Builtins:    ${builtins}`);
     console.log(`  Attachments: ${req.input?.attachments?.length ?? 0} file(s) | Voice: ${req.input?.voice?.length ? 'yes' : 'no'}`);
     console.log(`  ${'-'.repeat(60)}`);
 }
-function processTestEvent(event, acc) {
-    if (event.type === 'thought' && event.text) {
-        dntShim.Deno.stdout.write(new TextEncoder().encode('.'));
-    }
-    else if (event.type === 'tool' && event.tool) {
-        console.log(`\n  ⚡ [Tool Dispatched] ${event.tool.name}(${JSON.stringify(event.tool.arguments ?? {})})`);
-    }
-    else if (event.type === 'structured') {
-        console.log(`\n  ✓ [Structured Schema Validated]`);
-    }
-    else if (event.type === 'media') {
-        console.log(`\n  ✓ [Media Generated] (${event.media?.mimeType})`);
-    }
-    else if (event.type === 'tokens' && event.tokens) {
+function processTestEvent(event, acc, options) {
+    if (event.type === 'tokens' && event.tokens) {
         acc.totalTokens = event.tokens.total;
     }
-    else if (event.type === 'error' && event.error) {
+    if (event.type === 'error' && event.error) {
         acc.hasError = true;
         acc.errorMessage = event.error;
-        console.error(`\n  [Test Error Detail]: ${event.error}`);
     }
+    printTestEvent(event, { verbose: options.verbose });
 }
 function printTestResult(passed, durationSec, acc) {
     if (passed) {
@@ -43,7 +34,7 @@ function printTestResult(passed, durationSec, acc) {
         console.log(`\n  ✗ STATUS: FAILED: ${acc.errorMessage} (${durationSec.toFixed(2)}s)`);
     }
 }
-export async function executeSingleTest(req, testName, provider) {
+export async function executeSingleTest(req, testName, provider, cliOptions = {}) {
     const start = Date.now();
     getProfile(req.profile);
     printTestHeader(req, testName);
@@ -56,8 +47,12 @@ export async function executeSingleTest(req, testName, provider) {
         if (!provider) {
             throw new Error('Theorum CLI does not create providers or read keys. Pass an explicit ModelProvider from the host app.');
         }
-        for await (const event of runTurn(req, provider)) {
-            processTestEvent(event, acc);
+        const traceCapture = cliOptions.trace ? createCliTraceCapture(cliOptions.traceDir) : undefined;
+        for await (const event of runTurn(req, provider, traceCapture?.sink)) {
+            processTestEvent(event, acc, cliOptions);
+        }
+        if (cliOptions.trace) {
+            printTraceRecord(traceCapture?.records.at(-1), cliOptions.verbose === true);
         }
     }
     catch (err) {
@@ -91,19 +86,19 @@ function resolveTargetProfiles(profileId, all) {
     console.error('Error: Please specify a profile ID (e.g. `theorum test --profile your-profile`) or `--all`.');
     return null;
 }
-async function runProfileMatrix(profile, provider) {
+async function runProfileMatrix(profile, provider, cliOptions = {}) {
     const results = [];
     const combos = synthesizeMatrixCombos(profile);
     for (const combo of combos) {
-        const res = await executeSingleTest(combo.req, `${profile.id} [${combo.name}]`, provider);
+        const res = await executeSingleTest(combo.req, `${profile.id} [${combo.name}]`, provider, cliOptions);
         results.push(res);
     }
     return results;
 }
-async function runProfileSingle(profile, options, provider) {
+async function runProfileSingle(profile, options, provider, cliOptions = {}) {
     const req = buildCustomTurnRequest(profile, options);
     const testName = options.lite ? `${profile.id} [Lite]` : `${profile.id} [Stress Combo]`;
-    return await executeSingleTest(req, testName, provider);
+    return await executeSingleTest(req, testName, provider, cliOptions);
 }
 function printSummaryReport(results) {
     console.log(`\n${'='.repeat(70)}`);
@@ -127,13 +122,18 @@ export async function testProfileCommand(profileId, options = {}) {
     if (!targetProfiles || targetProfiles.length === 0) {
         return false;
     }
+    const cliOptions = {
+        verbose: options.verbose,
+        trace: options.trace,
+        traceDir: options.traceDir,
+    };
     const results = [];
     for (const profile of targetProfiles) {
         if (options.matrix) {
-            results.push(...(await runProfileMatrix(profile, options.provider)));
+            results.push(...(await runProfileMatrix(profile, options.provider, cliOptions)));
         }
         else {
-            results.push(await runProfileSingle(profile, options, options.provider));
+            results.push(await runProfileSingle(profile, options, options.provider, cliOptions));
         }
     }
     return printSummaryReport(results);

package/esm/src/cli/event-log.d.ts

@@ -0,0 +1,19 @@
+import type { TurnEvent } from '../kernel/types.js';
+import { type TraceSink } from '../observability/trace.js';
+import type { TraceRecord } from '../observability/trace-record.js';
+export interface CliEventLogOptions {
+    verbose?: boolean;
+}
+export interface CliTraceCapture {
+    sink: TraceSink;
+    records: TraceRecord[];
+}
+/** Attach an in-memory trace sink; optionally mirror to a JSONL directory. */
+declare function createCliTraceCapture(traceDir?: string): CliTraceCapture;
+/** Print one turn event for `theorum run`. */
+declare function printRunEvent(event: TurnEvent, options?: CliEventLogOptions): void;
+/** Print one turn event for `theorum test`. */
+declare function printTestEvent(event: TurnEvent, options?: CliEventLogOptions): void;
+/** Dump the last captured trace record after a CLI turn. */
+declare function printTraceRecord(record: TraceRecord | undefined, verbose: boolean): void;
+export { createCliTraceCapture, printRunEvent, printTestEvent, printTraceRecord };

package/esm/src/cli/event-log.js

@@ -0,0 +1,147 @@
+/**
+ * Shared CLI turn-event printing and trace capture for `run` and `test`.
+ *
+ * @module
+ */
+import * as dntShim from "../../_dnt.shims.js";
+import { jsonlSink, memorySink } from '../observability/trace.js';
+/** Attach an in-memory trace sink; optionally mirror to a JSONL directory. */
+function createCliTraceCapture(traceDir) {
+    const records = [];
+    const sinks = [memorySink(records)];
+    if (traceDir?.trim()) {
+        sinks.push(jsonlSink(traceDir.trim()));
+    }
+    return {
+        records,
+        sink: {
+            write: async (record) => {
+                for (const sink of sinks) {
+                    await sink.write(record);
+                }
+            },
+        },
+    };
+}
+function printVerboseEvidence(event) {
+    if (event.type !== 'evidence' || !event.evidence) {
+        return;
+    }
+    const e = event.evidence;
+    if (e.raw) {
+        console.log('\n\x1b[2m[verbose evidence.raw]\x1b[0m');
+        console.log(JSON.stringify(e.raw, null, 2));
+    }
+}
+function printVerboseError(event) {
+    if (event.type !== 'error' || !event.errorInternal) {
+        return;
+    }
+    console.error(`\n\x1b[2m[verbose errorInternal]\x1b[0m ${event.errorInternal}`);
+}
+function printRunEvidence(event, verbose) {
+    const e = event.evidence;
+    if (!e) {
+        return;
+    }
+    if (e.kind === 'code_execution_call') {
+        console.log(`\n\x1b[36m🐍 [code_execution_call]\x1b[0m\n${e.code ?? ''}`);
+    }
+    else if (e.kind === 'code_execution_result') {
+        console.log(`\n\x1b[36m🐍 [code_execution_result]\x1b[0m isError=${String(e.isError)}\n${e.result ?? ''}`);
+    }
+    else if (e.kind) {
+        console.log(`\n\x1b[36m📎 [evidence]\x1b[0m ${e.kind}`);
+    }
+    if (verbose) {
+        printVerboseEvidence(event);
+    }
+}
+/** Print one turn event for `theorum run`. */
+function printRunEvent(event, options = {}) {
+    const verbose = options.verbose === true;
+    if (event.type === 'thought' && event.text) {
+        dntShim.Deno.stdout.write(new TextEncoder().encode(`\x1b[2m${event.text}\x1b[0m`));
+    }
+    else if (event.type === 'text' && event.text) {
+        dntShim.Deno.stdout.write(new TextEncoder().encode(event.text));
+    }
+    else if (event.type === 'tool' && event.tool) {
+        console.log(`\n\x1b[33m⚡ [Tool Call] ${event.tool.name}\x1b[0m:`, event.tool.arguments);
+    }
+    else if (event.type === 'evidence') {
+        printRunEvidence(event, verbose);
+    }
+    else if (event.type === 'structured' && event.structured) {
+        console.log('\n\x1b[32m✓ [Structured Output]\x1b[0m:');
+        console.log(JSON.stringify(event.structured, null, 2));
+    }
+    else if (event.type === 'media' && event.media) {
+        console.log(`\n\x1b[34m🖼 [Media Output]\x1b[0m (${event.media.mimeType})`);
+    }
+    else if (event.type === 'error' && event.error) {
+        console.error(`\n\x1b[31m✗ Error\x1b[0m: ${event.error}`);
+        if (verbose) {
+            printVerboseError(event);
+        }
+    }
+}
+function printTestEvidence(event, verbose) {
+    const e = event.evidence;
+    if (!e) {
+        return;
+    }
+    if (e.kind === 'code_execution_call') {
+        const preview = (e.code ?? '').replaceAll('\n', ' ').slice(0, 80);
+        console.log(`\n  🐍 [code_execution_call] ${preview || e.id || ''}`);
+    }
+    else if (e.kind === 'code_execution_result') {
+        const preview = (e.result ?? '').replaceAll('\n', ' ').slice(0, 80);
+        console.log(`\n  🐍 [code_execution_result] isError=${String(e.isError)} ${preview}`);
+    }
+    else if (e.kind) {
+        console.log(`\n  📎 [evidence] ${e.kind}`);
+    }
+    if (verbose) {
+        printVerboseEvidence(event);
+    }
+}
+/** Print one turn event for `theorum test`. */
+function printTestEvent(event, options = {}) {
+    const verbose = options.verbose === true;
+    if (event.type === 'thought' && event.text) {
+        dntShim.Deno.stdout.write(new TextEncoder().encode('.'));
+    }
+    else if (event.type === 'tool' && event.tool) {
+        console.log(`\n  ⚡ [Tool Dispatched] ${event.tool.name}(${JSON.stringify(event.tool.arguments ?? {})})`);
+    }
+    else if (event.type === 'evidence') {
+        printTestEvidence(event, verbose);
+    }
+    else if (event.type === 'structured') {
+        console.log('\n  ✓ [Structured Schema Validated]');
+    }
+    else if (event.type === 'media') {
+        console.log(`\n  ✓ [Media Generated] (${event.media?.mimeType})`);
+    }
+    else if (event.type === 'error' && event.error) {
+        console.error(`\n  [Test Error Detail]: ${event.error}`);
+        if (verbose) {
+            printVerboseError(event);
+        }
+    }
+}
+/** Dump the last captured trace record after a CLI turn. */
+function printTraceRecord(record, verbose) {
+    if (!record) {
+        console.error('\n\x1b[33m[trace]\x1b[0m No trace record captured for this turn.');
+        return;
+    }
+    console.log('\n\x1b[35m════ TRACE RECORD ════\x1b[0m');
+    console.log(JSON.stringify(record, null, 2));
+    if (verbose && record.upstreamLog) {
+        console.log('\n\x1b[35m════ UPSTREAM LOG ════\x1b[0m');
+        console.log(JSON.stringify(record.upstreamLog, null, 2));
+    }
+}
+export { createCliTraceCapture, printRunEvent, printTestEvent, printTraceRecord };

package/esm/src/cli/index.js

@@ -3,6 +3,7 @@ import "../../_dnt.polyfills.js";
 import "../../_dnt.polyfills.js";
 import * as dntShim from "../../_dnt.shims.js";
 import { benchCommand } from './commands/bench.js';
+import { fuzzCanaryCommand } from './commands/fuzz-canary.js';
 import { fuzzGuardrailsCommand } from './commands/fuzz-guardrails.js';
 import { listProfilesCommand, showProfileCommand } from './commands/profile.js';
 import { runCommand } from './commands/run.js';
@@ -20,7 +21,18 @@ COMMANDS:
     --iterations <n>   Measurement iterations (default: 50)
     --warmup <n>       Warmup iterations (default: 5)
 
-  fuzz                 Adversarial guardrail fuzzer
+  fuzz                 Adversarial inbound sanitization fuzzer
+  fuzz-canary          Adversarial canary egress fuzzer (stream + Live gates)
+
+  run                  Execute a turn with real-time streaming output
+    --profile, -p <id> Target profile ID
+    --prompt <text>    User prompt string
+    --mode <fast|smart> Reasoning mode
+    --search           Requires googleSearch on the selected model's builtInTools
+    --map              Requires googleMaps on the selected model's builtInTools
+    --verbose, -v      Print errorInternal and evidence.raw while running
+    --trace            Capture a TraceRecord and print it after the turn
+    --trace-dir <path> Also append trace JSONL under this directory
 
   test                 Run stress matrix or custom profile tests
     --profile, -p <id> Target profile ID registered by your host app
@@ -28,15 +40,11 @@ COMMANDS:
     --lite             Minimal fast-path connectivity ping
     --matrix           Run all valid permutations for the profile
     --mode <fast|smart> Reasoning mode override
-    --search           Optional: set tool id googleSearch when allowlisted
-    --map              Optional: set tool id googleMaps when allowlisted
-
-  run                  Execute a turn with real-time streaming output
-    --profile, -p <id> Target profile ID
-    --prompt <text>    User prompt string
-    --mode <fast|smart> Reasoning mode
-    --search           Optional: set tool id googleSearch when allowlisted
-    --map              Optional: set tool id googleMaps when allowlisted
+    --search           Requires googleSearch on the selected model's builtInTools
+    --map              Requires googleMaps on the selected model's builtInTools
+    --verbose, -v      Print errorInternal and evidence.raw while running
+    --trace            Capture a TraceRecord and print it after each turn
+    --trace-dir <path> Also append trace JSONL under this directory
 
   profile              Inspect registered profile blueprints
     list               List all registered profiles
@@ -45,6 +53,13 @@ COMMANDS:
   help, --help, -h     Show this help message
 `);
 }
+function cliDiagnostics(flags) {
+    return {
+        verbose: Boolean(flags.verbose || flags.v),
+        trace: Boolean(flags.trace),
+        traceDir: typeof flags['trace-dir'] === 'string' ? flags['trace-dir'] : undefined,
+    };
+}
 function parseLongFlag(arg, nextArg, flags) {
     const key = arg.slice(2);
     if (nextArg && !nextArg.startsWith('-')) {
@@ -64,6 +79,8 @@ function parseShortFlag(arg, nextArg, flags) {
         flags.all = true;
     else if (key === 'h')
         flags.help = true;
+    else if (key === 'v')
+        flags.verbose = true;
     else
         flags[key] = true;
     return 0;
@@ -96,6 +113,7 @@ function extractProfileId(flags) {
 }
 async function handleTest(flags) {
     const profile = extractProfileId(flags);
+    const diagnostics = cliDiagnostics(flags);
     const success = await testProfileCommand(profile, {
         all: Boolean(flags.all || flags.a),
         lite: Boolean(flags.lite),
@@ -103,6 +121,9 @@ async function handleTest(flags) {
         mode: typeof flags.mode === 'string' ? flags.mode : undefined,
         search: flags.search ? true : undefined,
         map: flags.map ? true : undefined,
+        verbose: diagnostics.verbose,
+        trace: diagnostics.trace,
+        traceDir: diagnostics.traceDir,
     });
     if (!success) {
         dntShim.Deno.exit(1);
@@ -115,12 +136,16 @@ async function handleRun(flags) {
         dntShim.Deno.exit(1);
     }
     const prompt = typeof flags.prompt === 'string' ? flags.prompt : flags._.slice(1).join(' ');
+    const diagnostics = cliDiagnostics(flags);
     await runCommand({
         profile,
         prompt,
         mode: typeof flags.mode === 'string' ? flags.mode : undefined,
         search: Boolean(flags.search),
         map: Boolean(flags.map),
+        verbose: diagnostics.verbose,
+        trace: diagnostics.trace,
+        traceDir: diagnostics.traceDir,
     });
 }
 function handleProfile(flags) {
@@ -140,7 +165,16 @@ export async function main(cliArgs = dntShim.Deno.args) {
     const flags = parseFlags(cliArgs);
     const command = flags._[0] || (flags.help ? 'help' : 'help');
     if (command === 'fuzz') {
-        fuzzGuardrailsCommand();
+        const ok = fuzzGuardrailsCommand();
+        if (!ok) {
+            dntShim.Deno.exit(1);
+        }
+    }
+    else if (command === 'fuzz-canary') {
+        const ok = await fuzzCanaryCommand();
+        if (!ok) {
+            dntShim.Deno.exit(1);
+        }
     }
     else if (command === 'bench') {
         await benchCommand({

package/esm/src/cli/matrix/synthesizer.d.ts

@@ -1,27 +1,25 @@
 /**
  * Stress / matrix turn synthesis for the THEORUM CLI.
  *
- * Tool enabling walks the profile allowlist and registered catalog metadata
- * (`conflictsWith`). Google `--search` / `--map` flags remain as convenience
- * aliases for `googleSearch` / `googleMaps` when those ids are allowed.
+ * Custom tools come from `profile.tools.allow` (visibility via loadTier).
+ * Provider builtins come from `model.config.*.builtInTools`. `--search` /
+ * `--map` only verify those ids are listed on the selected model.
  *
  * @module
  */
-import type { Profile, ToolId, TurnRequest } from '../../kernel/types.js';
+import type { Profile, TurnRequest } from '../../kernel/types.js';
 export interface MatrixOptions {
     lite?: boolean;
-    /** Convenience alias for enabling/disabling `googleSearch` when allowed. */
+    /** Requires googleSearch on the selected model's builtInTools. */
     search?: boolean;
-    /** Convenience alias for enabling/disabling `googleMaps` when allowed. */
+    /** Requires googleMaps on the selected model's builtInTools. */
     map?: boolean;
     mode?: string;
     attachmentPaths?: string[];
     voicePath?: string;
 }
 export declare function synthesizeLiteCombo(profile: Profile): TurnRequest;
-export declare function synthesizeStressCombo(profile: Profile, options?: {
-    preferTool?: ToolId;
-}): TurnRequest;
+export declare function synthesizeStressCombo(profile: Profile): TurnRequest;
 export declare function synthesizeMatrixCombos(profile: Profile): Array<{
     name: string;
     req: TurnRequest;