theorum 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -27
- package/docs/AGENT_PROFILE_CONTRACT.md +45 -18
- package/docs/CLI_SPEC.md +2 -2
- package/docs/SECRETS.md +27 -22
- package/esm/_dnt.polyfills.d.ts +99 -0
- package/esm/_dnt.polyfills.js +130 -1
- package/esm/mod.d.ts +19 -3
- package/esm/mod.js +17 -3
- package/esm/src/cli/commands/profile.d.ts +2 -0
- package/esm/src/cli/commands/profile.js +53 -0
- package/esm/src/cli/commands/run.d.ts +10 -0
- package/esm/src/cli/commands/run.js +55 -0
- package/esm/src/cli/commands/test.d.ts +15 -0
- package/esm/src/cli/commands/test.js +140 -0
- package/esm/src/cli/index.d.ts +4 -0
- package/esm/src/cli/index.js +148 -0
- package/esm/src/cli/matrix/fixtures.d.ts +13 -0
- package/esm/src/cli/matrix/fixtures.js +77 -0
- package/esm/src/cli/matrix/synthesizer.d.ts +29 -0
- package/esm/src/cli/matrix/synthesizer.js +187 -0
- package/esm/src/guardrails/error.d.ts +1 -3
- package/esm/src/guardrails/error.js +4 -6
- package/esm/src/guardrails/mod.d.ts +1 -1
- package/esm/src/guardrails/mod.js +1 -1
- package/esm/src/guardrails/sanitize.js +1 -1
- package/esm/src/host/mint-trace.d.ts +27 -0
- package/esm/src/host/mint-trace.js +33 -0
- package/esm/src/host/mod.d.ts +13 -0
- package/esm/src/host/mod.js +12 -0
- package/esm/src/host/reply.d.ts +14 -0
- package/esm/src/host/reply.js +27 -0
- package/esm/src/kernel/engine/delta.js +31 -4
- package/esm/src/kernel/engine/repair.d.ts +1 -3
- package/esm/src/kernel/engine/repair.js +1 -1
- package/esm/src/kernel/engine/runner/gates.d.ts +4 -0
- package/esm/src/kernel/engine/runner/gates.js +206 -0
- package/esm/src/kernel/engine/runner/mod.d.ts +14 -0
- package/esm/src/kernel/engine/runner/mod.js +103 -0
- package/esm/src/kernel/engine/runner/state.d.ts +16 -0
- package/esm/src/kernel/engine/runner/state.js +8 -0
- package/esm/src/kernel/engine/runner/steps.d.ts +15 -0
- package/esm/src/kernel/engine/runner/steps.js +119 -0
- package/esm/src/kernel/engine/runner/stream.d.ts +11 -0
- package/esm/src/kernel/engine/runner/stream.js +65 -0
- package/esm/src/kernel/engine/runner/tokens.d.ts +3 -0
- package/esm/src/kernel/engine/runner/tokens.js +38 -0
- package/esm/src/kernel/engine/runner/tools.d.ts +13 -0
- package/esm/src/kernel/engine/runner/tools.js +197 -0
- package/esm/src/kernel/engine/runner.d.ts +1 -9
- package/esm/src/kernel/engine/runner.js +1 -726
- package/esm/src/kernel/mod.d.ts +3 -1
- package/esm/src/kernel/mod.js +2 -1
- package/esm/src/{providers → kernel/registry}/attachments.d.ts +2 -3
- package/esm/src/{providers → kernel/registry}/attachments.js +6 -6
- package/esm/src/kernel/registry/catalog.d.ts +28 -21
- package/esm/src/kernel/registry/catalog.js +55 -133
- package/esm/src/kernel/registry/ingress.d.ts +13 -0
- package/esm/src/kernel/registry/ingress.js +163 -0
- package/esm/src/kernel/registry/profiles.d.ts +2 -4
- package/esm/src/kernel/registry/profiles.js +14 -5
- package/esm/src/kernel/registry/provider-request.d.ts +4 -0
- package/esm/src/kernel/registry/provider-request.js +25 -0
- package/esm/src/kernel/registry/resolve.d.ts +0 -1
- package/esm/src/kernel/registry/resolve.js +35 -53
- package/esm/src/kernel/registry/tools.js +2 -2
- package/esm/src/kernel/registry/vault.d.ts +11 -0
- package/esm/src/kernel/registry/vault.js +18 -0
- package/esm/src/kernel/types.d.ts +102 -78
- package/esm/src/observability/trace-attach.js +6 -20
- package/esm/src/presets/google.d.ts +50 -0
- package/esm/src/presets/google.js +96 -0
- package/esm/src/presets/mod.d.ts +11 -0
- package/esm/src/presets/mod.js +10 -0
- package/esm/src/providers/create-provider.d.ts +29 -0
- package/esm/src/providers/create-provider.js +38 -0
- package/esm/src/providers/gemini-tape.d.ts +1 -2
- package/esm/src/providers/gemini-tape.js +1 -1
- package/esm/src/providers/interactions.js +41 -16
- package/esm/src/{guardrails → providers}/keys.d.ts +10 -3
- package/esm/src/{guardrails → providers}/keys.js +10 -13
- package/esm/src/providers/mod.d.ts +4 -7
- package/esm/src/providers/mod.js +2 -5
- package/esm/src/providers/openrouter-mod.d.ts +13 -0
- package/esm/src/providers/openrouter-mod.js +12 -0
- package/esm/src/providers/openrouter-payload.d.ts +11 -3
- package/esm/src/providers/openrouter-payload.js +28 -26
- package/esm/src/providers/openrouter.d.ts +5 -7
- package/esm/src/providers/openrouter.js +488 -231
- package/esm/src/providers/pcm.d.ts +7 -0
- package/esm/src/providers/pcm.js +35 -0
- package/esm/src/providers/provider.d.ts +3 -1
- package/esm/src/providers/provider.js +45 -2
- package/esm/src/providers/speech.d.ts +23 -0
- package/esm/src/providers/speech.js +125 -0
- package/package.json +20 -3
- package/esm/src/providers/media.d.ts +0 -5
- package/esm/src/providers/media.js +0 -125
- package/esm/src/providers/tts.d.ts +0 -24
- package/esm/src/providers/tts.js +0 -144
package/esm/mod.js
CHANGED
|
@@ -11,7 +11,20 @@
|
|
|
11
11
|
*
|
|
12
12
|
* const profile = defineProfile({
|
|
13
13
|
* id: "assistant.basic",
|
|
14
|
-
* model: {
|
|
14
|
+
* model: {
|
|
15
|
+
* allow: ["gemini35FlashLite"],
|
|
16
|
+
* config: {
|
|
17
|
+
* gemini35FlashLite: {
|
|
18
|
+
* apiId: "gemini-3.5-flash-lite",
|
|
19
|
+
* thinking: { on: "high", off: "minimal" },
|
|
20
|
+
* thinkingLevels: ["minimal", "low", "medium", "high"],
|
|
21
|
+
* summaries: { on: "auto", off: "none" },
|
|
22
|
+
* maxOutputTokens: 8192,
|
|
23
|
+
* temperature: 1,
|
|
24
|
+
* keyBuiltins: [],
|
|
25
|
+
* },
|
|
26
|
+
* },
|
|
27
|
+
* },
|
|
15
28
|
* tools: { allow: [] },
|
|
16
29
|
* inputs: { text: true },
|
|
17
30
|
* outputs: {},
|
|
@@ -27,9 +40,10 @@ import "./_dnt.polyfills.js";
|
|
|
27
40
|
export { publicError, TheorumError } from './src/guardrails/error.js';
|
|
28
41
|
export { PROJECT_ID_MAX, sanitizeProjectId, sanitizeText, sanitizeTurnRequest, } from './src/guardrails/sanitize.js';
|
|
29
42
|
export { runTurn } from './src/kernel/engine/runner.js';
|
|
30
|
-
export {
|
|
43
|
+
export { CATALOG, clampThinkingLevel, clampThinkingLevelForApiId, getTool, listBuiltinIds, modelEntryByApiId, registerTools, requireModelSpec, resetTools, } from './src/kernel/registry/catalog.js';
|
|
44
|
+
export { clearProfiles, defineProfile, getProfile, hasProfile, listProfiles, registerProfile, registerProfiles, } from './src/kernel/registry/profiles.js';
|
|
31
45
|
export { projectProfile, resolveTurn } from './src/kernel/registry/resolve.js';
|
|
32
46
|
export { getStructured, registerStructured } from './src/kernel/registry/schemas.js';
|
|
33
47
|
export { executeTool } from './src/kernel/registry/tools.js';
|
|
34
48
|
export { jsonlSink, memorySink, noopSink, resolveTraceDir, sinkFromDir, writeTrace, } from './src/observability/trace.js';
|
|
35
|
-
export {
|
|
49
|
+
export { createProvider } from './src/providers/mod.js';
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { getProfile, listProfiles } from '../../kernel/registry/profiles.js';
|
|
2
|
+
function formatProfileInputs(p) {
|
|
3
|
+
const inputs = [];
|
|
4
|
+
if (p.inputs.text !== false)
|
|
5
|
+
inputs.push('text');
|
|
6
|
+
if (p.inputs.voice)
|
|
7
|
+
inputs.push('voice');
|
|
8
|
+
if (p.inputs.attachments)
|
|
9
|
+
inputs.push(`attachments [${p.inputs.attachments.accept?.join(', ')}]`);
|
|
10
|
+
return inputs.join(' | ');
|
|
11
|
+
}
|
|
12
|
+
function printProfileCard(p) {
|
|
13
|
+
const tools = p.tools.allow?.length ? p.tools.allow.join(', ') : 'none';
|
|
14
|
+
const models = p.model.allow?.join(', ') || 'default';
|
|
15
|
+
const structured = typeof p.outputs.structured === 'string'
|
|
16
|
+
? p.outputs.structured
|
|
17
|
+
: p.outputs.structured
|
|
18
|
+
? 'custom'
|
|
19
|
+
: 'none';
|
|
20
|
+
console.log(` • Profile: ${p.id.padEnd(16)} (handle: ${p.identity.handle})`);
|
|
21
|
+
console.log(` - Models: ${models}`);
|
|
22
|
+
console.log(` - Inputs: ${formatProfileInputs(p)}`);
|
|
23
|
+
console.log(` - Tools: ${tools}`);
|
|
24
|
+
console.log(` - Structured: ${structured}`);
|
|
25
|
+
console.log(` - Key Bucket: ${p.model.key ?? 'freeA'}`);
|
|
26
|
+
console.log('-'.repeat(70));
|
|
27
|
+
}
|
|
28
|
+
export function listProfilesCommand() {
|
|
29
|
+
const profiles = listProfiles();
|
|
30
|
+
console.log('\n Registered Theorum Profiles:');
|
|
31
|
+
console.log('='.repeat(70));
|
|
32
|
+
if (profiles.length === 0) {
|
|
33
|
+
console.log(' (No profiles registered in runtime)');
|
|
34
|
+
console.log(`${'='.repeat(70)}\n`);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
for (const p of profiles) {
|
|
38
|
+
printProfileCard(p);
|
|
39
|
+
}
|
|
40
|
+
console.log(`Total: ${profiles.length} profiles registered.\n`);
|
|
41
|
+
}
|
|
42
|
+
export function showProfileCommand(profileId) {
|
|
43
|
+
try {
|
|
44
|
+
const profile = getProfile(profileId);
|
|
45
|
+
console.log(`\n Profile Details: ${profileId}`);
|
|
46
|
+
console.log('='.repeat(70));
|
|
47
|
+
console.log(JSON.stringify(profile, null, 2));
|
|
48
|
+
console.log(`${'='.repeat(70)}\n`);
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
console.error(`\n Error: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ModelProvider } from '../../kernel/types.js';
|
|
2
|
+
export interface RunOptions {
|
|
3
|
+
profile: string;
|
|
4
|
+
prompt?: string;
|
|
5
|
+
mode?: string;
|
|
6
|
+
search?: boolean;
|
|
7
|
+
map?: boolean;
|
|
8
|
+
provider?: ModelProvider;
|
|
9
|
+
}
|
|
10
|
+
export declare function runCommand(options: RunOptions): Promise<void>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
|
+
import { runTurn } from '../../kernel/engine/runner.js';
|
|
3
|
+
import { getProfile } from '../../kernel/registry/profiles.js';
|
|
4
|
+
function handleRunEvent(event) {
|
|
5
|
+
if (event.type === 'thought' && event.text) {
|
|
6
|
+
dntShim.Deno.stdout.write(new TextEncoder().encode(`\x1b[2m${event.text}\x1b[0m`));
|
|
7
|
+
}
|
|
8
|
+
else if (event.type === 'text' && event.text) {
|
|
9
|
+
dntShim.Deno.stdout.write(new TextEncoder().encode(event.text));
|
|
10
|
+
}
|
|
11
|
+
else if (event.type === 'tool' && event.tool) {
|
|
12
|
+
console.log(`\n\x1b[33m⚡ [Tool Call] ${event.tool.name}\x1b[0m:`, event.tool.arguments);
|
|
13
|
+
}
|
|
14
|
+
else if (event.type === 'structured' && event.structured) {
|
|
15
|
+
console.log('\n\x1b[32m✓ [Structured Output]\x1b[0m:');
|
|
16
|
+
console.log(JSON.stringify(event.structured, null, 2));
|
|
17
|
+
}
|
|
18
|
+
else if (event.type === 'media' && event.media) {
|
|
19
|
+
console.log(`\n\x1b[34m🖼 [Media Output]\x1b[0m (${event.media.mimeType})`);
|
|
20
|
+
}
|
|
21
|
+
else if (event.type === 'error' && event.error) {
|
|
22
|
+
console.error(`\n\x1b[31m✗ Error\x1b[0m: ${event.error}`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export async function runCommand(options) {
|
|
26
|
+
getProfile(options.profile);
|
|
27
|
+
const provider = options.provider;
|
|
28
|
+
if (!provider) {
|
|
29
|
+
console.error('\n\x1b[31mExecution Failed\x1b[0m: Theorum CLI does not create providers or read keys. Run turns from a host app with an explicit ModelProvider.\n');
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const prompt = options.prompt || 'Hello! Please introduce your capabilities.';
|
|
33
|
+
const tools = {};
|
|
34
|
+
if (options.search)
|
|
35
|
+
tools.googleSearch = true;
|
|
36
|
+
if (options.map)
|
|
37
|
+
tools.googleMaps = true;
|
|
38
|
+
const req = {
|
|
39
|
+
profile: options.profile,
|
|
40
|
+
select: options.mode,
|
|
41
|
+
tools: Object.keys(tools).length > 0 ? tools : undefined,
|
|
42
|
+
input: { text: prompt },
|
|
43
|
+
};
|
|
44
|
+
console.log(`\n▶ [RUNNING TURN] Profile: ${options.profile} (${options.mode ?? 'default'})`);
|
|
45
|
+
console.log(`Prompt: "${prompt}"\n`);
|
|
46
|
+
try {
|
|
47
|
+
for await (const event of runTurn(req, provider)) {
|
|
48
|
+
handleRunEvent(event);
|
|
49
|
+
}
|
|
50
|
+
console.log('\n');
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
console.error(`\n\x1b[31mExecution Failed\x1b[0m: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ModelProvider, TurnRequest } from '../../kernel/types.js';
|
|
2
|
+
import { type MatrixOptions } from '../matrix/synthesizer.js';
|
|
3
|
+
export interface TestRunResult {
|
|
4
|
+
passed: boolean;
|
|
5
|
+
name: string;
|
|
6
|
+
durationSec: number;
|
|
7
|
+
tokensTotal?: number;
|
|
8
|
+
error?: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function executeSingleTest(req: TurnRequest, testName: string, provider?: ModelProvider): Promise<TestRunResult>;
|
|
11
|
+
export declare function testProfileCommand(profileId?: string, options?: MatrixOptions & {
|
|
12
|
+
all?: boolean;
|
|
13
|
+
matrix?: boolean;
|
|
14
|
+
provider?: ModelProvider;
|
|
15
|
+
}): Promise<boolean>;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
|
+
import { runTurn } from '../../kernel/engine/runner.js';
|
|
3
|
+
import { getProfile, listProfiles } from '../../kernel/registry/profiles.js';
|
|
4
|
+
import { buildCustomTurnRequest, synthesizeMatrixCombos, } from '../matrix/synthesizer.js';
|
|
5
|
+
function printTestHeader(req, testName) {
|
|
6
|
+
const activeTools = Object.entries(req.tools ?? {})
|
|
7
|
+
.filter(([, v]) => Boolean(v))
|
|
8
|
+
.map(([k]) => k)
|
|
9
|
+
.join(', ');
|
|
10
|
+
console.log(`\n▶ [THEORUM TEST] ${testName}`);
|
|
11
|
+
console.log(` Profile: ${req.profile} (Mode: ${req.select ?? 'default'})`);
|
|
12
|
+
console.log(` Tools: ${activeTools || 'none'}`);
|
|
13
|
+
console.log(` Attachments: ${req.input?.attachments?.length ?? 0} file(s) | Voice: ${req.input?.voice?.length ? 'yes' : 'no'}`);
|
|
14
|
+
console.log(` ${'-'.repeat(60)}`);
|
|
15
|
+
}
|
|
16
|
+
function processTestEvent(event, acc) {
|
|
17
|
+
if (event.type === 'thought' && event.text) {
|
|
18
|
+
dntShim.Deno.stdout.write(new TextEncoder().encode('.'));
|
|
19
|
+
}
|
|
20
|
+
else if (event.type === 'tool' && event.tool) {
|
|
21
|
+
console.log(`\n ⚡ [Tool Dispatched] ${event.tool.name}(${JSON.stringify(event.tool.arguments ?? {})})`);
|
|
22
|
+
}
|
|
23
|
+
else if (event.type === 'structured') {
|
|
24
|
+
console.log(`\n ✓ [Structured Schema Validated]`);
|
|
25
|
+
}
|
|
26
|
+
else if (event.type === 'media') {
|
|
27
|
+
console.log(`\n ✓ [Media Generated] (${event.media?.mimeType})`);
|
|
28
|
+
}
|
|
29
|
+
else if (event.type === 'tokens' && event.tokens) {
|
|
30
|
+
acc.totalTokens = event.tokens.total;
|
|
31
|
+
}
|
|
32
|
+
else if (event.type === 'error' && event.error) {
|
|
33
|
+
acc.hasError = true;
|
|
34
|
+
acc.errorMessage = event.error;
|
|
35
|
+
console.error(`\n [Test Error Detail]: ${event.error}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function printTestResult(passed, durationSec, acc) {
|
|
39
|
+
if (passed) {
|
|
40
|
+
console.log(`\n ✓ STATUS: PASSED (took ${durationSec.toFixed(2)}s, ${acc.totalTokens ? `${acc.totalTokens} tokens` : 'ok'})`);
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
console.log(`\n ✗ STATUS: FAILED: ${acc.errorMessage} (${durationSec.toFixed(2)}s)`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export async function executeSingleTest(req, testName, provider) {
|
|
47
|
+
const start = Date.now();
|
|
48
|
+
getProfile(req.profile);
|
|
49
|
+
printTestHeader(req, testName);
|
|
50
|
+
const acc = {
|
|
51
|
+
totalTokens: 0,
|
|
52
|
+
hasError: false,
|
|
53
|
+
errorMessage: '',
|
|
54
|
+
};
|
|
55
|
+
try {
|
|
56
|
+
if (!provider) {
|
|
57
|
+
throw new Error('Theorum CLI does not create providers or read keys. Pass an explicit ModelProvider from the host app.');
|
|
58
|
+
}
|
|
59
|
+
for await (const event of runTurn(req, provider)) {
|
|
60
|
+
processTestEvent(event, acc);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
acc.hasError = true;
|
|
65
|
+
acc.errorMessage = err instanceof Error ? err.message : String(err);
|
|
66
|
+
}
|
|
67
|
+
const durationSec = (Date.now() - start) / 1000;
|
|
68
|
+
const passed = !acc.hasError;
|
|
69
|
+
printTestResult(passed, durationSec, acc);
|
|
70
|
+
return {
|
|
71
|
+
passed,
|
|
72
|
+
name: testName,
|
|
73
|
+
durationSec,
|
|
74
|
+
tokensTotal: acc.totalTokens,
|
|
75
|
+
error: acc.errorMessage,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
function resolveTargetProfiles(profileId, all) {
|
|
79
|
+
if (all) {
|
|
80
|
+
return listProfiles();
|
|
81
|
+
}
|
|
82
|
+
if (profileId) {
|
|
83
|
+
try {
|
|
84
|
+
return [getProfile(profileId)];
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
console.error(`\n Error: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
console.error('Error: Please specify a profile ID (e.g. `theorum test --profile your-profile`) or `--all`.');
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
async function runProfileMatrix(profile, provider) {
|
|
95
|
+
const results = [];
|
|
96
|
+
const combos = synthesizeMatrixCombos(profile);
|
|
97
|
+
for (const combo of combos) {
|
|
98
|
+
const res = await executeSingleTest(combo.req, `${profile.id} [${combo.name}]`, provider);
|
|
99
|
+
results.push(res);
|
|
100
|
+
}
|
|
101
|
+
return results;
|
|
102
|
+
}
|
|
103
|
+
async function runProfileSingle(profile, options, provider) {
|
|
104
|
+
const req = buildCustomTurnRequest(profile, options);
|
|
105
|
+
const testName = options.lite ? `${profile.id} [Lite]` : `${profile.id} [Stress Combo]`;
|
|
106
|
+
return await executeSingleTest(req, testName, provider);
|
|
107
|
+
}
|
|
108
|
+
function printSummaryReport(results) {
|
|
109
|
+
console.log(`\n${'='.repeat(70)}`);
|
|
110
|
+
console.log(' TEST SUMMARY:');
|
|
111
|
+
console.log('='.repeat(70));
|
|
112
|
+
let allPassed = true;
|
|
113
|
+
for (const r of results) {
|
|
114
|
+
const status = r.passed ? '✓ PASSED' : '✗ FAILED';
|
|
115
|
+
console.log(` ${status.padEnd(10)} ${r.name.padEnd(45)} (${r.durationSec.toFixed(2)}s)`);
|
|
116
|
+
if (!r.passed) {
|
|
117
|
+
allPassed = false;
|
|
118
|
+
if (r.error)
|
|
119
|
+
console.log(` Reason: ${r.error}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
console.log(`${'='.repeat(70)}\n`);
|
|
123
|
+
return allPassed;
|
|
124
|
+
}
|
|
125
|
+
export async function testProfileCommand(profileId, options = {}) {
|
|
126
|
+
const targetProfiles = resolveTargetProfiles(profileId, options.all);
|
|
127
|
+
if (!targetProfiles || targetProfiles.length === 0) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
const results = [];
|
|
131
|
+
for (const profile of targetProfiles) {
|
|
132
|
+
if (options.matrix) {
|
|
133
|
+
results.push(...(await runProfileMatrix(profile, options.provider)));
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
results.push(await runProfileSingle(profile, options, options.provider));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return printSummaryReport(results);
|
|
140
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import "../../_dnt.polyfills.js";
|
|
3
|
+
import "../../_dnt.polyfills.js";
|
|
4
|
+
import * as dntShim from "../../_dnt.shims.js";
|
|
5
|
+
import { listProfilesCommand, showProfileCommand } from './commands/profile.js';
|
|
6
|
+
import { runCommand } from './commands/run.js';
|
|
7
|
+
import { testProfileCommand } from './commands/test.js';
|
|
8
|
+
function printHelp() {
|
|
9
|
+
console.log(`
|
|
10
|
+
Theorum CLI - Profile Testing and Profile Inspection
|
|
11
|
+
|
|
12
|
+
USAGE:
|
|
13
|
+
theorum <command> [options]
|
|
14
|
+
|
|
15
|
+
COMMANDS:
|
|
16
|
+
test Run stress matrix or custom profile tests
|
|
17
|
+
--profile, -p <id> Target profile ID registered by your host app
|
|
18
|
+
--all, -a Test all registered profiles
|
|
19
|
+
--lite Minimal fast-path connectivity ping
|
|
20
|
+
--matrix Run all valid permutations for the profile
|
|
21
|
+
--mode <fast|smart> Reasoning mode override
|
|
22
|
+
--search Optional: set tool id googleSearch when allowlisted
|
|
23
|
+
--map Optional: set tool id googleMaps when allowlisted
|
|
24
|
+
|
|
25
|
+
run Execute a turn with real-time streaming output
|
|
26
|
+
--profile, -p <id> Target profile ID
|
|
27
|
+
--prompt <text> User prompt string
|
|
28
|
+
--mode <fast|smart> Reasoning mode
|
|
29
|
+
--search Optional: set tool id googleSearch when allowlisted
|
|
30
|
+
--map Optional: set tool id googleMaps when allowlisted
|
|
31
|
+
|
|
32
|
+
profile Inspect registered profile blueprints
|
|
33
|
+
list List all registered profiles
|
|
34
|
+
show <id> Show detailed JSON profile specification
|
|
35
|
+
|
|
36
|
+
help, --help, -h Show this help message
|
|
37
|
+
`);
|
|
38
|
+
}
|
|
39
|
+
function parseLongFlag(arg, nextArg, flags) {
|
|
40
|
+
const key = arg.slice(2);
|
|
41
|
+
if (nextArg && !nextArg.startsWith('-')) {
|
|
42
|
+
flags[key] = nextArg;
|
|
43
|
+
return 1;
|
|
44
|
+
}
|
|
45
|
+
flags[key] = true;
|
|
46
|
+
return 0;
|
|
47
|
+
}
|
|
48
|
+
function parseShortFlag(arg, nextArg, flags) {
|
|
49
|
+
const key = arg.slice(1);
|
|
50
|
+
if (key === 'p' && nextArg) {
|
|
51
|
+
flags.profile = nextArg;
|
|
52
|
+
return 1;
|
|
53
|
+
}
|
|
54
|
+
if (key === 'a')
|
|
55
|
+
flags.all = true;
|
|
56
|
+
else if (key === 'h')
|
|
57
|
+
flags.help = true;
|
|
58
|
+
else
|
|
59
|
+
flags[key] = true;
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
function parseFlags(args) {
|
|
63
|
+
const flags = { _: [] };
|
|
64
|
+
let i = 0;
|
|
65
|
+
while (i < args.length) {
|
|
66
|
+
const arg = args[i];
|
|
67
|
+
const next = args[i + 1];
|
|
68
|
+
if (arg.startsWith('--')) {
|
|
69
|
+
i += 1 + parseLongFlag(arg, next, flags);
|
|
70
|
+
}
|
|
71
|
+
else if (arg.startsWith('-')) {
|
|
72
|
+
i += 1 + parseShortFlag(arg, next, flags);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
flags._.push(arg);
|
|
76
|
+
i += 1;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return flags;
|
|
80
|
+
}
|
|
81
|
+
function extractProfileId(flags) {
|
|
82
|
+
if (typeof flags.profile === 'string')
|
|
83
|
+
return flags.profile;
|
|
84
|
+
if (typeof flags.p === 'string')
|
|
85
|
+
return flags.p;
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
async function handleTest(flags) {
|
|
89
|
+
const profile = extractProfileId(flags);
|
|
90
|
+
const success = await testProfileCommand(profile, {
|
|
91
|
+
all: Boolean(flags.all || flags.a),
|
|
92
|
+
lite: Boolean(flags.lite),
|
|
93
|
+
matrix: Boolean(flags.matrix),
|
|
94
|
+
mode: typeof flags.mode === 'string' ? flags.mode : undefined,
|
|
95
|
+
search: flags.search ? true : undefined,
|
|
96
|
+
map: flags.map ? true : undefined,
|
|
97
|
+
});
|
|
98
|
+
if (!success) {
|
|
99
|
+
dntShim.Deno.exit(1);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async function handleRun(flags) {
|
|
103
|
+
const profile = extractProfileId(flags);
|
|
104
|
+
if (!profile) {
|
|
105
|
+
console.error('Error: Profile ID required (e.g. `theorum run --profile your-profile`)');
|
|
106
|
+
dntShim.Deno.exit(1);
|
|
107
|
+
}
|
|
108
|
+
const prompt = typeof flags.prompt === 'string' ? flags.prompt : flags._.slice(1).join(' ');
|
|
109
|
+
await runCommand({
|
|
110
|
+
profile,
|
|
111
|
+
prompt,
|
|
112
|
+
mode: typeof flags.mode === 'string' ? flags.mode : undefined,
|
|
113
|
+
search: Boolean(flags.search),
|
|
114
|
+
map: Boolean(flags.map),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
function handleProfile(flags) {
|
|
118
|
+
const sub = flags._[1] || 'list';
|
|
119
|
+
if (sub === 'list') {
|
|
120
|
+
listProfilesCommand();
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const id = sub === 'show' ? flags._[2] || flags.profile : sub;
|
|
124
|
+
if (typeof id !== 'string' || !id) {
|
|
125
|
+
console.error('Error: Profile ID required (e.g. `theorum profile show your-profile`)');
|
|
126
|
+
dntShim.Deno.exit(1);
|
|
127
|
+
}
|
|
128
|
+
showProfileCommand(id);
|
|
129
|
+
}
|
|
130
|
+
export async function main(cliArgs = dntShim.Deno.args) {
|
|
131
|
+
const flags = parseFlags(cliArgs);
|
|
132
|
+
const command = flags._[0] || (flags.help ? 'help' : 'help');
|
|
133
|
+
if (command === 'test') {
|
|
134
|
+
await handleTest(flags);
|
|
135
|
+
}
|
|
136
|
+
else if (command === 'run') {
|
|
137
|
+
await handleRun(flags);
|
|
138
|
+
}
|
|
139
|
+
else if (command === 'profile') {
|
|
140
|
+
handleProfile(flags);
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
printHelp();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (globalThis[Symbol.for("import-meta-ponyfill-esmodule")](import.meta).main) {
|
|
147
|
+
await main();
|
|
148
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in synthetic media fixtures for zero-dependency matrix testing.
|
|
3
|
+
*/
|
|
4
|
+
export declare const FIXTURE_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
|
|
5
|
+
export declare const FIXTURE_PDF_BASE64: string;
|
|
6
|
+
export declare function createSyntheticWavBase64(durationSec?: number, sampleRate?: number): string;
|
|
7
|
+
export declare const FIXTURE_WAV_BASE64: string;
|
|
8
|
+
export declare const FIXTURE_CSV_BASE64: string;
|
|
9
|
+
export declare const FIXTURE_TEXT_BASE64: string;
|
|
10
|
+
export declare function getFixtureForMime(mime: string): {
|
|
11
|
+
data: string;
|
|
12
|
+
mimeType: string;
|
|
13
|
+
} | undefined;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in synthetic media fixtures for zero-dependency matrix testing.
|
|
3
|
+
*/
|
|
4
|
+
// 1x1 pixel PNG (base64)
|
|
5
|
+
export const FIXTURE_PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
6
|
+
// Minimal valid PDF-1.4 document (base64)
|
|
7
|
+
export const FIXTURE_PDF_BASE64 = btoa('%PDF-1.4\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n2 0 obj<</Type/Pages/Count 1/Kids[3 0 R]>>endobj\n3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<<>>>>endobj\nxref\n0 4\n0000000000 65535 f \n0000000009 00000 n \n0000000056 00000 n \n0000000111 00000 n \ntrailer<</Size 4/Root 1 0 R>>\nstartxref\n188\n%%EOF\n');
|
|
8
|
+
// Minimal valid 16kHz 16-bit mono PCM/WAV (0.1s tone) (base64)
|
|
9
|
+
export function createSyntheticWavBase64(durationSec = 0.1, sampleRate = 16000) {
|
|
10
|
+
const numSamples = Math.floor(sampleRate * durationSec);
|
|
11
|
+
const dataSize = numSamples * 2; // 16-bit = 2 bytes per sample
|
|
12
|
+
const buffer = new ArrayBuffer(44 + dataSize);
|
|
13
|
+
const view = new DataView(buffer);
|
|
14
|
+
// RIFF identifier
|
|
15
|
+
view.setUint8(0, 0x52); // R
|
|
16
|
+
view.setUint8(1, 0x49); // I
|
|
17
|
+
view.setUint8(2, 0x46); // F
|
|
18
|
+
view.setUint8(3, 0x46); // F
|
|
19
|
+
view.setUint32(4, 36 + dataSize, true); // file size - 8
|
|
20
|
+
view.setUint8(8, 0x57); // W
|
|
21
|
+
view.setUint8(9, 0x41); // A
|
|
22
|
+
view.setUint8(10, 0x56); // V
|
|
23
|
+
view.setUint8(11, 0x45); // E
|
|
24
|
+
// fmt subchunk
|
|
25
|
+
view.setUint8(12, 0x66); // f
|
|
26
|
+
view.setUint8(13, 0x6d); // m
|
|
27
|
+
view.setUint8(14, 0x74); // t
|
|
28
|
+
view.setUint8(15, 0x20); // ' '
|
|
29
|
+
view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM)
|
|
30
|
+
view.setUint16(20, 1, true); // AudioFormat (1 = PCM)
|
|
31
|
+
view.setUint16(22, 1, true); // NumChannels (1 = Mono)
|
|
32
|
+
view.setUint32(24, sampleRate, true); // SampleRate
|
|
33
|
+
view.setUint32(28, sampleRate * 2, true); // ByteRate (SampleRate * NumChannels * BitsPerSample/8)
|
|
34
|
+
view.setUint16(32, 2, true); // BlockAlign (NumChannels * BitsPerSample/8)
|
|
35
|
+
view.setUint16(34, 16, true); // BitsPerSample (16 bits)
|
|
36
|
+
// data subchunk
|
|
37
|
+
view.setUint8(36, 0x64); // d
|
|
38
|
+
view.setUint8(37, 0x61); // a
|
|
39
|
+
view.setUint8(38, 0x74); // t
|
|
40
|
+
view.setUint8(39, 0x61); // a
|
|
41
|
+
view.setUint32(40, dataSize, true);
|
|
42
|
+
// Write a simple sine wave (440Hz)
|
|
43
|
+
const freq = 440;
|
|
44
|
+
for (let i = 0; i < numSamples; i++) {
|
|
45
|
+
const t = i / sampleRate;
|
|
46
|
+
const sample = Math.sin(2 * Math.PI * freq * t) * 0.2 * 32767;
|
|
47
|
+
view.setInt16(44 + i * 2, sample, true);
|
|
48
|
+
}
|
|
49
|
+
const bytes = new Uint8Array(buffer);
|
|
50
|
+
let binary = '';
|
|
51
|
+
for (let i = 0; i < bytes.byteLength; i++) {
|
|
52
|
+
binary += String.fromCharCode(bytes[i]);
|
|
53
|
+
}
|
|
54
|
+
return btoa(binary);
|
|
55
|
+
}
|
|
56
|
+
export const FIXTURE_WAV_BASE64 = createSyntheticWavBase64(0.1, 16000);
|
|
57
|
+
export const FIXTURE_CSV_BASE64 = btoa('item_id,label,category,score\nitem_1,Alpha,test,0.95\nitem_2,Beta,test,0.88\n');
|
|
58
|
+
export const FIXTURE_TEXT_BASE64 = btoa('This is a synthetic fixture text document for Theorum validation testing.\n');
|
|
59
|
+
export function getFixtureForMime(mime) {
|
|
60
|
+
const m = mime.toLowerCase().trim();
|
|
61
|
+
if (m.startsWith('image/')) {
|
|
62
|
+
return { data: FIXTURE_PNG_BASE64, mimeType: 'image/png' };
|
|
63
|
+
}
|
|
64
|
+
if (m === 'application/pdf') {
|
|
65
|
+
return { data: FIXTURE_PDF_BASE64, mimeType: 'application/pdf' };
|
|
66
|
+
}
|
|
67
|
+
if (m === 'audio/wav' || m === 'audio/x-wav' || m.startsWith('audio/')) {
|
|
68
|
+
return { data: FIXTURE_WAV_BASE64, mimeType: 'audio/wav' };
|
|
69
|
+
}
|
|
70
|
+
if (m === 'text/csv') {
|
|
71
|
+
return { data: FIXTURE_CSV_BASE64, mimeType: 'text/csv' };
|
|
72
|
+
}
|
|
73
|
+
if (m === 'text/plain') {
|
|
74
|
+
return { data: FIXTURE_TEXT_BASE64, mimeType: 'text/plain' };
|
|
75
|
+
}
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stress / matrix turn synthesis for the THEORUM CLI.
|
|
3
|
+
*
|
|
4
|
+
* Tool enabling walks the profile allowlist and registered catalog metadata
|
|
5
|
+
* (`conflictsWith`). Google `--search` / `--map` flags remain as convenience
|
|
6
|
+
* aliases for `googleSearch` / `googleMaps` when those ids are allowed.
|
|
7
|
+
*
|
|
8
|
+
* @module
|
|
9
|
+
*/
|
|
10
|
+
import type { Profile, ToolId, TurnRequest } from '../../kernel/types.js';
|
|
11
|
+
export interface MatrixOptions {
|
|
12
|
+
lite?: boolean;
|
|
13
|
+
/** Convenience alias for enabling/disabling `googleSearch` when allowed. */
|
|
14
|
+
search?: boolean;
|
|
15
|
+
/** Convenience alias for enabling/disabling `googleMaps` when allowed. */
|
|
16
|
+
map?: boolean;
|
|
17
|
+
mode?: string;
|
|
18
|
+
attachmentPaths?: string[];
|
|
19
|
+
voicePath?: string;
|
|
20
|
+
}
|
|
21
|
+
export declare function synthesizeLiteCombo(profile: Profile): TurnRequest;
|
|
22
|
+
export declare function synthesizeStressCombo(profile: Profile, options?: {
|
|
23
|
+
preferTool?: ToolId;
|
|
24
|
+
}): TurnRequest;
|
|
25
|
+
export declare function synthesizeMatrixCombos(profile: Profile): Array<{
|
|
26
|
+
name: string;
|
|
27
|
+
req: TurnRequest;
|
|
28
|
+
}>;
|
|
29
|
+
export declare function buildCustomTurnRequest(profile: Profile, options: MatrixOptions): TurnRequest;
|