theorum 0.1.2 → 0.1.4

package/src/kernel/engine/runner/stream.ts

@@ -0,0 +1,83 @@
+import { publicError } from '../../../guardrails/error.ts';
+import { providerCompleteRequest } from '../../registry/provider-request.ts';
+import type { ModelProvider, Profile, ResolvedGeneration, TurnEvent } from '../../types.ts';
+import { eventHasCanary, redactCanary } from '../boundary.ts';
+import { dispatchModelTool } from './tools.ts';
+
+function systemFromProfile(profile: Profile, role: string): string {
+  const { identity } = profile;
+  const { systemByRole, system } = identity;
+  if (systemByRole) {
+    const byRole = systemByRole[role];
+    if (byRole) {
+      return byRole;
+    }
+  }
+  if (system) {
+    return system;
+  }
+  return '';
+}
+
+function* interceptProviderTool(
+  event: TurnEvent,
+  profile: Profile,
+  generation: ResolvedGeneration,
+): Generator<TurnEvent> {
+  const tool = event.tool;
+  if (!tool) {
+    return;
+  }
+  const isDynamic = generation.dynamicTools?.some((d) => d.name === tool.name);
+  if (isDynamic) {
+    yield event;
+  } else {
+    for (const item of dispatchModelTool(profile, event, generation.custom)) {
+      yield item;
+    }
+  }
+}
+
+function shouldSkipStreamEvent(event: TurnEvent, profile: Profile): boolean {
+  return event.type === 'thought' && profile.outputs.streaming?.streamThoughts === false;
+}
+
+function* processNormalEvent(
+  event: TurnEvent,
+  profile: Profile,
+  generation: ResolvedGeneration,
+): Generator<TurnEvent> {
+  if (event.type === 'tool') {
+    yield* interceptProviderTool(event, profile, generation);
+  } else if (event.type === 'error') {
+    yield { type: 'error', error: publicError(event.error) };
+  } else {
+    yield event;
+  }
+}
+
+async function* yieldProviderEvents(args: {
+  profile: Profile;
+  generation: ResolvedGeneration;
+  system: string;
+  provider: ModelProvider;
+  gemini: Record<string, unknown>[];
+}): AsyncGenerator<TurnEvent> {
+  const { profile, generation, system, provider, gemini } = args;
+  const { canary } = generation;
+  for await (const event of provider.complete({
+    ...providerCompleteRequest(generation, system),
+    tapGemini: (row) => {
+      gemini.push(row);
+    },
+  })) {
+    if (canary && eventHasCanary(event, canary)) {
+      yield redactCanary(event, canary);
+      yield { type: 'error', error: publicError('canary leaked') };
+      return;
+    }
+    yield* processNormalEvent(event, profile, generation);
+  }
+}
+
+export { shouldSkipStreamEvent, systemFromProfile, yieldProviderEvents };

package/src/kernel/engine/runner/tokens.ts

@@ -0,0 +1,47 @@
+import type { TurnEvent, TurnRequest } from '../../types.ts';
+
+function calculateInputChars(safe: TurnRequest, system: string): number {
+  const input = safe.input ?? {};
+  return (
+    (input.text?.length ?? 0) +
+    (input.repair?.previousOutput?.length ?? 0) +
+    (system?.length ?? 800)
+  );
+}
+
+function calculateOutputChars(events: TurnEvent[]): { outputChars: number; thinkingChars: number } {
+  let outputChars = 0;
+  let thinkingChars = 0;
+  for (const e of events) {
+    if (e.type === 'text' && e.text) outputChars += e.text.length;
+    if (e.type === 'structured' && e.structured) {
+      outputChars += JSON.stringify(e.structured).length;
+    }
+    if (e.type === 'thought' && e.text) thinkingChars += e.text.length;
+  }
+  return { outputChars, thinkingChars };
+}
+
+function* calculateFallbackTokens(
+  safe: TurnRequest,
+  system: string,
+  events: TurnEvent[],
+): Generator<TurnEvent> {
+  const inputChars = calculateInputChars(safe, system);
+  const { outputChars, thinkingChars } = calculateOutputChars(events);
+  const inputTokens = Math.max(1, Math.round(inputChars / 4));
+  const outputTokens = Math.max(1, Math.round(outputChars / 4));
+  const thinkingTokens = thinkingChars ? Math.round(thinkingChars / 4) : 0;
+  yield {
+    type: 'tokens',
+    tokens: {
+      input: inputTokens,
+      output: outputTokens,
+      thinking: thinkingTokens,
+      toolUse: 0,
+      total: inputTokens + outputTokens + thinkingTokens,
+    },
+  };
+}
+
+export { calculateFallbackTokens };

package/src/kernel/engine/runner/tools.ts

@@ -0,0 +1,266 @@
+import { publicError } from '../../../guardrails/error.ts';
+import { CATALOG } from '../../registry/catalog.ts';
+import { executeTool } from '../../registry/tools.ts';
+import type {
+  CustomToolId,
+  DynamicToolDeclaration,
+  Profile,
+  ResolvedGeneration,
+  ToolEnvelope,
+  ToolId,
+  TurnEvent,
+  TurnRequest,
+} from '../../types.ts';
+
+function formatToolFinding(res: ToolEnvelope): string {
+  if (res.finding) {
+    return res.finding;
+  }
+  if (res.data) {
+    return JSON.stringify(res.data);
+  }
+  return 'ok';
+}
+
+function* invokeFromUi(profile: Profile, req: TurnRequest): Generator<TurnEvent> {
+  const invoke = req.toolInvoke;
+  if (!invoke) {
+    return;
+  }
+  try {
+    const result = executeTool(profile, invoke.name, invoke.arguments);
+    yield {
+      type: 'tool',
+      tool: { name: invoke.name, arguments: invoke.arguments, result },
+    };
+    if (result.status === 'error') {
+      yield { type: 'error', error: publicError(formatToolFinding(result)) };
+      return;
+    }
+    yield { type: 'done' };
+  } catch (err) {
+    yield { type: 'error', error: publicError(err) };
+    yield { type: 'done' };
+  }
+}
+
+function executeCustomModelTool(
+  profile: Profile,
+  name: CustomToolId,
+  args: Record<string, unknown>,
+): TurnEvent[] {
+  try {
+    const result = executeTool(profile, name, args);
+    const events: TurnEvent[] = [
+      {
+        type: 'tool',
+        tool: { name, arguments: args, result },
+      },
+    ];
+    if (result.status === 'error') {
+      events.push({ type: 'error', error: publicError(formatToolFinding(result)) });
+    }
+    return events;
+  } catch (err) {
+    return [{ type: 'error', error: publicError(err) }];
+  }
+}
+
+function dispatchModelTool(profile: Profile, event: TurnEvent, gated: CustomToolId[]): TurnEvent[] {
+  const { tool } = event;
+  if (!tool) {
+    return [];
+  }
+  const name = tool.name as ToolId;
+  if (CATALOG.tools[name]?.kind === 'builtin') {
+    return [event];
+  }
+  if (!gated.includes(name as CustomToolId)) {
+    return [
+      {
+        type: 'error',
+        error: publicError(`Tool '${name}' is not gated on this turn`),
+      },
+    ];
+  }
+  const args = tool.arguments ?? {};
+  return executeCustomModelTool(profile, name as CustomToolId, args);
+}
+
+function findDynamicDeclaration(
+  tools: DynamicToolDeclaration[] | undefined,
+  name: string,
+): DynamicToolDeclaration | undefined {
+  return tools?.find((t) => t.name === name);
+}
+
+async function checkDynamicAuthorization(
+  decl: DynamicToolDeclaration,
+  args: Record<string, unknown>,
+  profile: Profile,
+  sessionPermissions?: string[],
+): Promise<ToolEnvelope | null> {
+  if (!decl.canExecute) {
+    return null;
+  }
+  try {
+    const decision = await decl.canExecute({
+      args,
+      profile,
+      sessionPermissions,
+    });
+    if (typeof decision === 'boolean' && !decision) {
+      return {
+        status: 'error',
+        finding: `Tool '${decl.name}' execution not authorized.`,
+      };
+    }
+    if (typeof decision === 'object' && decision !== null) {
+      return decision;
+    }
+    return null;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      status: 'error',
+      finding: `Authorization error for '${decl.name}': ${msg}`,
+    };
+  }
+}
+
+function checkDynamicPermissionTier(
+  decl: DynamicToolDeclaration,
+  args: Record<string, unknown>,
+  sessionPermissions?: string[],
+): ToolEnvelope | null {
+  if (decl.permissionTier === 'session_consent' || decl.permissionTier === 'always_confirm') {
+    const isGranted = sessionPermissions?.includes(decl.name) || sessionPermissions?.includes('*');
+    if (!isGranted) {
+      return {
+        status: 'pause',
+        finding: `Tool '${decl.name}' requires ${decl.permissionTier} authorization.`,
+        data: { tool: decl.name, permissionTier: decl.permissionTier, args },
+      };
+    }
+  }
+  return null;
+}
+
+async function runDynamicHandler(
+  decl: DynamicToolDeclaration,
+  args: Record<string, unknown>,
+): Promise<ToolEnvelope> {
+  if (!decl.handler) {
+    return {
+      status: 'ok',
+      finding: `${decl.name} accepted (no handler)`,
+      data: args,
+    };
+  }
+  try {
+    return await decl.handler(args);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return { status: 'error', finding: msg };
+  }
+}
+
+async function executeDynamicTool(
+  decl: DynamicToolDeclaration,
+  args: Record<string, unknown>,
+  profile: Profile,
+  sessionPermissions?: string[],
+): Promise<ToolEnvelope> {
+  const authEnvelope = await checkDynamicAuthorization(decl, args, profile, sessionPermissions);
+  if (authEnvelope) {
+    return authEnvelope;
+  }
+
+  const permissionEnvelope = checkDynamicPermissionTier(decl, args, sessionPermissions);
+  if (permissionEnvelope) {
+    return permissionEnvelope;
+  }
+
+  return runDynamicHandler(decl, args);
+}
+
+function mergeDynamicTools(
+  current: DynamicToolDeclaration[] | undefined,
+  loaded: DynamicToolDeclaration[],
+): DynamicToolDeclaration[] {
+  const merged = [...(current ?? [])];
+  for (const tool of loaded) {
+    const existing = merged.findIndex((item) => item.name === tool.name);
+    if (existing >= 0) {
+      merged[existing] = tool;
+    } else {
+      merged.push(tool);
+    }
+  }
+  return merged;
+}
+
+async function executeDynamicToolLoader(args: {
+  decl: DynamicToolDeclaration;
+  toolArgs: Record<string, unknown>;
+  profile: Profile;
+  generation: ResolvedGeneration;
+}): Promise<ToolEnvelope> {
+  const { decl, toolArgs, profile, generation } = args;
+  const loader = generation.dynamicToolLoader;
+  if (!loader) {
+    return {
+      status: 'error',
+      finding: `Tool '${decl.name}' is marked as a loader but no loader is configured.`,
+    };
+  }
+  const loaded = await loader({
+    name: decl.name,
+    args: toolArgs,
+    profile,
+    currentTools: generation.dynamicTools ?? [],
+    sessionPermissions: generation.sessionPermissions,
+  });
+  generation.dynamicTools = mergeDynamicTools(generation.dynamicTools, loaded);
+  return {
+    status: 'ok',
+    finding: `Loaded ${String(loaded.length)} dynamic tool schema(s).`,
+    data: { loadedTools: loaded.map((tool) => tool.name) },
+  };
+}
+
+function isActionableDynamicDeclaration(
+  decl: DynamicToolDeclaration | undefined,
+): decl is DynamicToolDeclaration {
+  return Boolean(
+    decl?.handler || decl?.canExecute || decl?.permissionTier || decl?.loadsDynamicTools,
+  );
+}
+
+async function executeDynamicDeclaration(args: {
+  decl: DynamicToolDeclaration;
+  toolArgs: Record<string, unknown>;
+  profile: Profile;
+  generation: ResolvedGeneration;
+}): Promise<ToolEnvelope> {
+  const { decl, toolArgs, profile, generation } = args;
+  const res = await executeDynamicTool(decl, toolArgs, profile, generation.sessionPermissions);
+  if (res.status === 'ok' && decl.loadsDynamicTools) {
+    return await executeDynamicToolLoader({
+      decl,
+      toolArgs,
+      profile,
+      generation,
+    });
+  }
+  return res;
+}
+
+export {
+  dispatchModelTool,
+  executeDynamicDeclaration,
+  findDynamicDeclaration,
+  formatToolFinding,
+  invokeFromUi,
+  isActionableDynamicDeclaration,
+};

package/src/kernel/engine/runner.ts

@@ -0,0 +1,7 @@
+/**
+ * Deterministic turn runner for THEORUM.
+ *
+ * @module
+ */
+
+export { runTurn } from './runner/mod.ts';

package/src/kernel/engine/tree.ts

@@ -0,0 +1,18 @@
+function mapStrings(value: unknown, map: (text: string) => string): unknown {
+  if (typeof value === 'string') {
+    return map(value);
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => mapStrings(item, map));
+  }
+  if (value && typeof value === 'object') {
+    const out: Record<string, unknown> = {};
+    for (const [key, nested] of Object.entries(value)) {
+      out[key] = mapStrings(nested, map);
+    }
+    return out;
+  }
+  return value;
+}
+
+export { mapStrings };

package/src/kernel/mod.ts

@@ -0,0 +1,39 @@
+/**
+ * Profile, turn, tool, provider, egress, and event contracts for THEORUM.
+ *
+ * This entrypoint is type-first. Host applications import from here when they
+ * want to declare profiles, tool schemas, provider adapters, turn requests, or
+ * trace-safe event handlers without importing provider implementations.
+ *
+ * @module
+ */
+
+export { runTurn } from './engine/runner.ts';
+export {
+  CATALOG,
+  clampThinkingLevel,
+  clampThinkingLevelForApiId,
+  geminiKindForMime,
+  getTool,
+  listBuiltinIds,
+  mimeAllowed,
+  mimeEssence,
+  modelEntryByApiId,
+  registerTools,
+  requireModelSpec,
+  resetTools,
+} from './registry/catalog.ts';
+export type { ProfileDefinition } from './registry/profiles.ts';
+export {
+  clearProfiles,
+  defineProfile,
+  getProfile,
+  hasProfile,
+  listProfiles,
+  registerProfile,
+  registerProfiles,
+} from './registry/profiles.ts';
+export { projectProfile, resolveTurn } from './registry/resolve.ts';
+export { getStructured, registerStructured } from './registry/schemas.ts';
+export { executeTool } from './registry/tools.ts';
+export type * from './types.ts';

package/src/kernel/registry/attachments.ts

@@ -0,0 +1,195 @@
+import { TheorumError } from '../../guardrails/error.ts';
+import { injectionSpans } from '../../guardrails/injection.ts';
+import { sensitiveSpans } from '../../guardrails/sensitive.ts';
+import { applySpans } from '../../observability/spans.ts';
+import type { MediaLimits, MimeInputs, Profile, TurnBlob } from '../types.ts';
+import { getProfile } from './profiles.ts';
+
+const B64_PAD = 2;
+const B64_WORD = 4;
+const B64_TRIPLET = 3;
+
+const CSV_FORMULA = /(^|,)(\s*)("?)(?:([=@])|([+-])(?![0-9."]))/gm;
+const B64_BODY = /^[A-Za-z0-9+/]*={0,2}$/;
+const TEXT_MIMES = new Set(['text/csv', 'text/plain', 'text/markdown']);
+const BYTES_PER_KIB = 1024;
+
+function formatMb(bytes: number): string {
+  const mb = bytes / (BYTES_PER_KIB * BYTES_PER_KIB);
+  return Number.isInteger(mb) ? `${String(mb)} MB` : `${mb.toFixed(1)} MB`;
+}
+
+function tooManyFilesMessage(maxFiles: number): string {
+  return maxFiles === 1
+    ? 'Only 1 file per message.'
+    : `Only ${String(maxFiles)} files per message.`;
+}
+
+function fileTooLargeMessage(maxBytes: number): string {
+  return `Each file must be ${formatMb(maxBytes)} or smaller.`;
+}
+
+function turnTooLargeMessage(maxTurnBytes: number): string {
+  return `Those files together are too large for one message (${formatMb(maxTurnBytes)} max).`;
+}
+
+function mediaLimits(inputs: MimeInputs): MediaLimits | undefined {
+  const { maxFiles, maxBytes, maxTurnBytes, limitsByMime } = inputs;
+  if (maxFiles && maxBytes && maxTurnBytes) {
+    return { maxFiles, maxBytes, maxTurnBytes, limitsByMime };
+  }
+  return undefined;
+}
+
+function maxBytesForMime(mimeType: string, limits: MediaLimits): number {
+  if (limits.limitsByMime) {
+    const cleanMime = mimeType.split(';')[0]?.trim().toLowerCase() ?? '';
+    if (limits.limitsByMime[cleanMime]) {
+      return limits.limitsByMime[cleanMime];
+    }
+    const [category] = cleanMime.split('/');
+    const wildCard = `${category}/*`;
+    if (limits.limitsByMime[wildCard]) {
+      return limits.limitsByMime[wildCard];
+    }
+  }
+  return limits.maxBytes;
+}
+
+function requireMediaLimits(profile: Profile): MediaLimits {
+  const limits = mediaLimits(profile.inputs);
+  if (!limits) {
+    throw new TheorumError(`Profile ${profile.id} must set maxFiles, maxBytes, and maxTurnBytes`);
+  }
+  return limits;
+}
+
+function b64DecodedLen(data: string): number {
+  let pad = 0;
+  if (data.endsWith('==')) {
+    pad = B64_PAD;
+  } else if (data.endsWith('=')) {
+    pad = 1;
+  }
+  return Math.floor((data.length * B64_TRIPLET) / B64_WORD) - pad;
+}
+
+function decodeB64(data: string): Uint8Array {
+  const binary = atob(data);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i += 1) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes;
+}
+
+function encodeB64(bytes: Uint8Array): string {
+  let binary = '';
+  for (const byte of bytes) {
+    binary += String.fromCharCode(byte);
+  }
+  return btoa(binary);
+}
+
+function decodeText(bytes: Uint8Array): string {
+  try {
+    return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+  } catch {
+    return new TextDecoder('latin1').decode(bytes);
+  }
+}
+
+function sanitizeCsvText(text: string): string {
+  return text.replace(CSV_FORMULA, (_full, ...groups: string[]) => {
+    const [a, b, c, d, e] = groups;
+    return `${a}${b}${c}'${d ?? ''}${e ?? ''}`;
+  });
+}
+
+function sanitizeTextBytes(mime: string, bytes: Uint8Array): Uint8Array {
+  let text = decodeText(bytes);
+  if (mime === 'text/csv') {
+    text = sanitizeCsvText(text);
+  }
+  return new TextEncoder().encode(
+    applySpans(text, [...injectionSpans(text), ...sensitiveSpans(text)]),
+  );
+}
+
+function assertAttachmentLimits(blobs: TurnBlob[], limits: MediaLimits): void {
+  if (blobs.length > limits.maxFiles) {
+    throw new TheorumError(tooManyFilesMessage(limits.maxFiles));
+  }
+  let total = 0;
+  for (const blob of blobs) {
+    const { data, mimeType } = blob;
+    if (!B64_BODY.test(data)) {
+      throw new TheorumError('attachment data must be base64');
+    }
+    const size = b64DecodedLen(data);
+    const maxAllowed = maxBytesForMime(mimeType, limits);
+    if (size > maxAllowed) {
+      throw new TheorumError(fileTooLargeMessage(maxAllowed));
+    }
+    total += size;
+  }
+  if (total > limits.maxTurnBytes) {
+    throw new TheorumError(turnTooLargeMessage(limits.maxTurnBytes));
+  }
+}
+
+function sanitizeAttachment(blob: TurnBlob): TurnBlob {
+  const { mimeType, data } = blob;
+  if (!TEXT_MIMES.has(mimeType.split(';')[0]?.trim().toLowerCase() ?? '')) {
+    return blob;
+  }
+  const bytes = sanitizeTextBytes(mimeType, decodeB64(data));
+  return { mimeType, data: encodeB64(bytes) };
+}
+
+function hasTurnBlobs(attachments?: TurnBlob[], voice?: TurnBlob[]): boolean {
+  return (attachments?.length ?? 0) > 0 || (voice?.length ?? 0) > 0;
+}
+
+function sanitizeTurnBlobs(
+  attachments: TurnBlob[] | undefined,
+  voice: TurnBlob[] | undefined,
+  limits: MediaLimits | undefined,
+): { attachments?: TurnBlob[]; voice?: TurnBlob[] } {
+  if (!hasTurnBlobs(attachments, voice)) {
+    return { attachments, voice };
+  }
+  const files = attachments ?? [];
+  const clips = voice ?? [];
+  if (!limits) {
+    throw new TheorumError('This profile does not accept files.');
+  }
+  assertAttachmentLimits([...files, ...clips], limits);
+  return {
+    attachments: files.length > 0 ? files.map(sanitizeAttachment) : attachments,
+    voice: clips.length > 0 ? clips.map(sanitizeAttachment) : voice,
+  };
+}
+
+function sanitizeTurnBlobsForProfile(
+  profileId: string,
+  attachments: TurnBlob[] | undefined,
+  voice: TurnBlob[] | undefined,
+): { attachments?: TurnBlob[]; voice?: TurnBlob[] } {
+  if (!hasTurnBlobs(attachments, voice)) {
+    return { attachments, voice };
+  }
+  const limits = requireMediaLimits(getProfile(profileId));
+  return sanitizeTurnBlobs(attachments, voice, limits);
+}
+
+export {
+  assertAttachmentLimits,
+  fileTooLargeMessage,
+  requireMediaLimits,
+  sanitizeCsvText,
+  sanitizeTurnBlobs,
+  sanitizeTurnBlobsForProfile,
+  tooManyFilesMessage,
+  turnTooLargeMessage,
+};