theorum 0.1.3 → 0.1.4

package/src/kernel/engine/runner/gates.ts

@@ -0,0 +1,338 @@
+import { publicError } from '../../../guardrails/error.ts';
+import { sanitizeTurnRequest } from '../../../guardrails/sanitize.ts';
+import { resolveTurn } from '../../registry/resolve.ts';
+import type {
+  ModelProvider,
+  Profile,
+  ProfileOutputsSpec,
+  ResolvedGeneration,
+  TurnEvent,
+  TurnRequest,
+} from '../../types.ts';
+import type { AttemptFlowState, StepExecutionState } from './state.ts';
+import { executeAttempt } from './steps.ts';
+
+function collectAttemptText(events: TurnEvent[]): string {
+  return events
+    .filter((e) => e.type === 'text' && e.text)
+    .map((e) => e.text)
+    .join('');
+}
+
+function buildRepairRequest(
+  safe: TurnRequest,
+  previousOutput: unknown,
+  rejection: string,
+  repairGuidance?: string,
+): TurnRequest {
+  return {
+    ...safe,
+    input: {
+      ...safe.input,
+      repair: {
+        previousOutput:
+          typeof previousOutput === 'string' ? previousOutput : JSON.stringify(previousOutput),
+        rejection,
+        guidance: repairGuidance,
+      },
+    },
+  };
+}
+
+function hasValidatableOutput(
+  validation: ProfileOutputsSpec['validation'],
+  latestStructured: unknown,
+): boolean {
+  if (!validation || latestStructured === undefined) {
+    return false;
+  }
+  const candidateOutput = validation.extract?.(latestStructured) ?? latestStructured;
+  return candidateOutput !== undefined && candidateOutput !== null;
+}
+
+async function evaluateValidationAttempt(
+  validation: NonNullable<ProfileOutputsSpec['validation']>,
+  latestStructured: unknown,
+  slots: Record<string, string> | undefined,
+): Promise<{ candidateOutput: unknown; isValid: boolean; error: string }> {
+  const candidateOutput = validation.extract?.(latestStructured) ?? latestStructured;
+  const check = await validation.validate(candidateOutput, slots);
+  const error = check.error || check.finding || 'Validation failed';
+  return { candidateOutput, isValid: Boolean(check.isValid), error };
+}
+
+type EgressOutcome =
+  | { action: 'pass' }
+  | { action: 'refusal'; event: TurnEvent }
+  | { action: 'retry'; nextRequest: TurnRequest }
+  | { action: 'withhold'; event: TurnEvent };
+
+async function evaluateEgressOutcome(args: {
+  egress: NonNullable<Profile['guardrails']['egress']>;
+  attemptEvents: TurnEvent[];
+  generation: ResolvedGeneration;
+  request: TurnRequest;
+  profile: Profile;
+  canRetry: boolean;
+}): Promise<EgressOutcome> {
+  const { egress, attemptEvents, generation, request, profile, canRetry } = args;
+  const attemptText = collectAttemptText(attemptEvents);
+  const result = await egress.enforce({
+    text: attemptText,
+    canary: generation.canary,
+    slots: request.input?.slots,
+    profile,
+    role: request.input?.role,
+  });
+
+  if (!result.blocked) {
+    return { action: 'pass' };
+  }
+
+  if (egress.onBlock === 'refuse_to_user') {
+    return { action: 'refusal', event: { type: 'text', text: result.text } };
+  }
+
+  if (canRetry) {
+    const rejectionMsg = result.rejectionMessage || 'Egress disclosure violation detected.';
+    const repairGuidance =
+      egress.repairGuidance ||
+      'Rewrite the message as corrected user-visible prose only. Keep the same helpful substance; scrub all internal tool names, leak phrases, and disclosure markers.';
+    const nextRequest = buildRepairRequest(request, attemptText, rejectionMsg, repairGuidance);
+    return { action: 'retry', nextRequest };
+  }
+
+  return {
+    action: 'withhold',
+    event: {
+      type: 'error',
+      error: publicError('Turn withheld: egress disclosure violation'),
+    },
+  };
+}
+
+type ValidationOutcome =
+  | { action: 'pass' }
+  | { action: 'retry'; nextRequest: TurnRequest }
+  | { action: 'accept'; event: TurnEvent };
+
+async function evaluateValidationOutcome(args: {
+  validation: NonNullable<ProfileOutputsSpec['validation']>;
+  latestStructured: unknown;
+  request: TurnRequest;
+  canRetry: boolean;
+}): Promise<ValidationOutcome> {
+  const { validation, latestStructured, request, canRetry } = args;
+  if (!hasValidatableOutput(validation, latestStructured)) {
+    return { action: 'pass' };
+  }
+
+  const { candidateOutput, isValid, error } = await evaluateValidationAttempt(
+    validation,
+    latestStructured,
+    request.input?.slots,
+  );
+
+  if (isValid) {
+    return { action: 'pass' };
+  }
+
+  if (canRetry) {
+    const nextRequest = buildRepairRequest(
+      request,
+      candidateOutput,
+      error,
+      validation.repairGuidance,
+    );
+    return { action: 'retry', nextRequest };
+  }
+
+  return {
+    action: 'accept',
+    event: { type: 'structured', structured: latestStructured },
+  };
+}
+
+function* yieldBufferedAttemptEvents(events: TurnEvent[]): Generator<TurnEvent> {
+  for (const ev of events) {
+    if (ev.type !== 'tokens') {
+      yield ev;
+    }
+  }
+}
+
+function updateFlowForRetry(flow: AttemptFlowState, nextReq: TurnRequest): void {
+  flow.currentAttempt++;
+  flow.currentReq = nextReq;
+  flow.currentGen = resolveTurn(sanitizeTurnRequest(nextReq)).generation;
+}
+
+async function* handleEgressGate(
+  egress: NonNullable<Profile['guardrails']['egress']>,
+  flow: AttemptFlowState,
+  state: StepExecutionState,
+  profile: Profile,
+  maxRetries: number,
+): AsyncGenerator<TurnEvent, 'continue' | 'terminal' | 'pass'> {
+  const canRetry = flow.currentAttempt < maxRetries;
+  const outcome = await evaluateEgressOutcome({
+    egress,
+    attemptEvents: state.attemptEvents,
+    generation: flow.currentGen,
+    request: flow.currentReq,
+    profile,
+    canRetry,
+  });
+
+  if (outcome.action === 'refusal') {
+    state.allEmittedEvents.push(outcome.event);
+    yield outcome.event;
+    return 'terminal';
+  }
+  if (outcome.action === 'withhold') {
+    yield outcome.event;
+    return 'terminal';
+  }
+  if (outcome.action === 'retry') {
+    updateFlowForRetry(flow, outcome.nextRequest);
+    return 'continue';
+  }
+  return 'pass';
+}
+
+async function* handleValidationGate(
+  validation: NonNullable<ProfileOutputsSpec['validation']>,
+  flow: AttemptFlowState,
+  state: StepExecutionState,
+  latestStructured: unknown,
+  maxRetries: number,
+): AsyncGenerator<TurnEvent, 'continue' | 'terminal' | 'pass'> {
+  const canRetry = flow.currentAttempt < maxRetries;
+  const outcome = await evaluateValidationOutcome({
+    validation,
+    latestStructured,
+    request: flow.currentReq,
+    canRetry,
+  });
+
+  if (outcome.action === 'retry') {
+    updateFlowForRetry(flow, outcome.nextRequest);
+    return 'continue';
+  }
+  if (outcome.action === 'accept') {
+    state.allEmittedEvents.push(outcome.event);
+    yield outcome.event;
+    return 'terminal';
+  }
+  return 'pass';
+}
+
+type AttemptStepAction =
+  | { status: 'terminal' }
+  | { status: 'continue' }
+  | {
+      status: 'success';
+    };
+
+function gateStatusToAction(
+  status: 'continue' | 'terminal' | 'pass',
+  terminalStatus: 'terminal' | 'success' = 'terminal',
+): AttemptStepAction | null {
+  if (status === 'terminal') {
+    return { status: terminalStatus };
+  }
+  if (status === 'continue') {
+    return { status: 'continue' };
+  }
+  return null;
+}
+
+async function* executeSingleAttemptCycle(args: {
+  flow: AttemptFlowState;
+  state: StepExecutionState;
+  profile: Profile;
+  system: string;
+  provider: ModelProvider;
+  gemini: Record<string, unknown>[];
+  maxRetries: number;
+}): AsyncGenerator<TurnEvent, AttemptStepAction> {
+  const { flow, state, profile, system, provider, gemini, maxRetries } = args;
+  const validation = profile.outputs.validation;
+  const egress = profile.guardrails.egress;
+
+  state.attemptEvents = [];
+  const { latestStructured } = yield* executeAttempt({
+    safe: flow.currentReq,
+    profile,
+    generation: flow.currentGen,
+    system,
+    provider,
+    gemini,
+    state,
+  });
+
+  if (egress?.enforce) {
+    const status = yield* handleEgressGate(egress, flow, state, profile, maxRetries);
+    const action = gateStatusToAction(status, 'terminal');
+    if (action) {
+      return action;
+    }
+  }
+
+  if (validation) {
+    const status = yield* handleValidationGate(
+      validation,
+      flow,
+      state,
+      latestStructured,
+      maxRetries,
+    );
+    const action = gateStatusToAction(status, 'success');
+    if (action) {
+      return action;
+    }
+  }
+
+  if (validation || egress?.enforce) {
+    yield* yieldBufferedAttemptEvents(state.attemptEvents);
+  }
+
+  return { status: 'success' };
+}
+
+async function* runAttemptsWithValidation(
+  safe: TurnRequest,
+  profile: Profile,
+  generation: ResolvedGeneration,
+  system: string,
+  provider: ModelProvider,
+  gemini: Record<string, unknown>[],
+  state: StepExecutionState,
+): AsyncGenerator<TurnEvent> {
+  const maxRetries = Math.max(
+    profile.outputs.validation?.maxRetries ?? 0,
+    profile.guardrails.egress?.maxRetries ?? 2,
+  );
+  const flow: AttemptFlowState = {
+    currentAttempt: 0,
+    currentGen: generation,
+    currentReq: safe,
+  };
+
+  while (flow.currentAttempt <= maxRetries) {
+    const step = yield* executeSingleAttemptCycle({
+      flow,
+      state,
+      profile,
+      system,
+      provider,
+      gemini,
+      maxRetries,
+    });
+    if (step.status === 'terminal' || step.status === 'success') {
+      break;
+    }
+  }
+}
+
+export { runAttemptsWithValidation };

package/src/kernel/engine/runner/mod.ts

@@ -0,0 +1,135 @@
+/**
+ * Deterministic turn runner for THEORUM.
+ *
+ * `runTurn` resolves a profile, sanitizes input, binds canary boundaries,
+ * streams provider events, executes allowed tools, applies validation and
+ * egress repair loops, writes traces, and emits one terminal `done` event.
+ *
+ * @module
+ */
+
+import { sanitizeTurnRequest } from '../../../guardrails/sanitize.ts';
+import { noopSink, type TraceSink, writeTrace } from '../../../observability/trace.ts';
+import { buildRecord } from '../../../observability/trace-record.ts';
+import { pickSystemRole, resolveTurn } from '../../registry/resolve.ts';
+import type {
+  ModelProvider,
+  Profile,
+  ResolvedGeneration,
+  TurnEvent,
+  TurnRequest,
+} from '../../types.ts';
+import { bindCanary } from '../boundary.ts';
+import { runAttemptsWithValidation } from './gates.ts';
+import type { StepExecutionState } from './state.ts';
+import { shouldSkipStreamEvent, systemFromProfile } from './stream.ts';
+import { calculateFallbackTokens } from './tokens.ts';
+import { invokeFromUi } from './tools.ts';
+
+async function* emitTurn(args: {
+  safe: TurnRequest;
+  profile: Profile;
+  generation: ResolvedGeneration;
+  system: string;
+  provider: ModelProvider;
+  gemini: Record<string, unknown>[];
+}): AsyncGenerator<TurnEvent> {
+  const { safe, profile, generation, system, provider, gemini } = args;
+  if (safe.toolInvoke) {
+    yield* invokeFromUi(profile, safe);
+    return;
+  }
+
+  const state: StepExecutionState = {
+    currentHistory: [...(generation.history ?? [])],
+    stepCount: 0,
+    sawTokensEvent: false,
+    allEmittedEvents: [],
+    attemptEvents: [],
+  };
+
+  yield* runAttemptsWithValidation(safe, profile, generation, system, provider, gemini, state);
+
+  if (!state.sawTokensEvent) {
+    yield* calculateFallbackTokens(safe, system, state.allEmittedEvents);
+  }
+
+  yield { type: 'done' };
+}
+
+/** Execute one host turn against a provider adapter. */
+async function* runTurn(
+  req: TurnRequest,
+  provider: ModelProvider,
+  sink: TraceSink = noopSink(),
+): AsyncGenerator<TurnEvent> {
+  const started = Date.now();
+  const seen: TurnEvent[] = [];
+  const gemini: Record<string, unknown>[] = [];
+  let model: string | undefined;
+  let bucket: string | undefined;
+  let canary = '';
+  let system: string | undefined;
+  let generation: ResolvedGeneration | undefined;
+  try {
+    const safe = sanitizeTurnRequest(req);
+    const { profile, generation: gen } = resolveTurn(safe);
+    generation = gen;
+    const { model: resolvedModel, geminiBucket, canary: turnCanary } = gen;
+    model = resolvedModel;
+    bucket = geminiBucket;
+    canary = turnCanary;
+    const role = pickSystemRole(profile, safe.input?.role);
+    const profileSys = systemFromProfile(profile, role);
+    const combinedSys = [profileSys, safe.system].filter(Boolean).join('\n\n');
+    const bound = bindCanary(combinedSys, turnCanary);
+    system = bound;
+    for await (const event of emitTurn({
+      safe,
+      profile,
+      generation: gen,
+      system: bound,
+      provider,
+      gemini,
+    })) {
+      seen.push(event);
+      if (shouldSkipStreamEvent(event, profile)) {
+        continue;
+      }
+      yield event;
+    }
+  } catch (err) {
+    await writeTrace(
+      sink,
+      buildRecord({
+        req,
+        events: seen,
+        started,
+        model,
+        bucket,
+        thrown: err,
+        gemini,
+        canary,
+        system,
+        generation,
+      }),
+    );
+    throw err;
+  }
+  await writeTrace(
+    sink,
+    buildRecord({
+      req,
+      events: seen,
+      started,
+      model,
+      bucket,
+      gemini,
+      canary,
+      system,
+      generation,
+    }),
+  );
+}
+
+export { runTurn };

package/src/kernel/engine/runner/state.ts

@@ -0,0 +1,31 @@
+import type {
+  ResolvedGeneration,
+  TurnEvent,
+  TurnHistoryMessage,
+  TurnRequest,
+} from '../../types.ts';
+
+interface StepExecutionState {
+  currentHistory: TurnHistoryMessage[];
+  stepCount: number;
+  sawTokensEvent: boolean;
+  allEmittedEvents: TurnEvent[];
+  attemptEvents: TurnEvent[];
+}
+
+interface AttemptFlowState {
+  currentAttempt: number;
+  currentReq: TurnRequest;
+  currentGen: ResolvedGeneration;
+}
+
+function recordStepEvent(event: TurnEvent, state: StepExecutionState): void {
+  if (event.type === 'tokens') {
+    state.sawTokensEvent = true;
+  }
+  state.allEmittedEvents.push(event);
+  state.attemptEvents.push(event);
+}
+
+export type { AttemptFlowState, StepExecutionState };
+export { recordStepEvent };

package/src/kernel/engine/runner/steps.ts

@@ -0,0 +1,183 @@
+import type {
+  ModelProvider,
+  Profile,
+  ResolvedGeneration,
+  ToolEnvelope,
+  TurnEvent,
+  TurnHistoryMessage,
+  TurnRequest,
+} from '../../types.ts';
+import { recordStepEvent, type StepExecutionState } from './state.ts';
+import { yieldProviderEvents } from './stream.ts';
+import {
+  executeDynamicDeclaration,
+  findDynamicDeclaration,
+  formatToolFinding,
+  isActionableDynamicDeclaration,
+} from './tools.ts';
+
+function isStepLimitReached(step: number, maxSteps: number): boolean {
+  if (maxSteps <= 0) {
+    return false;
+  }
+  return step >= maxSteps;
+}
+
+async function* executeAutonomousStep(
+  args: {
+    profile: Profile;
+    generation: ResolvedGeneration;
+    system: string;
+    provider: ModelProvider;
+    gemini: Record<string, unknown>[];
+  },
+  state: StepExecutionState,
+  bufferOutputs = false,
+): AsyncGenerator<TurnEvent, { pendingTools: TurnEvent[]; latestStructured?: unknown }> {
+  const { profile, generation, system, provider, gemini } = args;
+  const genForStep = { ...generation, history: state.currentHistory };
+  const pendingTools: TurnEvent[] = [];
+  let latestStructured: unknown;
+
+  for await (const event of yieldProviderEvents({
+    profile,
+    generation: genForStep,
+    system,
+    provider,
+    gemini,
+  })) {
+    if (event.type === 'structured') {
+      latestStructured = event.structured;
+    }
+    if (event.type === 'done') {
+      continue;
+    }
+    if (event.type === 'tool' && event.tool) {
+      pendingTools.push(event);
+      continue;
+    }
+    recordStepEvent(event, state);
+    if (!bufferOutputs || event.type === 'tokens') {
+      yield event;
+    }
+  }
+
+  return { pendingTools, latestStructured };
+}
+
+function appendToolTurnToHistory(
+  history: TurnHistoryMessage[],
+  toolEv: TurnEvent,
+  res: ToolEnvelope,
+): void {
+  const tool = toolEv.tool;
+  if (!tool) {
+    return;
+  }
+  const callId = tool.id ?? `call_${tool.name}_${String(Date.now())}`;
+  history.push({
+    role: 'assistant',
+    tool_calls: [
+      {
+        id: callId,
+        type: 'function',
+        function: {
+          name: tool.name,
+          arguments: JSON.stringify(tool.arguments ?? {}),
+        },
+      },
+    ],
+  });
+  history.push({
+    role: 'tool',
+    tool_call_id: callId,
+    name: tool.name,
+    content: formatToolFinding(res),
+  });
+}
+
+async function* handlePendingDynamicTools(
+  pendingTools: TurnEvent[],
+  generation: ResolvedGeneration,
+  profile: Profile,
+  state: StepExecutionState,
+): AsyncGenerator<TurnEvent, boolean> {
+  let hasRunnableHandler = false;
+  for (const toolEv of pendingTools) {
+    const tool = toolEv.tool;
+    if (!tool) {
+      continue;
+    }
+    const decl = findDynamicDeclaration(generation.dynamicTools, tool.name);
+    if (!isActionableDynamicDeclaration(decl)) {
+      state.allEmittedEvents.push(toolEv);
+      yield toolEv;
+      continue;
+    }
+
+    hasRunnableHandler = true;
+    const finalResult = await executeDynamicDeclaration({
+      decl,
+      toolArgs: tool.arguments ?? {},
+      profile,
+      generation,
+    });
+    const enrichedEvent: TurnEvent = {
+      type: 'tool',
+      tool: { ...tool, result: finalResult },
+    };
+    state.allEmittedEvents.push(enrichedEvent);
+    yield enrichedEvent;
+    appendToolTurnToHistory(state.currentHistory, toolEv, finalResult);
+  }
+  return hasRunnableHandler;
+}
+
+async function* executeAttempt(args: {
+  safe: TurnRequest;
+  profile: Profile;
+  generation: ResolvedGeneration;
+  system: string;
+  provider: ModelProvider;
+  gemini: Record<string, unknown>[];
+  state: StepExecutionState;
+}): AsyncGenerator<TurnEvent, { pendingTools: TurnEvent[]; latestStructured?: unknown }> {
+  const { profile, generation, system, provider, gemini, state } = args;
+  let latestStructured: unknown;
+  let pendingTools: TurnEvent[] = [];
+  let stepInAttempt = 0;
+  const shouldBuffer =
+    Boolean(profile.outputs.validation) || Boolean(profile.guardrails.egress?.enforce);
+
+  while (!isStepLimitReached(stepInAttempt, generation.maxSteps)) {
+    stepInAttempt++;
+    state.stepCount++;
+    const stepResult = yield* executeAutonomousStep(
+      { profile, generation, system, provider, gemini },
+      state,
+      shouldBuffer,
+    );
+    if (stepResult.latestStructured !== undefined) {
+      latestStructured = stepResult.latestStructured;
+    }
+    pendingTools = stepResult.pendingTools;
+
+    if (pendingTools.length === 0) {
+      break;
+    }
+
+    const hasRunnableHandler = yield* handlePendingDynamicTools(
+      pendingTools,
+      generation,
+      profile,
+      state,
+    );
+    if (!hasRunnableHandler) {
+      break;
+    }
+  }
+
+  return { pendingTools, latestStructured };
+}
+
+export { executeAttempt };