theorum 0.1.15 → 1.1.3

package/esm/src/kernel/engine/runner/mod.js

@@ -7,28 +7,52 @@
  *
  * @module
  */
-import { throwIfAborted } from '../../../guardrails/error.js';
-import { sanitizeTurnRequest } from '../../../guardrails/sanitize.js';
+import { bindCanary } from '../../../guardrails/canary.js';
+import { isAbortError, throwIfAborted } from '../../../guardrails/error.js';
+import { projectGuardrailTurnEvent } from '../../../guardrails/events.js';
+import { resolveGuardrailPolicy } from '../../../guardrails/policy.js';
+import { sanitizeTurnRequestWithEvents } from '../../../guardrails/sanitize.js';
+import { resolveTraceWriter } from '../../../observability/policy.js';
 import { noopSink, writeTrace } from '../../../observability/trace.js';
 import { buildRecord } from '../../../observability/trace-record.js';
-import { pickSystemRole, resolveTurn } from '../../registry/resolve.js';
-import { CONTINUE_INSTRUCTION } from '../../stop.js';
-import { bindCanary } from '../boundary.js';
+import { getProfile } from '../../registry/profiles.js';
+import { resolveTurn } from '../../registry/resolve.js';
+import { cloneTurnToolSnapshot, expandT1Policy } from '../../tools/resolve.js';
 import { resolveCompactionTokens, shouldCompact, splitForCompaction } from '../compaction.js';
 import { runAttemptsWithValidation } from './gates.js';
-import { shouldSkipStreamEvent, systemFromProfile } from './stream.js';
+import { applyTurnStage } from './stages.js';
+import { shouldSkipStreamEvent } from './stream.js';
 import { calculateFallbackTokens } from './tokens.js';
-import { invokeFromUi } from './tools.js';
+function projectForObs(event, policy) {
+    return projectGuardrailTurnEvent(event, policy?.include.guardrailMatchPreview ?? false);
+}
+function* emitObservedStage(ctx, stageEv, profile) {
+    const stageOut = projectForObs(stageEv, ctx.observability);
+    ctx.seen.push(stageOut);
+    if (!shouldSkipStreamEvent(stageOut, profile))
+        yield stageOut;
+}
 function getCompactionSpec(profile, modelId) {
-    return profile.model.config[modelId]?.compaction;
+    return profile.models[modelId]?.compaction;
+}
+/** Fold one history message to a compaction transcript line (media → markers). */
+function compactionTranscriptLine(m) {
+    const text = m.content ?? '';
+    const mediaMarkers = m.parts
+        ?.filter((p) => p.type !== 'text')
+        .map((p) => `[${p.type}]`)
+        .join('') ?? '';
+    const fromTextParts = !text && m.parts
+        ? m.parts
+            .filter((p) => p.type === 'text')
+            .map((p) => p.text)
+            .join('')
+        : '';
+    const content = `${text || fromTextParts}${mediaMarkers}`;
+    return `[${m.role}]: ${content}`;
 }
 async function runCompactionTurn(toCompact, spec, provider, signal) {
-    const compactText = toCompact
-        .map((m) => {
-        const content = m.content ?? m.parts?.map((p) => ('text' in p ? p.text : '')).join('') ?? '';
-        return `[${m.role}]: ${content}`;
-    })
-        .join('\n');
+    const compactText = toCompact.map(compactionTranscriptLine).join('\n');
     const events = [];
     for await (const event of runTurn({
         profile: spec.profile,
@@ -94,11 +118,7 @@ async function attachAfterCompaction(event, args) {
     return { ...event, compaction: signal };
 }
 async function* emitTurn(args) {
-    const { safe, profile, generation, system, provider, gemini } = args;
-    if (safe.toolInvoke) {
-        yield* invokeFromUi(profile, safe);
-        return;
-    }
+    const { safe, profile, generation, system, provider, upstream } = args;
     const state = {
         currentHistory: [...(generation.history ?? [])],
         stepCount: 0,
@@ -106,14 +126,95 @@ async function* emitTurn(args) {
         allEmittedEvents: [],
         attemptEvents: [],
     };
-    yield* runAttemptsWithValidation(safe, profile, generation, system, provider, gemini, state);
+    args.outState.state = state;
+    const pre = yield* applyTurnStage({
+        profile,
+        generation,
+        state,
+        stage: 'pre_turn',
+        step: 1,
+        onStage: safe.onStage,
+        signal: safe.signal,
+        foldInput: true,
+        host: generation.host,
+    });
+    if (pre.abort) {
+        const stop = {
+            kind: 'cancelled',
+            ...(typeof pre.abort === 'object' && pre.abort.reason ? { native: pre.abort.reason } : {}),
+        };
+        const done = { type: 'done', stop };
+        state.allEmittedEvents.push(done);
+        yield done;
+        yield* applyTurnStage({
+            profile,
+            generation,
+            state,
+            stage: 'post_turn',
+            step: 1,
+            onStage: safe.onStage,
+            signal: safe.signal,
+            stop,
+            host: generation.host,
+        });
+        return;
+    }
+    yield* runAttemptsWithValidation(safe, profile, generation, system, provider, upstream, state);
     if (!state.sawTokensEvent) {
         yield* calculateFallbackTokens(safe, system, state.allEmittedEvents);
     }
-    yield {
+    const stop = state.lastStop ?? { kind: 'completed' };
+    const done = {
         type: 'done',
-        stop: state.lastStop ?? { kind: 'completed' },
+        stop,
+        ...(stop.kind === 'tool' && state.toolSnapshot
+            ? { tools: cloneTurnToolSnapshot(state.toolSnapshot) }
+            : {}),
+        ...(stop.kind === 'gate' && state.toolSnapshot
+            ? { tools: cloneTurnToolSnapshot(state.toolSnapshot) }
+            : {}),
     };
+    state.allEmittedEvents.push(done);
+    yield done;
+}
+async function* streamTurnEvents(ctx, profile, gen, provider, compactionSpec, isCompacting) {
+    if (!ctx.safe || ctx.system === undefined)
+        return;
+    const outState = {};
+    for await (const event of emitTurn({
+        safe: ctx.safe,
+        profile,
+        generation: gen,
+        system: ctx.system,
+        provider,
+        upstream: ctx.upstream,
+        outState,
+    })) {
+        const attached = await maybeAttachAfter(event, ctx, gen, compactionSpec, isCompacting);
+        const out = projectForObs(attached, ctx.observability);
+        ctx.seen.push(out);
+        if (!shouldSkipStreamEvent(out, profile))
+            yield out;
+        if (out.type === 'done' && outState.state) {
+            // Skip duplicate post_turn when pre_turn abort already emitted it inside emitTurn.
+            const alreadyPost = outState.state.allEmittedEvents.some((e) => e.type === 'stage' && e.stage === 'post_turn');
+            if (!alreadyPost) {
+                for await (const stageEv of applyTurnStage({
+                    profile,
+                    generation: gen,
+                    state: outState.state,
+                    stage: 'post_turn',
+                    step: Math.max(outState.state.stepCount, 1),
+                    onStage: ctx.safe.onStage,
+                    signal: ctx.safe.signal,
+                    stop: out.stop,
+                    host: gen.host,
+                })) {
+                    yield* emitObservedStage(ctx, stageEv, profile);
+                }
+            }
+        }
+    }
 }
 async function flushTurnTrace(sink, ctx) {
     await writeTrace(sink, buildRecord({
@@ -121,50 +222,105 @@ async function flushTurnTrace(sink, ctx) {
         events: ctx.seen,
         started: ctx.started,
         model: ctx.model,
-        bucket: ctx.bucket,
+        keySlot: ctx.keySlot,
         thrown: ctx.thrown,
-        gemini: ctx.gemini,
+        upstreamLog: ctx.upstream,
         canary: ctx.canary,
         system: ctx.system,
         generation: ctx.generation,
+        protocol: ctx.protocol,
         sanitizedReq: ctx.safe,
+        observability: ctx.observability,
     }));
 }
 /** Execute one host turn against a provider adapter. */
-async function* runTurn(req, provider, sink = noopSink()) {
+async function* runTurn(req, provider, sinkOverride) {
     const ctx = {
         req,
         seen: [],
         started: Date.now(),
         canary: '',
-        gemini: [],
+        upstream: [],
     };
+    let sink = sinkOverride ?? noopSink();
     try {
+        const profile = getProfile(req.profile);
+        const resolved = resolveTraceWriter({
+            override: sinkOverride,
+            observability: profile.observability,
+        });
+        sink = resolved.sink;
+        ctx.observability = resolved.policy;
         yield* runTurnBody(ctx, provider);
     }
     catch (err) {
+        if (isAbortError(err)) {
+            yield* emitCancelledDoneAfterAbort(ctx);
+            await flushTurnTrace(sink, { ...ctx, thrown: err });
+            return;
+        }
         await flushTurnTrace(sink, { ...ctx, thrown: err });
         throw err;
     }
     await flushTurnTrace(sink, ctx);
 }
+/**
+ * Slice 1: host AbortSignal ends with cancelled `done` + `post_turn`, not a bare throw.
+ * Skip when a terminal `done` already reached the host stream.
+ */
+async function* emitCancelledDoneAfterAbort(ctx) {
+    if (ctx.seen.some((e) => e.type === 'done'))
+        return;
+    const stop = { kind: 'cancelled' };
+    const done = { type: 'done', stop };
+    const outDone = projectForObs(done, ctx.observability);
+    ctx.seen.push(outDone);
+    yield outDone;
+    const profile = getProfile(ctx.req.profile);
+    const gen = ctx.generation;
+    if (!gen || !ctx.safe)
+        return;
+    const state = {
+        currentHistory: [...(gen.history ?? [])],
+        stepCount: 0,
+        sawTokensEvent: false,
+        allEmittedEvents: [...ctx.seen],
+        attemptEvents: [],
+    };
+    for await (const stageEv of applyTurnStage({
+        profile,
+        generation: gen,
+        state,
+        stage: 'post_turn',
+        step: 1,
+        onStage: ctx.safe.onStage,
+        stop,
+        host: gen.host,
+    })) {
+        yield* emitObservedStage(ctx, stageEv, profile);
+    }
+}
 async function* runTurnBody(ctx, provider) {
-    ctx.safe = sanitizeTurnRequest(ctx.req);
-    throwIfAborted(ctx.safe.signal);
+    const sanitized = sanitizeTurnRequestWithEvents(ctx.req);
+    ctx.safe = sanitized.request;
+    for (const event of sanitized.events) {
+        const out = projectForObs(event, ctx.observability);
+        ctx.seen.push(out);
+        yield out;
+    }
     const { profile, generation: gen } = resolveTurn(ctx.safe);
+    await expandT1Policy(gen.tools, profile, ctx.safe);
+    gen.builtins = gen.tools.builtins;
     ctx.generation = gen;
     ctx.model = gen.model;
-    ctx.bucket = gen.geminiBucket;
+    ctx.keySlot = gen.keySlot;
     ctx.canary = gen.canary;
+    ctx.protocol = profile.models[gen.model]?.protocol;
+    throwIfAborted(ctx.safe.signal);
     const isCompacting = ctx.req.metadata?._compacting === true;
     const compactionSpec = isCompacting ? undefined : getCompactionSpec(profile, gen.model);
     await maybeCompactBefore(ctx, gen, compactionSpec, provider);
-    const role = pickSystemRole(profile, ctx.safe.input?.role);
-    const continueSys = ctx.safe.continueFrom ? CONTINUE_INSTRUCTION : '';
-    const combinedSys = [systemFromProfile(profile, role), ctx.safe.system, continueSys]
-        .filter(Boolean)
-        .join('\n\n');
-    ctx.system = bindCanary(combinedSys, ctx.canary);
+    ctx.system = bindCanary(gen.resolvedSystem, ctx.canary, resolveGuardrailPolicy(profile.guardrails).canaryBindNote);
     yield* streamTurnEvents(ctx, profile, gen, provider, compactionSpec, isCompacting);
 }
 async function maybeCompactBefore(ctx, gen, compactionSpec, provider) {
@@ -179,23 +335,6 @@ async function maybeCompactBefore(ctx, gen, compactionSpec, provider) {
         signal: ctx.safe.signal,
     });
 }
-async function* streamTurnEvents(ctx, profile, gen, provider, compactionSpec, isCompacting) {
-    if (!ctx.safe || ctx.system === undefined)
-        return;
-    for await (const event of emitTurn({
-        safe: ctx.safe,
-        profile,
-        generation: gen,
-        system: ctx.system,
-        provider,
-        gemini: ctx.gemini,
-    })) {
-        const out = await maybeAttachAfter(event, ctx, gen, compactionSpec, isCompacting);
-        ctx.seen.push(out);
-        if (!shouldSkipStreamEvent(out, profile))
-            yield out;
-    }
-}
 async function maybeAttachAfter(event, ctx, gen, compactionSpec, isCompacting) {
     if (!(event.type === 'done' && compactionSpec?.timing === 'after' && !isCompacting && ctx.safe)) {
         return event;
@@ -207,4 +346,4 @@ async function maybeAttachAfter(event, ctx, gen, compactionSpec, isCompacting) {
         seen: ctx.seen,
     });
 }
-export { runTurn };
+export { compactionTranscriptLine, runTurn };

package/esm/src/kernel/engine/runner/schema-validation.js

@@ -41,7 +41,7 @@ function joinPath(prefix, key) {
     return prefix ? `${prefix}.${key}` : key;
 }
 function pushMissing(path, failures) {
-    failures.push({ path, error: `required field '${path}' is missing` });
+    failures.push({ path, error: `required field '${path}' is missing` }); // lexicon-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
 }
 async function runFieldValidator(path, value, validator, slots, failures) {
     if (!validator) {
@@ -53,7 +53,7 @@ async function runFieldValidator(path, value, validator, slots, failures) {
     }
     failures.push({
         path,
-        error: check.error || check.finding || `Validation failed for '${path}'`,
+        error: check.error || check.finding || `Validation failed for '${path}'`, // lexicon-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
     });
 }
 async function visitProperty(args) {
@@ -95,7 +95,7 @@ async function walkObject(schema, value, pathPrefix, fields, slots, failures) {
 async function collectValidationFailures(jsonSchema, structured, fields, slots) {
     const root = asObjectSchema(jsonSchema);
     if (!root) {
-        throw new TheorumError('structured validation requires a JSON Schema object root');
+        throw new TheorumError('structured validation requires a JSON Schema object root'); // lexicon-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
     }
     const failures = [];
     if (!asRecord(structured)) {

package/esm/src/kernel/engine/runner/stages.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Text `runTurn` wiring around the stage spine: record stage events in step
+ * state, fold the opening input, and append sanitized injects to turn history.
+ *
+ * Contract: `docs/contracts/stages.md`. The stage itself runs in `runStage`.
+ *
+ * @module
+ */
+import { type StageApplyWarning, type StageCallBag, type StageContext, type StageHandler } from '../../stages.js';
+import type { Profile, ResolvedGeneration, TurnEvent, TurnHistoryMessage } from '../../types.js';
+import type { StepExecutionState } from './state.js';
+export interface ApplyTurnStageArgs extends StageCallBag {
+    profile: Profile;
+    generation: ResolvedGeneration;
+    state: StepExecutionState;
+    stage: StageContext['stage'];
+    step: number;
+    onStage?: StageHandler;
+    signal?: AbortSignal;
+    /** Fold generation.input into history when true (pre_turn + handler present). */
+    foldInput?: boolean;
+    /** Another provider step would exceed maxSteps — reject inject. */
+    injectWouldExceedMaxSteps?: boolean;
+    host?: unknown;
+}
+export interface ApplyTurnStageResult {
+    abort?: boolean | {
+        reason?: string;
+    };
+    injectCount: number;
+    warnings: StageApplyWarning[];
+}
+/**
+ * Append already-sanitized inject messages to turn history. Mirrors them into
+ * the Interactions continuation when one is active. Returns how many landed.
+ */
+export declare function applyStageInjects(state: StepExecutionState, inject: readonly TurnHistoryMessage[]): number;
+/** Run a turn stage, recording every stage event on step state. */
+export declare function applyTurnStage(args: ApplyTurnStageArgs): AsyncGenerator<TurnEvent, ApplyTurnStageResult>;

package/esm/src/kernel/engine/runner/stages.js

@@ -0,0 +1,89 @@
+/**
+ * Text `runTurn` wiring around the stage spine: record stage events in step
+ * state, fold the opening input, and append sanitized injects to turn history.
+ *
+ * Contract: `docs/contracts/stages.md`. The stage itself runs in `runStage`.
+ *
+ * @module
+ */
+import { throwIfAborted } from '../../../guardrails/error.js';
+import { wireInteractionPart } from '../../interaction-parts.js';
+import { runStage, } from '../../stages.js';
+import { profileAllowsInject } from '../../stop.js';
+function stageMessageToInteractionStep(msg) {
+    const type = msg.role === 'assistant' ? 'model_output' : 'user_input';
+    if (msg.parts && msg.parts.length > 0) {
+        return {
+            type,
+            content: msg.parts.map(wireInteractionPart),
+        };
+    }
+    return { type, content: [{ type: 'text', text: msg.content ?? '' }] };
+}
+/** Move opening user `generation.input` into history (Interactions-safe fold). */
+function foldGenerationInputIntoHistory(generation, state) {
+    if (state.foldedForGeneration === generation)
+        return;
+    state.foldedForGeneration = generation;
+    if (generation.input.length === 0)
+        return;
+    const textOnly = generation.input.every((p) => p.type === 'text');
+    if (textOnly) {
+        state.currentHistory.push({
+            role: 'user',
+            content: generation.input.map((p) => (p.type === 'text' ? p.text : '')).join(''),
+        });
+    }
+    else {
+        state.currentHistory.push({
+            role: 'user',
+            parts: [...generation.input],
+        });
+    }
+    generation.input = [];
+}
+/**
+ * Append already-sanitized inject messages to turn history. Mirrors them into
+ * the Interactions continuation when one is active. Returns how many landed.
+ */
+export function applyStageInjects(state, inject) {
+    if (inject.length === 0)
+        return 0;
+    state.currentHistory.push(...inject);
+    if (state.interactionsContinuation) {
+        for (const msg of inject) {
+            if (msg.role === 'tool')
+                continue;
+            state.interactionsContinuation.input.push(stageMessageToInteractionStep(msg));
+        }
+    }
+    return inject.length;
+}
+/** Run a turn stage, recording every stage event on step state. */
+export async function* applyTurnStage(args) {
+    const { profile, generation, state, onStage, foldInput, host, ...call } = args;
+    throwIfAborted(call.signal);
+    if (foldInput && onStage) {
+        foldGenerationInputIntoHistory(generation, state);
+    }
+    const run = runStage({
+        ...call,
+        history: state.currentHistory,
+        handlers: onStage ? [onStage] : [],
+        guardrails: profile.guardrails,
+        injectAllowed: profileAllowsInject(profile),
+        host: host ?? generation.host,
+    });
+    let next = await run.next();
+    while (!next.done) {
+        state.allEmittedEvents.push(next.value);
+        yield next.value;
+        next = await run.next();
+    }
+    const out = next.value;
+    return {
+        abort: out.abort,
+        injectCount: applyStageInjects(state, out.inject),
+        warnings: out.warnings,
+    };
+}

package/esm/src/kernel/engine/runner/state.d.ts

@@ -1,3 +1,5 @@
+import type { TurnTaint } from '../../../guardrails/types.js';
+import type { TurnToolSnapshot } from '../../tools/types.js';
 import type { ResolvedGeneration, TurnEvent, TurnHistoryMessage, TurnRequest, TurnStop } from '../../types.js';
 interface StepExecutionState {
     currentHistory: TurnHistoryMessage[];
@@ -5,8 +7,37 @@ interface StepExecutionState {
     sawTokensEvent: boolean;
     allEmittedEvents: TurnEvent[];
     attemptEvents: TurnEvent[];
+    /**
+     * True when progressive yield withheld user-visible events during this attempt.
+     *
+     * The attempt gate needs it: if the mid-stream window tripped but the final
+     * verdict on the whole text passes, nothing was streamed, so the buffered
+     * text must be released rather than silently dropped.
+     */
+    withheldVisible?: boolean;
+    /**
+     * Untrusted remote content this turn has already read.
+     *
+     * Accumulates across tool calls so a later call can be judged against what the
+     * turn has ingested, not just its own arguments.
+     */
+    taint?: TurnTaint;
     /** Last provider stop from a discarded provider `done` event. */
     lastStop?: TurnStop;
+    /** Tool snapshot at tool pause — emitted on terminal `done` when `stop.kind === 'tool'`. */
+    toolSnapshot?: TurnToolSnapshot;
+    /** Latest Google Interactions id observed on the current provider stream. */
+    lastInteractionId?: string;
+    /** Pending Interactions `function_result` continuation for the next provider step. */
+    interactionsContinuation?: {
+        previousInteractionId: string;
+        input: Record<string, unknown>[];
+    };
+    /**
+     * Generation whose opening `input` was folded into `currentHistory` for steering.
+     * Compared by identity so repair retries (new generation) fold again.
+     */
+    foldedForGeneration?: ResolvedGeneration;
 }
 interface AttemptFlowState {
     currentAttempt: number;

package/esm/src/kernel/engine/runner/steps.d.ts

@@ -6,7 +6,7 @@ declare function executeAttempt(args: {
     generation: ResolvedGeneration;
     system: string;
     provider: ModelProvider;
-    gemini: Record<string, unknown>[];
+    upstream: Record<string, unknown>[];
     state: StepExecutionState;
 }): AsyncGenerator<TurnEvent, {
     pendingTools: TurnEvent[];