theorum 0.1.2 → 0.1.4

package/src/observability/trace-attach.ts

@@ -0,0 +1,96 @@
+import { providerCompleteRequest } from '../kernel/registry/provider-request.ts';
+import type { ResolvedGeneration, TurnRequest } from '../kernel/types.ts';
+import { tapeGemini } from '../providers/gemini-tape.ts';
+import { toInteractionsBody } from '../providers/interactions.ts';
+import type { TraceRecord } from './trace-record.ts';
+import { httpStatus } from './trace-usage.ts';
+
+function attachResolved(
+  record: TraceRecord,
+  args: {
+    safe: TurnRequest;
+    model?: string;
+    bucket?: string;
+    generation?: ResolvedGeneration;
+  },
+): void {
+  const { safe, model, bucket, generation } = args;
+  if (safe.projectId) {
+    record.projectId = safe.projectId;
+  }
+  if (safe.select) {
+    record.select = safe.select;
+  }
+  if (safe.thinking !== undefined) {
+    record.thinking = safe.thinking;
+  }
+  if (safe.tools) {
+    record.tools = safe.tools;
+  }
+  if (safe.metadata) {
+    record.metadata = safe.metadata;
+  }
+  if (model) {
+    record.model = {
+      id: model,
+      apiId: generation?.apiId ?? model,
+    };
+  }
+  if (bucket) {
+    record.bucket = bucket;
+  }
+  if (generation) {
+    record.generation = {
+      thinking: generation.thinking,
+      summaries: generation.summaries,
+      temperature: generation.temperature,
+      maxOutputTokens: generation.maxOutputTokens,
+      builtins: generation.builtins,
+      custom: generation.custom,
+      structured: generation.structured,
+      image: generation.image,
+    };
+  }
+}
+
+async function attachTape(
+  record: TraceRecord,
+  args: {
+    gemini?: unknown;
+    canary?: string;
+    system?: string;
+    generation?: ResolvedGeneration;
+  },
+): Promise<void> {
+  const { gemini, canary, system, generation } = args;
+  if (gemini !== undefined) {
+    record.gemini = await tapeGemini(gemini, canary ?? '');
+  }
+  if (generation && system !== undefined) {
+    record.wire = await tapeGemini(
+      toInteractionsBody(providerCompleteRequest(generation, system)),
+      canary ?? '',
+    );
+  }
+}
+
+function attachUsage(
+  record: TraceRecord,
+  gemini: unknown,
+  done: Record<string, unknown> | undefined,
+): void {
+  if (done?.usage !== undefined) {
+    record.usage = done.usage;
+  }
+  const upStatus = httpStatus(gemini);
+  if (upStatus !== undefined || done) {
+    record.upstream = {
+      status: upStatus,
+      id: done?.id,
+      finish: done?.status,
+      serviceTier: done?.service_tier ?? done?.serviceTier,
+    };
+  }
+}
+
+export { attachResolved, attachTape, attachUsage };

package/src/observability/trace-record.ts

@@ -0,0 +1,254 @@
+/**
+ * Trace record construction types.
+ *
+ * Trace records preserve useful execution evidence while hashing or omitting
+ * unsafe media bytes and canary-sensitive content.
+ *
+ * @module
+ */
+
+import { publicError } from '../guardrails/error.ts';
+import { sanitizeText, sanitizeTurnRequest } from '../guardrails/sanitize.ts';
+import { OMIT_CANARY } from '../kernel/engine/boundary.ts';
+import { sha256 } from '../kernel/engine/hash.ts';
+import type { ResolvedGeneration, TurnBlob, TurnEvent, TurnRequest } from '../kernel/types.ts';
+import { attachResolved, attachTape, attachUsage } from './trace-attach.ts';
+import { completedInteraction } from './trace-usage.ts';
+
+const TRACE_VERSION = 2;
+const TITLE_MAX = 80;
+
+/** Hash-only image reference stored in trace records. */
+export interface TraceImage {
+  mimeType: string;
+  sha256: string;
+}
+
+/** Trace-safe copy of a public turn event. */
+export interface TraceEvent {
+  type: string;
+  text?: string;
+  tool?: {
+    name: string;
+    arguments?: Record<string, unknown>;
+    result?: { status: string; finding?: string; data?: Record<string, unknown> };
+  };
+  structured?: unknown;
+  media?: TraceImage;
+  grounding?: TurnEvent['grounding'];
+  evidence?: TurnEvent['evidence'];
+  error?: string;
+}
+
+/** Complete trace-safe record for one attempted turn. */
+interface TraceRecord {
+  v: number;
+  id: string;
+  ts: number;
+  ms: number;
+  streamed: boolean;
+  cancelled: boolean;
+  previousInteractionId: string | null;
+  store: boolean | null;
+  profile: string;
+  title?: string;
+  projectId?: string;
+  select?: string;
+  thinking?: boolean;
+  tools?: TurnRequest['tools'];
+  metadata?: Record<string, unknown>;
+  model?: { id: string; apiId: string };
+  bucket?: string;
+  generation?: {
+    thinking: string;
+    summaries: string;
+    temperature: number;
+    maxOutputTokens: number;
+    builtins: string[];
+    custom: string[];
+    structured: string | null;
+    image: unknown;
+  };
+  input: {
+    text?: string;
+    role?: string;
+    slots?: Record<string, string>;
+    attachments: TraceImage[];
+    voice: TraceImage[];
+    images?: TraceImage[];
+    audio?: TraceImage[];
+  };
+  wire?: unknown;
+  events: TraceEvent[];
+  gemini?: unknown;
+  usage?: unknown;
+  upstream?: {
+    status?: unknown;
+    id?: unknown;
+    finish?: unknown;
+    serviceTier?: unknown;
+  };
+  ok: boolean;
+  error?: string;
+  errorInternal?: string;
+  app?: Record<string, unknown>;
+  cutout?: {
+    ok: boolean;
+    ms: number;
+    url?: string;
+    inSha256?: string;
+    outSha256?: string;
+    http?: unknown;
+    error?: string;
+  };
+}
+
+function hashBlobs(blobs: TurnBlob[] | undefined): Promise<TraceImage[]> {
+  if (!blobs) {
+    return Promise.resolve([]);
+  }
+  return Promise.all(
+    blobs.map(async (blob) => ({
+      mimeType: blob.mimeType,
+      sha256: await sha256(blob.data),
+    })),
+  );
+}
+
+async function snapshotEvent(event: TurnEvent): Promise<TraceEvent> {
+  const row: TraceEvent = { type: event.type };
+  if (event.text) {
+    row.text = sanitizeText(event.text);
+  }
+  if (event.error) {
+    row.error = event.error;
+  }
+  if (event.structured !== undefined) {
+    row.structured = event.structured;
+  }
+  if (event.tool) {
+    const { name, arguments: args, result } = event.tool;
+    row.tool = { name, arguments: args };
+    if (result) {
+      row.tool.result = {
+        status: result.status,
+        ...(result.finding ? { finding: result.finding } : {}),
+        ...(result.data ? { data: result.data } : {}),
+      };
+    }
+  }
+  if (event.media) {
+    row.media = {
+      mimeType: event.media.mimeType,
+      sha256: await sha256(event.media.data),
+    };
+  }
+  if (event.grounding) {
+    row.grounding = event.grounding;
+  }
+  if (event.evidence) {
+    row.evidence = event.evidence;
+  }
+  return row;
+}
+
+function requestForTrace(req: TurnRequest): TurnRequest {
+  try {
+    return sanitizeTurnRequest(req);
+  } catch {
+    return { profile: req.profile, input: {} };
+  }
+}
+
+function internalError(err: unknown): string | undefined {
+  if (typeof err === 'string') {
+    return sanitizeText(err);
+  }
+  if (err instanceof Error && err.message) {
+    return sanitizeText(err.message);
+  }
+  return undefined;
+}
+
+function titleFrom(text: string | undefined): string | undefined {
+  if (!text) {
+    return undefined;
+  }
+  const trimmed = text.trim().replaceAll(/\s+/g, ' ');
+  if (!trimmed) {
+    return undefined;
+  }
+  return trimmed.slice(0, TITLE_MAX);
+}
+
+function attachFailure(
+  record: TraceRecord,
+  thrown: unknown,
+  lastErr: TraceEvent | undefined,
+  canary: string | undefined,
+): void {
+  if (!record.ok) {
+    record.error = publicError(thrown ?? lastErr?.error);
+    const inside = internalError(thrown) ?? lastErr?.error;
+    if (inside) {
+      record.errorInternal = inside;
+    }
+  }
+  if (canary && JSON.stringify(record).includes(canary)) {
+    record.errorInternal = OMIT_CANARY;
+  }
+}
+
+async function buildRecord(args: {
+  req: TurnRequest;
+  events: TurnEvent[];
+  started: number;
+  model?: string;
+  bucket?: string;
+  thrown?: unknown;
+  gemini?: unknown;
+  canary?: string;
+  system?: string;
+  generation?: ResolvedGeneration;
+}): Promise<TraceRecord> {
+  const { req, events, started, model, bucket, thrown, gemini, canary, system, generation } = args;
+  const safe = requestForTrace(req);
+  const input = safe.input ?? {};
+  const snapped = await Promise.all(events.map((event) => snapshotEvent(event)));
+  const lastErr = [...snapped].reverse().find((row) => row.type === 'error');
+  const done = completedInteraction(gemini);
+  const status = done?.status;
+  const ok = !(thrown || lastErr) && status !== 'cancelled';
+  const record: TraceRecord = {
+    v: TRACE_VERSION,
+    id: crypto.randomUUID(),
+    ts: started,
+    ms: Date.now() - started,
+    streamed: true,
+    cancelled: status === 'cancelled',
+    previousInteractionId: safe.previousInteractionId ?? null,
+    store: safe.store ?? null,
+    profile: safe.profile,
+    input: {
+      text: input.text,
+      role: input.role,
+      slots: input.slots,
+      attachments: await hashBlobs(input.attachments),
+      voice: await hashBlobs(input.voice),
+    },
+    events: snapped,
+    ok,
+  };
+  const title = titleFrom(input.text);
+  if (title) {
+    record.title = title;
+  }
+  await attachTape(record, { gemini, canary, system, generation });
+  attachUsage(record, gemini, done);
+  attachResolved(record, { safe, model, bucket, generation });
+  attachFailure(record, thrown, lastErr, canary);
+  return record;
+}
+
+export type { TraceRecord };
+export { buildRecord };

package/src/observability/trace-usage.ts

@@ -0,0 +1,36 @@
+import { asRecord } from '../kernel/engine/record.ts';
+
+function eventName(row: Record<string, unknown>): string {
+  return String(row.event_type ?? row.eventType ?? '');
+}
+
+function completedInteraction(gemini: unknown): Record<string, unknown> | undefined {
+  if (!Array.isArray(gemini)) {
+    return undefined;
+  }
+  for (let i = gemini.length - 1; i >= 0; i -= 1) {
+    const row = asRecord(gemini[i]);
+    if (row) {
+      const name = eventName(row);
+      if (name === 'interaction.completed' || name === 'interaction.complete') {
+        return asRecord(row.interaction) ?? row;
+      }
+    }
+  }
+  return undefined;
+}
+
+function httpStatus(gemini: unknown): unknown {
+  if (!Array.isArray(gemini)) {
+    return undefined;
+  }
+  for (const item of gemini) {
+    const row = asRecord(item);
+    if (row && eventName(row) === 'http_response') {
+      return row.status;
+    }
+  }
+  return undefined;
+}
+
+export { completedInteraction, httpStatus };

package/src/observability/trace.ts

@@ -0,0 +1,143 @@
+/**
+ * Trace sink primitives for THEORUM.
+ *
+ * Tracing is host-injected: the kernel can write to a provided sink, a memory
+ * sink, a JSONL directory, or a noop sink. It does not read environment
+ * variables or own a database destination.
+ *
+ * @module
+ */
+
+import type { TraceRecord } from './trace-record.ts';
+
+const RETAIN_DAYS = 14;
+const HOURS_PER_DAY = 24;
+const MIN_PER_HOUR = 60;
+const SEC_PER_MIN = 60;
+const MS_PER_SEC = 1000;
+const RETAIN_MS = RETAIN_DAYS * HOURS_PER_DAY * MIN_PER_HOUR * SEC_PER_MIN * MS_PER_SEC;
+const KIB = 1024;
+const MIB = KIB * KIB;
+const ROTATE_MIB = 32;
+const ROTATE_BYTES = ROTATE_MIB * MIB;
+const FILE_DAY = /^turns-(\d{4}-\d{2}-\d{2})(?:-\d+)?\.jsonl$/;
+
+/** Minimal async destination for completed turn trace records. */
+interface TraceSink {
+  write: (record: TraceRecord) => Promise<void>;
+}
+
+/** Write a trace record without allowing trace failures to fail the turn. */
+async function writeTrace(sink: TraceSink, record: Promise<TraceRecord>): Promise<void> {
+  try {
+    await sink.write(await record);
+  } catch {
+    // Tracing must not fail the turn.
+  }
+}
+
+/** Trace sink that drops records. */
+function noopSink(): TraceSink {
+  return { write: () => Promise.resolve() };
+}
+
+/** Trace sink that appends records to a caller-owned array. */
+function memorySink(into: TraceRecord[]): TraceSink {
+  return {
+    write: (record) => {
+      into.push(record);
+      return Promise.resolve();
+    },
+  };
+}
+
+function dayStamp(ms: number): string {
+  return new Date(ms).toISOString().slice(0, 10);
+}
+
+function fileDay(name: string): string | undefined {
+  return FILE_DAY.exec(name)?.[1];
+}
+
+async function pruneTraces(dir: string, now: number): Promise<void> {
+  const cutoff = now - RETAIN_MS;
+  for await (const entry of Deno.readDir(dir)) {
+    const day = fileDay(entry.name);
+    if (day && Date.parse(`${day}T00:00:00.000Z`) < cutoff) {
+      await Deno.remove(`${dir}/${entry.name}`);
+    }
+  }
+}
+
+async function pickFile(dir: string, now: number): Promise<string> {
+  const day = dayStamp(now);
+  const base = `${dir}/turns-${day}.jsonl`;
+  try {
+    const info = await Deno.stat(base);
+    if ((info.size ?? 0) < ROTATE_BYTES) {
+      return base;
+    }
+  } catch {
+    return base;
+  }
+  return `${dir}/turns-${day}-${now}.jsonl`;
+}
+
+/** Trace sink that writes daily rotating JSONL files under a host-selected directory. */
+function jsonlSink(dir: string, now: () => number = Date.now): TraceSink {
+  return {
+    write: async (record) => {
+      const at = now();
+      await Deno.mkdir(dir, { recursive: true });
+      await pruneTraces(dir, at);
+      const path = await pickFile(dir, at);
+      await Deno.writeTextFile(path, `${JSON.stringify(record)}\n`, { append: true });
+    },
+  };
+}
+
+function insideDir(path: string, root: string): boolean {
+  let base = root;
+  if (root.endsWith('/')) {
+    base = root.slice(0, -1);
+  }
+  if (path === base) {
+    return true;
+  }
+  return path.startsWith(`${base}/`);
+}
+
+/** Resolve a trace directory while refusing relative paths or paths inside the clone. */
+function resolveTraceDir(args: {
+  dir?: string;
+  fallbackDir?: string;
+  cwd?: string;
+}): string | undefined {
+  const cwd = args.cwd ?? Deno.cwd();
+  if (args.dir === '') {
+    return undefined;
+  }
+  let dir = args.dir?.trim() || args.fallbackDir;
+  if (!dir) {
+    return undefined;
+  }
+  if (!dir.startsWith('/') || insideDir(dir, cwd)) {
+    dir = args.fallbackDir;
+  }
+  if (!dir || insideDir(dir, cwd)) {
+    return undefined;
+  }
+  return dir;
+}
+
+/** Build a JSONL sink from a host-supplied directory or return a noop sink. */
+function sinkFromDir(dir?: string, fallbackDir?: string): TraceSink {
+  const resolved = resolveTraceDir({ dir, fallbackDir });
+  if (!resolved) {
+    return noopSink();
+  }
+  return jsonlSink(resolved);
+}
+
+export type { TraceSink };
+export { jsonlSink, memorySink, noopSink, resolveTraceDir, sinkFromDir, writeTrace };

package/src/presets/google.ts

@@ -0,0 +1,147 @@
+/**
+ * Google / Gemini convenience preset.
+ *
+ * Kernel stays open (`string` pins). This pack owns Google vocabularies as
+ * real unions so host profiles stay typed when they opt into the preset.
+ *
+ * Call `registerGooglePreset()` at host startup for tools; use the exported
+ * types/constants when authoring image/speech-adjacent profile fields.
+ *
+ * @module
+ */
+
+import { registerTools } from '../kernel/registry/catalog.ts';
+import type { ProfileImageSpec, ProfileSpeechSpec, ToolCatalogEntry } from '../kernel/types.ts';
+
+/** Google Interactions / Gemini grounding builtins. */
+const GOOGLE_BUILTIN_TOOLS: Record<string, ToolCatalogEntry> = {
+  googleSearch: {
+    kind: 'builtin',
+    ui: true,
+    interactionsType: 'google_search',
+    openRouterPlugin: 'web',
+  },
+  googleMaps: {
+    kind: 'builtin',
+    ui: true,
+    interactionsType: 'google_maps',
+    conflictsWith: ['googleSearch', 'urlContext'],
+  },
+  urlContext: {
+    kind: 'builtin',
+    ui: true,
+    interactionsType: 'url_context',
+  },
+};
+
+/** Common Gemini image input MIME allowlist. */
+const GOOGLE_IMAGE_INPUT_MIMES = [
+  'image/png',
+  'image/jpeg',
+  'image/webp',
+  'image/heic',
+  'image/heif',
+] as const;
+
+/** Common voice/audio input MIME allowlist for Gemini multimodal. */
+const GOOGLE_VOICE_INPUT_MIMES = ['audio/webm', 'audio/wav', 'audio/mpeg', 'audio/mp4'] as const;
+
+/** Common Gemini TTS voice names for `outputs.speech.voice`. */
+const GOOGLE_SPEECH_VOICES = [
+  'Zephyr',
+  'Puck',
+  'Charon',
+  'Kore',
+  'Fenrir',
+  'Leda',
+  'Orus',
+  'Aoede',
+  'Callirrhoe',
+  'Autonoe',
+  'Enceladus',
+  'Iapetus',
+  'Umbriel',
+  'Algieba',
+  'Despina',
+  'Erinome',
+  'Algenib',
+  'Rasalgethi',
+  'Laomedeia',
+  'Achernar',
+  'Alnilam',
+  'Schedar',
+  'Gacrux',
+  'Pulcherrima',
+  'Achird',
+  'Zubenelgenubi',
+  'Vindemiatrix',
+  'Sadachbia',
+  'Sadaltager',
+  'Sulafat',
+] as const;
+
+type GoogleSpeechVoice = (typeof GOOGLE_SPEECH_VOICES)[number];
+
+/**
+ * `outputs.speech` pins narrowed to Google TTS vocabulary.
+ * Assignable to kernel `ProfileSpeechSpec`.
+ */
+type GoogleSpeechPins = Omit<ProfileSpeechSpec, 'voice'> & {
+  voice?: GoogleSpeechVoice;
+};
+
+/** Aspect ratios commonly accepted by Gemini image models. */
+const GOOGLE_IMAGE_ASPECT_RATIOS = [
+  '1:1',
+  '3:2',
+  '2:3',
+  '3:4',
+  '4:3',
+  '4:5',
+  '5:4',
+  '9:16',
+  '16:9',
+  '21:9',
+] as const;
+
+/** Image sizes commonly accepted by Gemini Flash Lite image. */
+const GOOGLE_IMAGE_SIZES = ['1K'] as const;
+
+type GoogleImageInputMime = (typeof GOOGLE_IMAGE_INPUT_MIMES)[number];
+type GoogleVoiceInputMime = (typeof GOOGLE_VOICE_INPUT_MIMES)[number];
+type GoogleImageAspectRatio = (typeof GOOGLE_IMAGE_ASPECT_RATIOS)[number];
+type GoogleImageSize = (typeof GOOGLE_IMAGE_SIZES)[number];
+
+/**
+ * `outputs.image` pins narrowed to Google image vocabulary.
+ * Assignable to kernel `ProfileImageSpec`.
+ */
+type GoogleImagePins = Omit<ProfileImageSpec, 'aspectRatio' | 'size' | 'mimeType'> & {
+  aspectRatio?: GoogleImageAspectRatio;
+  size?: GoogleImageSize;
+  mimeType?: GoogleImageInputMime | 'image/jpeg';
+};
+
+/** Register Google provider builtins into the process-local tool catalog. */
+function registerGooglePreset(): void {
+  registerTools(GOOGLE_BUILTIN_TOOLS);
+}
+
+export type {
+  GoogleImageAspectRatio,
+  GoogleImageInputMime,
+  GoogleImagePins,
+  GoogleImageSize,
+  GoogleSpeechPins,
+  GoogleSpeechVoice,
+  GoogleVoiceInputMime,
+};
+export {
+  GOOGLE_BUILTIN_TOOLS,
+  GOOGLE_IMAGE_ASPECT_RATIOS,
+  GOOGLE_IMAGE_INPUT_MIMES,
+  GOOGLE_IMAGE_SIZES,
+  GOOGLE_SPEECH_VOICES,
+  GOOGLE_VOICE_INPUT_MIMES,
+  registerGooglePreset,
+};

package/src/presets/mod.ts

@@ -0,0 +1,24 @@
+/**
+ * Optional THEORUM presets.
+ *
+ * Presets register host-convenience catalogs (provider builtins, media
+ * vocabularies, later model packs) without baking product opinions into the kernel.
+ *
+ * @module
+ */
+
+export type {
+  GoogleImageAspectRatio,
+  GoogleImageInputMime,
+  GoogleImagePins,
+  GoogleImageSize,
+  GoogleVoiceInputMime,
+} from './google.ts';
+export {
+  GOOGLE_BUILTIN_TOOLS,
+  GOOGLE_IMAGE_ASPECT_RATIOS,
+  GOOGLE_IMAGE_INPUT_MIMES,
+  GOOGLE_IMAGE_SIZES,
+  GOOGLE_VOICE_INPUT_MIMES,
+  registerGooglePreset,
+} from './google.ts';