tinker-agent 1.0.65

package/src/context/context-revision-compiler.ts

@@ -0,0 +1,281 @@
+import type { AgentMessage } from "../agent/types";
+import type { ContextRevisionId } from "../ids/runtime-id";
+import { CompiledContextValidator } from "./compiled-context-validator";
+import {
+  activeOverrideManifestHash,
+  canonicalRenderedMessageHash,
+  canonicalSequenceHash,
+  renderedMessageHash,
+} from "./compiled-context-hash";
+import { ContextProtocolValidator } from "./context-protocol-validator";
+import type {
+  CompiledContextEntry,
+  CompiledRevisionContext,
+  StoredContextSnapshotV5,
+  StoredInitialContextRevisionV5,
+  SwapOverride,
+} from "./context-revision";
+import {
+  immutableRecord,
+  materializeAgentMessages,
+  type ProtocolContextView,
+} from "./protocol-frame";
+
+export class ContextRevisionError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ContextRevisionError";
+  }
+}
+
+export class ContextRevisionCompiler {
+  constructor(
+    private readonly protocolValidator = new ContextProtocolValidator(),
+    private readonly compiledValidator = new CompiledContextValidator(),
+  ) {}
+
+  compileActive(snapshot: StoredContextSnapshotV5): CompiledRevisionContext {
+    validateSnapshotIdentity(snapshot);
+    this.protocolValidator.validate(snapshot.canonical);
+    const overrides = overrideMap(snapshot.activeOverrides);
+    const compiled = compileEntries({
+      canonical: snapshot.canonical,
+      revisionId: snapshot.revision.revisionId,
+      overrides,
+    });
+    this.compiledValidator.validateActive(
+      compiled,
+      snapshot.canonical,
+      snapshot.activeOverrides,
+    );
+    if (
+      compiled.manifest.canonicalSequenceHash !==
+        canonicalSequenceHash(snapshot.canonical) ||
+      canonicalSequenceHash(
+        snapshot.canonical,
+        snapshot.revision.sourceThroughOrdinal,
+      ) !== snapshot.revision.canonicalSequenceSha256 ||
+      renderedMessageHash(compiled.entries, snapshot.revision.sourceThroughOrdinal) !==
+        snapshot.revision.renderedMessageSha256
+    ) {
+      throw new ContextRevisionError(
+        "Active context revision prefix hash does not match stored history.",
+      );
+    }
+    return compiled;
+  }
+
+  compileProspective(input: {
+    active: CompiledRevisionContext;
+    canonical: ProtocolContextView;
+    activeOverrides: readonly SwapOverride[];
+    addedOverrides: readonly SwapOverride[];
+  }): CompiledRevisionContext {
+    this.protocolValidator.validate(input.canonical);
+    if (
+      input.active.sessionId !== input.canonical.sessionId ||
+      input.active.canonicalThroughOrdinal !== input.canonical.messages.length ||
+      input.active.manifest.canonicalSequenceHash !==
+        canonicalSequenceHash(input.canonical)
+    ) {
+      throw new ContextRevisionError(
+        "Prospective compilation base does not match canonical history.",
+      );
+    }
+    this.compiledValidator.validateActive(
+      input.active,
+      input.canonical,
+      input.activeOverrides,
+    );
+    const candidateOverrides = [...input.activeOverrides, ...input.addedOverrides];
+    const overrides = overrideMap(candidateOverrides);
+    if (overrides.size !== candidateOverrides.length) {
+      throw new ContextRevisionError("Prospective overrides contain duplicate IDs.");
+    }
+    const compiled = compileEntries({
+      canonical: input.canonical,
+      revisionId: input.active.revisionId,
+      overrides,
+    });
+    this.compiledValidator.validateProspective(
+      compiled,
+      input.canonical,
+      candidateOverrides,
+    );
+    return compiled;
+  }
+}
+
+function compileEntries(input: {
+  canonical: ProtocolContextView;
+  revisionId: CompiledRevisionContext["revisionId"];
+  overrides: ReadonlyMap<string, SwapOverride>;
+}): CompiledRevisionContext {
+  const materialized = materializeAgentMessages(input.canonical.messages);
+  const entries = input.canonical.messages.map((record, index) => {
+    const canonicalMessage = requireItem(materialized, index, "canonical message");
+    const override = input.overrides.get(record.messageId);
+    const message =
+      override === undefined
+        ? canonicalMessage
+        : swappedToolMessage(canonicalMessage, override);
+    return Object.freeze<CompiledContextEntry>({
+      frameId: record.frameId,
+      messageId: record.messageId,
+      ordinal: record.ordinal,
+      representation: override === undefined ? "canonical" : "swapped",
+      sourceContentSha256: record.contentSha256,
+      message: immutableRecord(message),
+    });
+  });
+  const manifest = Object.freeze({
+    frameCount: input.canonical.frames.length,
+    messageCount: entries.length,
+    canonicalSequenceHash: canonicalSequenceHash(input.canonical),
+    renderedMessageHash: renderedMessageHash(entries),
+  });
+  return Object.freeze({
+    sessionId: input.canonical.sessionId,
+    revisionId: input.revisionId,
+    canonicalThroughOrdinal: input.canonical.messages.length,
+    entries: Object.freeze(entries),
+    manifest,
+  });
+}
+
+function swappedToolMessage(
+  message: AgentMessage,
+  override: SwapOverride,
+): AgentMessage {
+  if (message.role !== "tool") {
+    throw new ContextRevisionError(
+      `Swap override at ordinal ${override.ordinal} does not target a tool message.`,
+    );
+  }
+  return {
+    ...message,
+    content: override.renderedContent,
+  };
+}
+
+function validateSnapshotIdentity(snapshot: StoredContextSnapshotV5): void {
+  if (
+    snapshot.meta.sessionId !== snapshot.canonical.sessionId ||
+    snapshot.revision.sessionId !== snapshot.canonical.sessionId ||
+    snapshot.meta.activeRevisionId !== snapshot.revision.revisionId
+  ) {
+    throw new ContextRevisionError(
+      "Active context revision identity does not match canonical history.",
+    );
+  }
+  if (snapshot.revision.keepFromOrdinal !== 1) {
+    throw new ContextRevisionError("Unsupported active context revision.");
+  }
+  const firstFrame = snapshot.canonical.frames[0];
+  const firstMessage = snapshot.canonical.messages[0];
+  if (
+    firstFrame?.kind !== "system" ||
+    firstFrame.firstOrdinal !== 1 ||
+    firstMessage?.role !== "system" ||
+    firstMessage.ordinal !== 1 ||
+    snapshot.canonical.messages.at(-1)?.ordinal !== snapshot.canonical.messages.length
+  ) {
+    throw new ContextRevisionError(
+      "Canonical context does not preserve the initial_full ordinal boundary.",
+    );
+  }
+  const boundaryFrame = snapshot.canonical.frames.find(
+    (frame) => frame.lastOrdinal === snapshot.revision.sourceThroughOrdinal,
+  );
+  if (
+    snapshot.revision.sourceThroughOrdinal > snapshot.canonical.messages.length ||
+    boundaryFrame?.state !== "closed"
+  ) {
+    throw new ContextRevisionError(
+      "Active context revision source boundary is not a closed frame boundary.",
+    );
+  }
+
+  if (snapshot.revision.kind === "initial_full") {
+    if (
+      snapshot.revision.revisionNumber !== 1 ||
+      snapshot.revision.parentRevisionId !== null ||
+      snapshot.revision.sourceThroughOrdinal !== 1 ||
+      snapshot.revision.addedOverrideCount !== 0 ||
+      snapshot.revision.totalOverrideCount !== 0 ||
+      snapshot.activeOverrides.length !== 0
+    ) {
+      throw new ContextRevisionError("Initial context revision invariant failed.");
+    }
+  } else if (
+    snapshot.revision.revisionNumber < 2 ||
+    snapshot.revision.addedOverrideCount < 1 ||
+    snapshot.revision.totalOverrideCount < snapshot.revision.addedOverrideCount ||
+    snapshot.revision.policyVersion !== "swap-only-v1" ||
+    snapshot.revision.rendererFormat !== "swap-observation-v1" ||
+    snapshot.activeOverrides.length !== snapshot.revision.totalOverrideCount
+  ) {
+    throw new ContextRevisionError("Swap context revision invariant failed.");
+  }
+
+  if (
+    activeOverrideManifestHash(snapshot.activeOverrides) !==
+    snapshot.revision.overrideManifestSha256
+  ) {
+    throw new ContextRevisionError(
+      "Active context revision override manifest does not match stored overrides.",
+    );
+  }
+}
+
+export function createInitialContextRevision(input: {
+  revisionId: ContextRevisionId;
+  canonical: ProtocolContextView;
+  createdAt: string;
+}): StoredInitialContextRevisionV5 {
+  const firstFrame = input.canonical.frames[0];
+  const firstMessage = input.canonical.messages[0];
+  if (
+    input.canonical.messages.length !== 1 ||
+    input.canonical.frames.length !== 1 ||
+    firstFrame?.kind !== "system" ||
+    firstFrame.state !== "closed" ||
+    firstFrame.firstOrdinal !== 1 ||
+    firstFrame.lastOrdinal !== 1 ||
+    firstMessage?.role !== "system" ||
+    firstMessage.ordinal !== 1
+  ) {
+    throw new ContextRevisionError(
+      "Initial context revision requires exactly one closed system frame.",
+    );
+  }
+  return Object.freeze({
+    revisionId: input.revisionId,
+    sessionId: input.canonical.sessionId,
+    revisionNumber: 1,
+    parentRevisionId: null,
+    kind: "initial_full",
+    keepFromOrdinal: 1,
+    sourceThroughOrdinal: 1,
+    addedOverrideCount: 0,
+    totalOverrideCount: 0,
+    overrideManifestSha256: activeOverrideManifestHash([]),
+    canonicalSequenceSha256: canonicalSequenceHash(input.canonical, 1),
+    renderedMessageSha256: canonicalRenderedMessageHash(input.canonical, 1),
+    createdAt: input.createdAt,
+  });
+}
+
+function overrideMap(
+  overrides: readonly SwapOverride[],
+): ReadonlyMap<string, SwapOverride> {
+  return new Map(overrides.map((override) => [override.messageId, override] as const));
+}
+
+function requireItem<T>(items: readonly T[], index: number, name: string): T {
+  const item = items[index];
+  if (item === undefined) {
+    throw new ContextRevisionError(`Missing ${name} at index ${index}.`);
+  }
+  return item;
+}

package/src/context/context-revision.ts

@@ -0,0 +1,111 @@
+import type { AgentMessage } from "../agent/types";
+import type {
+  ContextRevisionId,
+  MessageId,
+  ProtocolFrameId,
+  SessionId,
+} from "../ids/runtime-id";
+import type { ModelRequestInput } from "../model/model-client";
+import type { MessageSource } from "./context-source";
+import type { ProtocolContextView } from "./protocol-frame";
+
+export type StoredInitialContextRevisionV5 = {
+  readonly revisionId: ContextRevisionId;
+  readonly sessionId: SessionId;
+  readonly revisionNumber: 1;
+  readonly parentRevisionId: null;
+  readonly kind: "initial_full";
+  readonly keepFromOrdinal: 1;
+  readonly sourceThroughOrdinal: 1;
+  readonly addedOverrideCount: 0;
+  readonly totalOverrideCount: 0;
+  readonly overrideManifestSha256: string;
+  readonly canonicalSequenceSha256: string;
+  readonly renderedMessageSha256: string;
+  readonly createdAt: string;
+};
+
+export type StoredSwapContextRevisionV5 = {
+  readonly revisionId: ContextRevisionId;
+  readonly sessionId: SessionId;
+  readonly revisionNumber: number;
+  readonly parentRevisionId: ContextRevisionId;
+  readonly kind: "swap_only";
+  readonly keepFromOrdinal: 1;
+  readonly sourceThroughOrdinal: number;
+  readonly addedOverrideCount: number;
+  readonly totalOverrideCount: number;
+  readonly overrideManifestSha256: string;
+  readonly canonicalSequenceSha256: string;
+  readonly renderedMessageSha256: string;
+  readonly policyVersion: "swap-only-v1";
+  readonly rendererFormat: "swap-observation-v1";
+  readonly planSha256: string;
+  readonly createdAt: string;
+};
+
+export type StoredContextRevisionV5 =
+  | StoredInitialContextRevisionV5
+  | StoredSwapContextRevisionV5;
+
+export type SwapOverride = {
+  readonly frameId: ProtocolFrameId;
+  readonly messageId: MessageId;
+  readonly ordinal: number;
+  readonly source: MessageSource;
+  readonly originalContentSha256: string;
+  readonly renderedContent: string;
+  readonly renderedContentSha256: string;
+  readonly originalBytes: number;
+  readonly renderedBytes: number;
+  readonly byteSavings: number;
+};
+
+export type StoredSwapOverrideV5 = SwapOverride & {
+  readonly introducedRevisionId: ContextRevisionId;
+  readonly rendererFormat: "swap-observation-v1";
+  readonly createdAt: string;
+};
+
+export type StoredContextSnapshotV5 = {
+  readonly meta: {
+    readonly sessionId: SessionId;
+    readonly activeRevisionId: ContextRevisionId;
+  };
+  readonly revision: StoredContextRevisionV5;
+  readonly activeOverrides: readonly StoredSwapOverrideV5[];
+  readonly canonical: ProtocolContextView;
+};
+
+export type CompiledContextEntry = {
+  readonly frameId: ProtocolFrameId;
+  readonly messageId: MessageId;
+  readonly ordinal: number;
+  readonly representation: "canonical" | "swapped";
+  readonly sourceContentSha256: string;
+  readonly message: AgentMessage;
+};
+
+export type CompiledContextManifest = {
+  readonly frameCount: number;
+  readonly messageCount: number;
+  readonly canonicalSequenceHash: string;
+  readonly renderedMessageHash: string;
+};
+
+export type CompiledRevisionContext = {
+  readonly sessionId: SessionId;
+  readonly revisionId: ContextRevisionId;
+  readonly canonicalThroughOrdinal: number;
+  readonly entries: readonly CompiledContextEntry[];
+  readonly manifest: CompiledContextManifest;
+};
+
+export type BuiltContextRequest = {
+  readonly canonical: ProtocolContextView;
+  readonly revision: StoredContextRevisionV5;
+  readonly activeOverrides: readonly SwapOverride[];
+  readonly compiled: CompiledRevisionContext;
+  readonly request: ModelRequestInput;
+  readonly candidateUserPromptIncluded: boolean;
+};

package/src/context/context-source.ts

@@ -0,0 +1,30 @@
+import { parseMessageId, type MessageId } from "../ids/runtime-id";
+
+export type MessageSource = `ctx://message/${string}`;
+
+const MESSAGE_SOURCE_PREFIX = "ctx://message/";
+
+export class MessageSourceParseError extends Error {
+  readonly code = "RECALL_SOURCE_INVALID" as const;
+
+  constructor(source: string, options?: ErrorOptions) {
+    super(`Invalid message source: ${JSON.stringify(source)}.`, options);
+    this.name = "MessageSourceParseError";
+  }
+}
+
+export function formatMessageSource(messageId: MessageId): MessageSource {
+  const canonicalId = parseMessageId(messageId);
+  return `${MESSAGE_SOURCE_PREFIX}${canonicalId}`;
+}
+
+export function parseMessageSource(source: string): MessageId {
+  if (!source.startsWith(MESSAGE_SOURCE_PREFIX)) {
+    throw new MessageSourceParseError(source);
+  }
+  try {
+    return parseMessageId(source.slice(MESSAGE_SOURCE_PREFIX.length));
+  } catch (error) {
+    throw new MessageSourceParseError(source, { cause: error });
+  }
+}

package/src/context/context-swap-renderer.ts

@@ -0,0 +1,272 @@
+import { contentHash } from "./protocol-frame";
+import { formatMessageSource } from "./context-source";
+import type { CanonicalMessageRecord, ToolResultRecord } from "./protocol-frame";
+import type { ToolRawResult, ToolRawResultKind } from "../tools/types";
+import { sha256, stableJsonStringify } from "../model/model-request-preflight";
+import type { SwapOverride } from "./context-revision";
+
+const MAX_METADATA_BYTES = 1_024;
+const MAX_SCALAR_BYTES = 256;
+
+export const SWAP_OBSERVATION_FORMAT = "swap-observation-v1" as const;
+
+export const SWAPPABLE_RAW_KINDS = Object.freeze([
+  "read",
+  "glob",
+  "grep",
+  "bash",
+  "task_output",
+  "web_search",
+  "web_fetch",
+  "mcp",
+] as const satisfies readonly ToolRawResultKind[]);
+
+export type SwappableRawKind = (typeof SWAPPABLE_RAW_KINDS)[number];
+
+export class SwapRenderUnsupportedError extends Error {
+  constructor(
+    readonly code: string,
+    message: string,
+  ) {
+    super(message);
+    this.name = "SwapRenderUnsupportedError";
+  }
+}
+
+export class ContextSwapRenderer {
+  render(input: {
+    message: Extract<CanonicalMessageRecord, { role: "tool" }>;
+    result: ToolResultRecord;
+  }): SwapOverride {
+    const { message, result } = input;
+    if (
+      result.toolMessageId !== message.messageId ||
+      result.frameId !== message.frameId ||
+      result.toolCallId !== message.toolCallId ||
+      result.observationSha256 !== message.contentSha256
+    ) {
+      throw new SwapRenderUnsupportedError(
+        "source_hash_mismatch",
+        "Canonical tool result does not match its message.",
+      );
+    }
+    if (result.completion.kind !== "returned") {
+      throw new SwapRenderUnsupportedError(
+        "synthetic_completion",
+        "Synthetic tool completions cannot be swapped.",
+      );
+    }
+    const raw = result.completion.raw;
+    if (!isSwappableRawResult(raw)) {
+      throw new SwapRenderUnsupportedError(
+        "raw_kind_not_allowlisted",
+        "Tool result kind is not eligible for swap rendering.",
+      );
+    }
+    assertHistoricalRawIsStable(raw);
+
+    const source = formatMessageSource(message.messageId);
+    const metadata = renderMetadata(raw);
+    const renderedContent = [
+      "[Tinker historical tool observation swapped]",
+      `source=${source}`,
+      `contentSha256=${message.contentSha256}`,
+      `tool=${stableJsonStringify(compactExternalString(message.name))}`,
+      `metadata=${metadata}`,
+      "historical=Use Recall get with source to recover the original observation.",
+      `current=${currentGuidance(raw.kind)}`,
+    ].join("\n");
+    const originalBytes = utf8Bytes(message.content);
+    const renderedBytes = utf8Bytes(renderedContent);
+    if (renderedBytes >= originalBytes) {
+      throw new SwapRenderUnsupportedError(
+        "placeholder_not_smaller",
+        "Rendered placeholder is not smaller than its canonical observation.",
+      );
+    }
+    return Object.freeze({
+      frameId: message.frameId,
+      messageId: message.messageId,
+      ordinal: message.ordinal,
+      source,
+      originalContentSha256: message.contentSha256,
+      renderedContent,
+      renderedContentSha256: contentHash(renderedContent),
+      originalBytes,
+      renderedBytes,
+      byteSavings: originalBytes - renderedBytes,
+    });
+  }
+}
+
+export function isSwappableRawResult(
+  raw: ToolRawResult,
+): raw is Extract<ToolRawResult, { kind: SwappableRawKind }> {
+  return (SWAPPABLE_RAW_KINDS as readonly string[]).includes(raw.kind);
+}
+
+function assertHistoricalRawIsStable(
+  raw: Extract<ToolRawResult, { kind: SwappableRawKind }>,
+): void {
+  if (raw.kind === "bash" && raw.status === "running") {
+    throw new SwapRenderUnsupportedError(
+      "running_task",
+      "Running Bash results cannot be rendered as historical placeholders.",
+    );
+  }
+  if (
+    raw.kind === "task_output" &&
+    (raw.status === "running" || raw.status === "stopping")
+  ) {
+    throw new SwapRenderUnsupportedError(
+      "running_task",
+      "Running task output cannot be rendered as a historical placeholder.",
+    );
+  }
+}
+
+function renderMetadata(
+  raw: Extract<ToolRawResult, { kind: SwappableRawKind }>,
+): string {
+  const entries: readonly (readonly [string, unknown])[] = metadataEntries(raw);
+  const metadata: Record<string, unknown> = {};
+  for (const [key, value] of entries) {
+    if (value === undefined) {
+      continue;
+    }
+    const next = {
+      ...metadata,
+      [key]: typeof value === "string" ? compactExternalString(value) : value,
+    };
+    if (utf8Bytes(stableJsonStringify(next)) <= MAX_METADATA_BYTES) {
+      Object.assign(metadata, { [key]: next[key] });
+    }
+  }
+  const rendered = stableJsonStringify(metadata);
+  if (utf8Bytes(rendered) > MAX_METADATA_BYTES) {
+    throw new SwapRenderUnsupportedError(
+      "metadata_too_large",
+      "Swap placeholder metadata exceeded its byte limit.",
+    );
+  }
+  return rendered;
+}
+
+function metadataEntries(
+  raw: Extract<ToolRawResult, { kind: SwappableRawKind }>,
+): readonly (readonly [string, unknown])[] {
+  switch (raw.kind) {
+    case "read":
+      return [
+        ["filePath", raw.filePath],
+        ["startLine", raw.startLine],
+        ["endLine", raw.endLine],
+        ["sha256", raw.sha256],
+        ["sizeBytes", raw.sizeBytes],
+      ];
+    case "glob":
+      return [
+        ["pattern", raw.pattern],
+        ["searchPath", raw.searchPath],
+        ["matchCount", raw.matchCount],
+      ];
+    case "grep":
+      return [
+        ["pattern", raw.pattern],
+        ["searchPath", raw.searchPath],
+        ["mode", raw.mode],
+        ["numMatches", raw.numMatches],
+        ["truncated", raw.truncated],
+      ];
+    case "bash":
+      return [
+        ["status", raw.status],
+        ["exitCode", raw.exitCode],
+        ["outputFilePath", raw.outputFilePath],
+        ["outputBytes", raw.outputBytes],
+        ["command", raw.command],
+      ];
+    case "task_output":
+      return [
+        ["taskId", raw.taskId],
+        ["status", raw.status],
+        ["outputFilePath", raw.outputFilePath],
+        ["outputBytes", raw.outputBytes],
+      ];
+    case "web_search":
+      return [
+        ["query", raw.query],
+        ["resultCount", raw.resultCount],
+        ["requestId", raw.requestId],
+      ];
+    case "web_fetch":
+      return [
+        ["url", raw.url],
+        ["finalUrl", raw.finalUrl],
+        ["title", raw.title],
+        ["httpStatusCode", raw.httpStatusCode],
+      ];
+    case "mcp":
+      return [
+        ["serverName", raw.serverName],
+        ["serverToolName", raw.serverToolName],
+        ["isError", raw.isError],
+        ["contentBlockCount", raw.contentBlockCount],
+      ];
+  }
+}
+
+function currentGuidance(kind: SwappableRawKind): string {
+  switch (kind) {
+    case "read":
+      return "Use Read to inspect the current file state before relying on historical content.";
+    case "glob":
+      return "Use Glob to inspect the current workspace matches.";
+    case "grep":
+      return "Use Grep to inspect current workspace matches.";
+    case "bash":
+      return "Inspect current state before deciding whether a historical command should be rerun.";
+    case "task_output":
+      return "Use TaskOutput to inspect the task's current recorded output and status.";
+    case "web_search":
+      return "Use WebSearch when current search results are required.";
+    case "web_fetch":
+      return "Use WebFetch when the current page content is required.";
+    case "mcp":
+      return "Call the MCP tool again only when current external state is required.";
+  }
+}
+
+function compactExternalString(
+  value: string,
+):
+  | string
+  | { readonly prefix: string; readonly byteLength: number; readonly sha256: string } {
+  const byteLength = utf8Bytes(value);
+  if (byteLength <= MAX_SCALAR_BYTES) {
+    return value;
+  }
+  return Object.freeze({
+    prefix: utf8Prefix(value, MAX_SCALAR_BYTES),
+    byteLength,
+    sha256: sha256(value),
+  });
+}
+
+function utf8Prefix(value: string, maximumBytes: number): string {
+  let prefix = "";
+  let bytes = 0;
+  for (const character of value) {
+    const characterBytes = utf8Bytes(character);
+    if (bytes + characterBytes > maximumBytes) {
+      break;
+    }
+    prefix += character;
+    bytes += characterBytes;
+  }
+  return prefix;
+}
+
+function utf8Bytes(value: string): number {
+  return Buffer.byteLength(value, "utf8");
+}