tinker-agent 1.0.65 → 1.2.0

package/src/image/image-types.ts

@@ -0,0 +1,262 @@
+import { createHash } from "node:crypto";
+import { isCanonicalUuidV7 } from "../ids/uuid-v7";
+import { sha256, stableJsonStringify } from "../model/model-request-preflight";
+import { IMAGE_INPUT_POLICY } from "./image-input-policy";
+
+export type ImageMimeType = (typeof IMAGE_INPUT_POLICY.allowedMimeTypes)[number];
+
+export type ImageAttachmentId = string & {
+  readonly __imageAttachmentId: "image-attachment";
+};
+
+export type ImageAssetId = string & {
+  readonly __imageAssetId: "image-asset";
+};
+
+export type CodePointRange = {
+  readonly start: number;
+  readonly end: number;
+};
+
+export type ImageAssetRef = {
+  readonly assetId: ImageAssetId;
+  readonly mimeType: ImageMimeType;
+  readonly byteLength: number;
+  readonly width: number;
+  readonly height: number;
+};
+
+export type UserImageAttachment = ImageAssetRef & {
+  readonly attachmentId: ImageAttachmentId;
+  readonly label: string;
+  readonly range: CodePointRange;
+  readonly originalName: string;
+};
+
+export type UserMessage = {
+  readonly role: "user";
+  readonly content: string;
+  readonly attachments?: readonly UserImageAttachment[];
+};
+
+export function imageAssetIdForBytes(bytes: Uint8Array): ImageAssetId {
+  return createHash("sha256").update(bytes).digest("hex") as ImageAssetId;
+}
+
+export function parseImageAssetId(value: string): ImageAssetId {
+  if (!/^[0-9a-f]{64}$/.test(value)) {
+    throw new Error(`Invalid image asset ID: ${JSON.stringify(value)}.`);
+  }
+  return value as ImageAssetId;
+}
+
+export function parseImageAttachmentId(value: string): ImageAttachmentId {
+  if (!isCanonicalUuidV7(value)) {
+    throw new Error(`Invalid image attachment ID: ${JSON.stringify(value)}.`);
+  }
+  return value as ImageAttachmentId;
+}
+
+export function normalizeOriginalImageName(value: string): string {
+  const normalized = value.normalize("NFC");
+  validateOriginalImageName(normalized);
+  return normalized;
+}
+
+export function validateOriginalImageName(value: string): void {
+  if (
+    value === "" ||
+    value !== value.normalize("NFC") ||
+    value === "." ||
+    value === ".." ||
+    value.includes("/") ||
+    value.includes("\\") ||
+    hasControlCharacter(value) ||
+    Buffer.byteLength(value, "utf8") > 255
+  ) {
+    throw new Error(`Invalid image original name: ${JSON.stringify(value)}.`);
+  }
+}
+
+export function validateImageAssetRef(asset: ImageAssetRef): void {
+  parseImageAssetId(asset.assetId);
+  if (!IMAGE_INPUT_POLICY.allowedMimeTypes.includes(asset.mimeType)) {
+    throw new Error(`Unsupported image MIME type: ${JSON.stringify(asset.mimeType)}.`);
+  }
+  requirePositiveSafeInteger(asset.byteLength, "image byteLength");
+  requirePositiveSafeInteger(asset.width, "image width");
+  requirePositiveSafeInteger(asset.height, "image height");
+  if (asset.byteLength > IMAGE_INPUT_POLICY.maxBytesPerImage) {
+    throw new Error(
+      `Image byte length ${asset.byteLength} exceeds ${IMAGE_INPUT_POLICY.maxBytesPerImage}.`,
+    );
+  }
+  if (
+    asset.width > IMAGE_INPUT_POLICY.maxLongEdge ||
+    asset.height > IMAGE_INPUT_POLICY.maxLongEdge
+  ) {
+    throw new Error(
+      `Image dimensions ${asset.width}x${asset.height} exceed ${IMAGE_INPUT_POLICY.maxLongEdge}px.`,
+    );
+  }
+  const pixels = asset.width * asset.height;
+  if (!Number.isSafeInteger(pixels) || pixels > IMAGE_INPUT_POLICY.maxPixels) {
+    throw new Error(
+      `Image pixel count ${pixels} exceeds ${IMAGE_INPUT_POLICY.maxPixels}.`,
+    );
+  }
+}
+
+export function validateUserMessage(message: UserMessage): void {
+  if (message.role !== "user" || typeof message.content !== "string") {
+    throw new Error("Invalid canonical user message.");
+  }
+  const attachments = message.attachments;
+  if (attachments !== undefined && attachments.length === 0) {
+    throw new Error("Canonical user message attachments must be omitted when empty.");
+  }
+  if (attachments === undefined) {
+    if (message.content.trim() === "") {
+      throw new Error("Canonical user message must not be empty.");
+    }
+    return;
+  }
+  if (attachments.length > IMAGE_INPUT_POLICY.maxImagesPerMessage) {
+    throw new Error(
+      `User message has ${attachments.length} images; maximum is ${IMAGE_INPUT_POLICY.maxImagesPerMessage}.`,
+    );
+  }
+  if (message.content.trim() === "" && attachments.length === 0) {
+    throw new Error("Canonical user message must contain text or an image.");
+  }
+
+  const chars = [...message.content];
+  const attachmentIds = new Set<string>();
+  const labels = new Set<string>();
+  let previousEnd = -1;
+  for (let index = 0; index < attachments.length; index += 1) {
+    const attachment = attachments[index];
+    parseImageAttachmentId(attachment.attachmentId);
+    validateImageAssetRef(attachment);
+    validateOriginalImageName(attachment.originalName);
+    const expectedLabel = imageLabel(index + 1);
+    if (attachment.label !== expectedLabel) {
+      throw new Error(
+        `Image attachment label must be ${expectedLabel}; received ${attachment.label}.`,
+      );
+    }
+    validateCodePointRange(attachment.range, chars.length);
+    if (attachment.range.start < previousEnd) {
+      throw new Error("Image attachment ranges must be ordered and non-overlapping.");
+    }
+    const selected = chars.slice(attachment.range.start, attachment.range.end).join("");
+    if (selected !== attachment.label) {
+      throw new Error("Image attachment range does not match its label.");
+    }
+    if (attachmentIds.has(attachment.attachmentId)) {
+      throw new Error("Image attachment IDs must be unique within a message.");
+    }
+    if (labels.has(attachment.label)) {
+      throw new Error("Image attachment labels must be unique within a message.");
+    }
+    attachmentIds.add(attachment.attachmentId);
+    labels.add(attachment.label);
+    previousEnd = attachment.range.end;
+  }
+
+  assertNoUnboundImageLabels(message.content, attachments);
+}
+
+export function canonicalUserMessageHash(message: UserMessage): string {
+  validateUserMessage(message);
+  return sha256(
+    stableJsonStringify({
+      content: message.content,
+      ...(message.attachments === undefined
+        ? {}
+        : {
+            attachments: message.attachments.map((attachment) => ({
+              attachmentId: attachment.attachmentId,
+              assetId: attachment.assetId,
+              label: attachment.label,
+              range: attachment.range,
+              mimeType: attachment.mimeType,
+              byteLength: attachment.byteLength,
+              width: attachment.width,
+              height: attachment.height,
+              originalName: attachment.originalName,
+            })),
+          }),
+    }),
+  );
+}
+
+export function imageLabel(index: number): string {
+  if (
+    !Number.isSafeInteger(index) ||
+    index < 1 ||
+    index > IMAGE_INPUT_POLICY.maxImagesPerMessage
+  ) {
+    throw new Error(`Image label index is outside the MVP range: ${index}.`);
+  }
+  return `[Image #${index}]`;
+}
+
+export function codePointSlice(value: string, range: CodePointRange): string {
+  const chars = [...value];
+  validateCodePointRange(range, chars.length);
+  return chars.slice(range.start, range.end).join("");
+}
+
+function validateCodePointRange(range: CodePointRange, length: number): void {
+  if (
+    !Number.isSafeInteger(range.start) ||
+    !Number.isSafeInteger(range.end) ||
+    range.start < 0 ||
+    range.end <= range.start ||
+    range.end > length
+  ) {
+    throw new Error(
+      `Invalid image code-point range [${range.start}, ${range.end}) for length ${length}.`,
+    );
+  }
+}
+
+function assertNoUnboundImageLabels(
+  content: string,
+  attachments: readonly UserImageAttachment[],
+): void {
+  const chars = [...content];
+  for (const attachment of attachments) {
+    const label = [...attachment.label];
+    for (let start = 0; start <= chars.length - label.length; start += 1) {
+      if (!label.every((char, offset) => chars[start + offset] === char)) {
+        continue;
+      }
+      if (
+        start !== attachment.range.start ||
+        start + label.length !== attachment.range.end
+      ) {
+        throw new Error(
+          `Prompt contains an unbound literal ${attachment.label}; rewrite it before submitting.`,
+        );
+      }
+    }
+  }
+}
+
+function requirePositiveSafeInteger(value: number, name: string): void {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new Error(`${name} must be a positive safe integer.`);
+  }
+}
+
+function hasControlCharacter(value: string): boolean {
+  return [...value].some((character) => {
+    const codePoint = character.codePointAt(0);
+    return (
+      codePoint !== undefined &&
+      (codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f))
+    );
+  });
+}

package/src/instructions/project-instructions.ts

@@ -44,7 +44,7 @@ export async function loadProjectInstructions(
   return { workspaceRoot: canonicalRoot };
 }
 
-export function buildSystemPrompt(input: {
+export function buildBaseSystemPrompt(input: {
   workspaceRoot: string;
   runtimeInstructions: string;
   projectInstructions: ProjectInstructionsSnapshot;
@@ -79,6 +79,9 @@ ${content}</instruction_file>
   return `${runtime}\n\n${project}`;
 }
 
+export const buildSystemPrompt = buildBaseSystemPrompt;
+export { buildActiveSystemPrompt } from "../skills/skill-context";
+
 export function projectInstructionManifest(
   snapshot: ProjectInstructionsSnapshot,
 ): ProjectInstructionManifest | undefined {

package/src/mcp/mcp-manager.ts

@@ -21,10 +21,20 @@ export type McpClientFactory = (
 ) => Promise<McpClientConnection>;
 
 export type McpManager = {
-  executors: ToolExecutor[];
+  readonly executors: ToolExecutor[];
+  readonly inventory: McpInventorySnapshot;
   dispose(): Promise<void>;
 };
 
+export type McpInventorySnapshot = {
+  readonly servers: readonly McpServerInventory[];
+};
+
+export type McpServerInventory = {
+  readonly name: string;
+  readonly tools: readonly string[];
+};
+
 export type CreateMcpManagerOptions = {
   config: McpConfig;
   runtimeSession: RuntimeSessionContext;
@@ -33,6 +43,17 @@ export type CreateMcpManagerOptions = {
   maxObservationChars?: number;
 };
 
+export class McpInitializationError extends Error {
+  constructor(
+    readonly serverName: string,
+    readonly stage: "connect" | "list_tools" | "validate_tools",
+    options?: ErrorOptions,
+  ) {
+    super(`MCP server ${serverName} failed during ${stage}.`, options);
+    this.name = "McpInitializationError";
+  }
+}
+
 export async function createMcpManager(
   options: CreateMcpManagerOptions,
 ): Promise<McpManager> {
@@ -44,6 +65,7 @@ export async function createMcpManager(
     parsePositiveIntegerEnv("TINKER_MCP_MAX_OBSERVATION_CHARS");
   const connections: McpClientConnection[] = [];
   const executors: ToolExecutor[] = [];
+  const servers: McpServerInventory[] = [];
 
   try {
     for (const [serverName, serverConfig] of options.config.servers) {
@@ -58,19 +80,22 @@ export async function createMcpManager(
           sessionId: options.runtimeSession.sessionId,
           data: {
             serverName,
-            error: error instanceof Error ? error.message : String(error),
+            error: "connection failed",
           },
         });
-        continue;
+        throw new McpInitializationError(serverName, "connect", {
+          cause: error,
+        });
       }
 
       try {
         tools = (await connection.client.listTools()).tools;
       } catch (error) {
+        let cause: unknown = error;
         try {
           await connection.close();
         } catch (closeError) {
-          throw new AggregateError(
+          cause = new AggregateError(
             [error, closeError],
             `Failed to inspect and close MCP server ${serverName}.`,
             { cause: closeError },
@@ -81,31 +106,55 @@ export async function createMcpManager(
           sessionId: options.runtimeSession.sessionId,
           data: {
             serverName,
-            error: error instanceof Error ? error.message : String(error),
+            error: "tool discovery failed",
           },
         });
-        continue;
+        throw new McpInitializationError(serverName, "list_tools", {
+          cause,
+        });
       }
 
       connections.push(connection);
 
       const seenToolNames = new Set<string>();
+      try {
+        for (const tool of tools) {
+          if (
+            tool.name.trim() === "" ||
+            typeof tool.inputSchema !== "object" ||
+            tool.inputSchema === null ||
+            Array.isArray(tool.inputSchema) ||
+            tool.inputSchema.type !== "object"
+          ) {
+            throw new Error("MCP tool name or input schema is invalid.");
+          }
+          if (seenToolNames.has(tool.name)) {
+            throw new Error(`Duplicate MCP tool name ${tool.name}.`);
+          }
 
-      for (const tool of tools) {
-        if (seenToolNames.has(tool.name)) {
-          continue;
+          seenToolNames.add(tool.name);
+          executors.push(
+            createMcpToolExecutor({
+              client: connection.client,
+              serverName,
+              tool,
+              timeoutMs,
+              maxObservationChars,
+            }),
+          );
         }
-
-        seenToolNames.add(tool.name);
-        executors.push(
-          createMcpToolExecutor({
-            client: connection.client,
+      } catch (error) {
+        await options.runtimeSession.append({
+          type: "mcp.server.failed",
+          sessionId: options.runtimeSession.sessionId,
+          data: {
             serverName,
-            tool,
-            timeoutMs,
-            maxObservationChars,
-          }),
-        );
+            error: "tool validation failed",
+          },
+        });
+        throw new McpInitializationError(serverName, "validate_tools", {
+          cause: error,
+        });
       }
 
       await options.runtimeSession.append({
@@ -113,7 +162,17 @@ export async function createMcpManager(
         sessionId: options.runtimeSession.sessionId,
         data: { serverName, toolCount: seenToolNames.size },
       });
+      servers.push(
+        Object.freeze({
+          name: serverName,
+          tools: Object.freeze([...seenToolNames].sort(compareText)),
+        }),
+      );
     }
+    executors.sort((left, right) =>
+      compareText(left.definition.name, right.definition.name),
+    );
+    servers.sort((left, right) => compareText(left.name, right.name));
   } catch (error) {
     const errors = [error];
     await closeConnections(connections, errors);
@@ -128,6 +187,7 @@ export async function createMcpManager(
   let disposePromise: Promise<void> | undefined;
   return {
     executors,
+    inventory: Object.freeze({ servers: Object.freeze(servers) }),
     dispose(): Promise<void> {
       disposePromise ??= disposeConnections(connections);
       return disposePromise;
@@ -135,6 +195,10 @@ export async function createMcpManager(
   };
 }
 
+function compareText(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
 async function disposeConnections(connections: McpClientConnection[]): Promise<void> {
   const errors: unknown[] = [];
   await closeConnections(connections, errors);

package/src/model/fake-model-client.ts

@@ -3,6 +3,7 @@ import { cancellationError } from "../agent/turn-cancellation";
 import type { ModelContextBudget } from "./model-context-profile";
 import type {
   ModelClient,
+  ModelMessageProtocol,
   ModelRequestInput,
   ModelRequestOptions,
   ModelRequestOutput,
@@ -13,6 +14,11 @@ import { sha256, stableJsonStringify } from "./model-request-preflight";
 import { estimatePromptSegments } from "./token-estimator";
 
 export class FakeModelClient implements ModelClient {
+  readonly inputModalities = Object.freeze(["text"] as const);
+  readonly messageProtocol: ModelMessageProtocol = Object.freeze({
+    adapter: "fake",
+    serializationVersion: "fake-v1",
+  });
   private steps = 0;
   private readonly preparedInputs = new WeakMap<object, ModelRequestInput>();
 
@@ -34,7 +40,8 @@ export class FakeModelClient implements ModelClient {
     const messageSegments = input.messages.map(toPromptSegment);
     const requestConfigHash = sha256(
       stableJsonStringify({
-        adapter: "fake-v1",
+        adapter: this.messageProtocol.adapter,
+        serializationVersion: this.messageProtocol.serializationVersion,
         mode: this.mode,
         model: this.options.model,
         requestMaxOutputTokens: this.options.contextBudget.requestMaxOutputTokens,
@@ -54,6 +61,7 @@ export class FakeModelClient implements ModelClient {
         toolSegments.map((segment) => segment.normalizedText).join("\n"),
       ),
       requestMaxOutputTokens: this.options.contextBudget.requestMaxOutputTokens,
+      mediaOccurrenceCount: 0,
       assistantReplaySegments: (message: AssistantMessage) => [
         toPromptSegment(message),
       ],

package/src/model/input-token-estimator.ts

@@ -0,0 +1,25 @@
+import type { MaterializedModelRequest } from "./model-client";
+
+export type InputTokenEstimate = {
+  readonly inputTokens: number;
+  readonly source: "provider_estimated";
+  readonly coverage: "messages" | "full_request";
+};
+
+export type InputTokenEstimatorCompatibility = {
+  readonly kind: "moonshot-estimate-token-count-v1";
+  readonly coverageVersion: "full-request-v1";
+  readonly model: string;
+  readonly endpoint: string;
+  readonly timeoutMs: number;
+  readonly maxRetries: 0;
+};
+
+export interface InputTokenEstimator {
+  readonly kind: string;
+  readonly compatibility: InputTokenEstimatorCompatibility;
+  estimate(
+    request: MaterializedModelRequest,
+    options: { signal: AbortSignal },
+  ): Promise<InputTokenEstimate>;
+}

package/src/model/model-client.ts

@@ -1,15 +1,39 @@
 import type { AgentMessage, AssistantMessage, IterationIdentity } from "../agent/types";
 import type { RuntimeSessionContext } from "../agent/runtime-session";
 import type { ToolDefinition } from "../tools/types";
+import type { ImageAssetStore } from "../image/image-asset-store";
+import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
+import type { InputTokenEstimator } from "./input-token-estimator";
 
 export interface ModelClient {
+  readonly messageProtocol: ModelMessageProtocol;
+  readonly inputTokenEstimator?: InputTokenEstimator;
+  readonly inputModalities?: readonly ("text" | "image")[];
   prepare(input: ModelRequestInput): PreparedModelRequest;
+  materialize?(
+    prepared: PreparedModelRequest,
+    options: ModelMaterializeOptions,
+  ): Promise<MaterializedModelRequest>;
   request(
     prepared: PreparedModelRequest,
     options: ModelRequestOptions,
   ): Promise<ModelRequestOutput>;
 }
 
+export class ModelRequestMediaAggregateError extends Error {
+  readonly code = "MODEL_REQUEST_MEDIA_AGGREGATE_LIMIT";
+
+  constructor(message: string) {
+    super(message);
+    this.name = "ModelRequestMediaAggregateError";
+  }
+}
+
+export type ModelMessageProtocol = {
+  adapter: "openai-chat" | "fake";
+  serializationVersion: string;
+};
+
 export type ModelRequestOptions = {
   signal: AbortSignal;
   identity?: {
@@ -34,6 +58,18 @@ export type PreparedPromptSegmentKind =
 export type PreparedPromptSegment = {
   kind: PreparedPromptSegmentKind;
   normalizedText: string;
+  media?: readonly PreparedMediaDescriptor[];
+};
+
+export type PreparedMediaDescriptor = {
+  assetId: ImageAssetId;
+  label: string;
+  range: CodePointRange;
+  mimeType: ImageMimeType;
+  byteLength: number;
+  width: number;
+  height: number;
+  planningTokens: number;
 };
 
 export type PreparedModelRequest = {
@@ -44,9 +80,33 @@ export type PreparedModelRequest = {
   requestConfigHash: string;
   toolSchemaHash: string;
   requestMaxOutputTokens: number;
+  mediaOccurrenceCount: number;
   assistantReplaySegments(message: AssistantMessage): PreparedPromptSegment[];
 };
 
+export type ModelMaterializeOptions = {
+  assetStore: ImageAssetStore;
+  signal: AbortSignal;
+};
+
+export type MaterializedModelRequest = PreparedModelRequest & {
+  readonly bodyBytes: number;
+};
+
+export async function materializeModelRequest(
+  model: ModelClient,
+  prepared: PreparedModelRequest,
+  options: ModelMaterializeOptions,
+): Promise<MaterializedModelRequest> {
+  if (model.materialize !== undefined) {
+    return model.materialize(prepared, options);
+  }
+  if (prepared.mediaOccurrenceCount !== 0) {
+    throw new Error("Model client does not implement image request materialization.");
+  }
+  return prepared as MaterializedModelRequest;
+}
+
 export type ModelRequestOutput = {
   message: AssistantMessage;
   finishReason?: string;

package/src/model/model-request-preflight.ts

@@ -3,6 +3,7 @@ import type { ModelContextBudget } from "./model-context-profile";
 
 export type ContextUsageSource =
   | "estimated_full"
+  | "provider_estimated"
   | "provider_measured"
   | "measured_plus_estimated_delta";