pi-provider-cursor-ask 0.1.0

package/src/stream/images.ts

@@ -0,0 +1,201 @@
+/**
+ * Image decoding and validation for the Cursor wire format.
+ *
+ * Cursor CLI's local-image path scales/compresses images to <= 5 MiB and
+ * accepts only jpeg/png/gif/webp by magic bytes, so anything we forward has to
+ * clear the same bar. Pure helpers with no logging, so any module may import it.
+ */
+import { createHash } from "node:crypto";
+
+import type { ParsedImageContent } from "./types.js";
+
+// Cursor CLI's local-image path scales/compresses images to <= 5 MiB
+// and accepts only jpeg/png/gif/webp by magic bytes.
+export const CURSOR_CLI_MAX_IMAGE_BYTES = 5_242_880;
+const MAX_INLINE_IMAGE_BASE64_CHARS = Math.ceil((CURSOR_CLI_MAX_IMAGE_BYTES * 4) / 3) + 1024;
+
+export const CURSOR_SUPPORTED_IMAGE_MIME_TYPES = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+]);
+
+export interface ImageDecodeOptions {
+  enforceCursorCliLimits?: boolean;
+  /**
+   * Return `undefined` instead of throwing when an image fails Cursor's limits.
+   *
+   * Rejecting loudly is right for the image a caller is attaching right now — they can resize it.
+   * It is wrong for images already sitting in the transcript: those are unfixable, and since the
+   * whole history is re-parsed on every request, one bad image would otherwise fail not just its
+   * own turn but every turn after it, permanently.
+   */
+  dropInvalid?: boolean;
+}
+
+export function normalizeImageMimeType(mimeType: string): string {
+  const normalized = mimeType.trim().toLowerCase();
+  return normalized === "image/jpg" ? "image/jpeg" : normalized;
+}
+
+export function sniffCursorImageMimeType(bytes: Uint8Array): string | undefined {
+  if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff)
+    return "image/jpeg";
+  if (
+    bytes.length >= 4 &&
+    bytes[0] === 0x89 &&
+    bytes[1] === 0x50 &&
+    bytes[2] === 0x4e &&
+    bytes[3] === 0x47
+  )
+    return "image/png";
+  if (bytes.length >= 3 && bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46)
+    return "image/gif";
+  if (
+    bytes.length >= 12 &&
+    bytes[0] === 0x52 &&
+    bytes[1] === 0x49 &&
+    bytes[2] === 0x46 &&
+    bytes[3] === 0x46 &&
+    bytes[8] === 0x57 &&
+    bytes[9] === 0x45 &&
+    bytes[10] === 0x42 &&
+    bytes[11] === 0x50
+  )
+    return "image/webp";
+  return undefined;
+}
+
+export function validateCursorCliImageLimits(bytes: Uint8Array): string {
+  if (bytes.length > CURSOR_CLI_MAX_IMAGE_BYTES) {
+    throw new Error(
+      `Image exceeds Cursor CLI's ${CURSOR_CLI_MAX_IMAGE_BYTES} byte limit after processing.`,
+    );
+  }
+  const sniffedMimeType = sniffCursorImageMimeType(bytes);
+  if (!sniffedMimeType || !CURSOR_SUPPORTED_IMAGE_MIME_TYPES.has(sniffedMimeType)) {
+    throw new Error("Unsupported image type: supported formats are jpeg, png, gif, or webp.");
+  }
+  return sniffedMimeType;
+}
+
+export function decodeBase64Image(
+  data: string,
+  mimeType: string,
+  options: ImageDecodeOptions = {},
+): ParsedImageContent | undefined {
+  const normalizedMimeType = normalizeImageMimeType(mimeType);
+  if (!normalizedMimeType.startsWith("image/")) return undefined;
+  if (options.enforceCursorCliLimits && data.length > MAX_INLINE_IMAGE_BASE64_CHARS) {
+    const error = new Error(
+      `Inline image exceeds Cursor CLI's ${MAX_INLINE_IMAGE_BASE64_CHARS} character encoded limit.`,
+    );
+    if (!options.dropInvalid) throw error;
+    return undefined;
+  }
+  const base64 = data.replace(/\s/g, "");
+  if (!base64) return undefined;
+  const bytes = new Uint8Array(Buffer.from(base64, "base64"));
+  if (bytes.length === 0) return undefined;
+  let finalMimeType = normalizedMimeType;
+  if (options.enforceCursorCliLimits) {
+    try {
+      finalMimeType = validateCursorCliImageLimits(bytes);
+    } catch (error) {
+      if (!options.dropInvalid) throw error;
+      return undefined;
+    }
+  }
+  return { data: bytes, mimeType: finalMimeType };
+}
+
+export function parseImageDataUrl(
+  url: string,
+  options: ImageDecodeOptions = {},
+): ParsedImageContent | undefined {
+  if (options.dropInvalid) {
+    try {
+      return parseImageDataUrlStrict(url, options);
+    } catch {
+      return undefined;
+    }
+  }
+  return parseImageDataUrlStrict(url, options);
+}
+
+function parseImageDataUrlStrict(
+  url: string,
+  options: ImageDecodeOptions,
+): ParsedImageContent | undefined {
+  const trimmed = url.trim();
+  if (/^https?:\/\//i.test(trimmed)) {
+    throw new Error(
+      "Remote image URLs are not supported by pi-cursor-provider. Attach the image or send an inline data:image/...;base64,... URL.",
+    );
+  }
+  if (!trimmed.startsWith("data:")) {
+    throw new Error(
+      "Only inline data:image/...;base64,... image_url values are supported by pi-cursor-provider.",
+    );
+  }
+  const match = trimmed.match(/^data:([^;,]+)(?:;[^,]*)?;base64,(.*)$/is);
+  if (!match) {
+    throw new Error("Unsupported image_url format. Expected data:image/...;base64,...");
+  }
+  const image = decodeBase64Image(match[2]!, match[1]!, options);
+  if (!image) {
+    throw new Error("Unsupported image_url MIME type. Expected data:image/...;base64,...");
+  }
+  return image;
+}
+
+// The whole transcript is re-parsed on every request, so the same image buffers are hashed
+// again and again across turns. Image bytes are treated as immutable once decoded, so a digest
+// keyed by buffer identity stays valid for the buffer's lifetime.
+const contentDigestCache = new WeakMap<Uint8Array, string>();
+
+function contentDigest(data: Uint8Array): string {
+  let digest = contentDigestCache.get(data);
+  if (digest === undefined) {
+    digest = createHash("sha256").update(data).digest("hex");
+    contentDigestCache.set(data, digest);
+  }
+  return digest;
+}
+
+export function imageKey(image: ParsedImageContent): string {
+  return `${image.mimeType}:${contentDigest(image.data)}`;
+}
+
+export function mergeImages(
+  ...groups: Array<ParsedImageContent[] | undefined>
+): ParsedImageContent[] | undefined {
+  const merged: ParsedImageContent[] = [];
+  // Two images can only be duplicates if they agree on MIME type and byte length, and that is
+  // cheap to check. Bucketing on it first means the common all-distinct case never hashes a
+  // payload at all; only images that collide on shape are digested to settle the tie.
+  const byShape = new Map<string, { first: ParsedImageContent; digests?: Set<string> }>();
+  for (const group of groups) {
+    for (const image of group ?? []) {
+      const shape = `${image.mimeType}:${image.data.byteLength}`;
+      const bucket = byShape.get(shape);
+      if (!bucket) {
+        byShape.set(shape, { first: image });
+        merged.push(image);
+        continue;
+      }
+      // Second image of this shape: the bucket's digest set is worth materializing now.
+      const digests = (bucket.digests ??= new Set([contentDigest(bucket.first.data)]));
+      const digest = contentDigest(image.data);
+      if (digests.has(digest)) continue;
+      digests.add(digest);
+      merged.push(image);
+    }
+  }
+  return merged.length > 0 ? merged : undefined;
+}
+
+export function cloneParsedImage(image: ParsedImageContent): ParsedImageContent {
+  return { data: new Uint8Array(image.data), mimeType: image.mimeType };
+}

package/src/stream/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Public stream surface for the Cursor provider.
+ *
+ * All chat traffic goes through the native `streamSimple` path. The legacy
+ * OpenAI-compatible local proxy that used to live alongside it in native-core
+ * was removed in favour of a single code path.
+ */
+export {
+  createCursorNativeStream,
+  getCursorModels,
+  getCursorParameterizedModels,
+  cleanupSessionState,
+  cleanupAllSessionState,
+  type CursorModel,
+  type CursorNativeStreamConfig,
+} from "./native-core.js";
+
+export { getCursorAgentUrl, getCursorClientVersion } from "./config.js";
+export {
+  resolveModelId,
+  resolveRequestedModelId,
+  type CursorNativeModelRouting,
+} from "./model-routing.js";
+export {
+  isContextModeSideChannelText,
+  normalizeMessagesForCursor,
+  frameContextModeSideChannel,
+  systemPromptHasSessionMemory,
+} from "./context-normalize.js";
+export {
+  planRecovery,
+  fingerprintCompletedTurns,
+  wrapRecoveredToolResults,
+  lostToolContinuationErrorBody,
+  formatLostToolContinuationDiagnostic,
+  type RecoveryDecision,
+  type PlanRecoveryInput,
+  type StoredConversation,
+} from "./recovery.js";
+export {
+  appendDriftDiagnostic,
+  enhanceCursorStreamError,
+  isAuthErrorMessage,
+  isProtocolMismatchMessage,
+} from "./protocol.js";
+export {
+  formatDriftSummary,
+  getDriftSignals,
+  hasStrandingDrift,
+  recordDriftSignal,
+  recordUnknownFields,
+  type DriftKind,
+  type DriftSignal,
+} from "./drift.js";
+export { handleInteractionQuery } from "./interaction-query.js";
+export { canRecoverAfterTransportLoss, canBlindIdleRestart } from "./tuning.js";
+export {
+  CHECKPOINT_CONTINUATION_PROMPT,
+  classifyBridgeExit,
+  formatTransportFailure,
+  type TransportFailure,
+} from "./transport-errors.js";
+export {
+  readConversationJournal,
+  writeConversationJournal,
+  serializeConversationJournal,
+  deserializeConversationJournal,
+} from "./run-journal.js";

package/src/stream/interaction-query.ts

@@ -0,0 +1,369 @@
+/**
+ * Handle Cursor InteractionQuery messages so the AgentService stream never
+ * stalls waiting for a permission / interaction reply that Pi never sends.
+ *
+ * Unanswered interaction queries are a primary cause of "model stops after a
+ * few minutes" — Cursor parks the run until InteractionResponse arrives.
+ */
+import { create, toBinary } from "@bufbuild/protobuf";
+import {
+  AgentClientMessageSchema,
+  AskQuestionErrorSchema,
+  AskQuestionInteractionResponseSchema,
+  AskQuestionResultSchema,
+  CreatePlanErrorSchema,
+  CreatePlanRequestResponseSchema,
+  CreatePlanResultSchema,
+  ExaFetchRequestResponseSchema,
+  ExaFetchRequestResponse_ApprovedSchema,
+  ExaFetchRequestResponse_RejectedSchema,
+  ExaSearchRequestResponseSchema,
+  ExaSearchRequestResponse_ApprovedSchema,
+  ExaSearchRequestResponse_RejectedSchema,
+  InteractionResponseSchema,
+  SetupVmEnvironmentResultSchema,
+  SetupVmEnvironmentSuccessSchema,
+  SwitchModeRequestResponseSchema,
+  SwitchModeRequestResponse_RejectedSchema,
+  WebSearchRequestResponseSchema,
+  WebSearchRequestResponse_ApprovedSchema,
+  WebSearchRequestResponse_RejectedSchema,
+  type InteractionQuery,
+  type InteractionResponse,
+} from "../proto/agent_pb.js";
+import { frameConnectMessage } from "../client/bridge.js";
+
+const CURSOR_WEB_FETCH_INTERACTION_FIELD = 9;
+
+const PI_REJECT_REASON =
+  "Not available through the Pi Cursor provider. Use Pi tools (web_search, fetch, bash, etc.) instead.";
+
+function encodeVarint(value: number): number[] {
+  const bytes: number[] = [];
+  let remaining = value >>> 0;
+  while (remaining >= 0x80) {
+    bytes.push((remaining & 0x7f) | 0x80);
+    remaining >>>= 7;
+  }
+  bytes.push(remaining);
+  return bytes;
+}
+
+function encodeLengthDelimitedField(fieldNo: number, data: Uint8Array): number[] {
+  return [(fieldNo << 3) | 2, ...encodeVarint(data.length), ...data];
+}
+
+/**
+ * Field #9 is unnamed in the generated proto (web-fetch shaped). Approving it is
+ * indistinguishable from granting a future destructive capability if Cursor reuses
+ * the number, so we always reject. We still have to *answer* — `handled: false`
+ * throws and kills the in-flight turn (#10).
+ *
+ * Wire shape mirrors ExaFetch/WebSearch: response oneof field 2 = rejected,
+ * rejected.reason = 1.
+ */
+function buildCursorWebFetchInteractionRejectionBytes(id: number): Uint8Array {
+  const reason = new TextEncoder().encode(PI_REJECT_REASON);
+  const rejected = new Uint8Array(encodeLengthDelimitedField(1, reason));
+  const result = new Uint8Array(encodeLengthDelimitedField(2, rejected));
+  const interactionResponse = new Uint8Array([
+    0x08,
+    ...encodeVarint(id),
+    ...encodeLengthDelimitedField(CURSOR_WEB_FETCH_INTERACTION_FIELD, result),
+  ]);
+  return new Uint8Array(encodeLengthDelimitedField(6, interactionResponse));
+}
+
+function hasUnknownInteractionField(query: InteractionQuery, fieldNo: number): boolean {
+  return ((query as unknown as { $unknown?: Array<{ no: number }> }).$unknown ?? []).some(
+    (field) => field.no === fieldNo,
+  );
+}
+
+function sendInteractionResponse(
+  response: InteractionResponse,
+  sendFrame: (data: Uint8Array) => void,
+): void {
+  const clientMsg = create(AgentClientMessageSchema, {
+    message: { case: "interactionResponse", value: response },
+  });
+  sendFrame(frameConnectMessage(toBinary(AgentClientMessageSchema, clientMsg)));
+}
+
+function approveWebSearch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "webSearchRequestResponse",
+        value: create(WebSearchRequestResponseSchema, {
+          result: {
+            case: "approved",
+            value: create(WebSearchRequestResponse_ApprovedSchema, {}),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function rejectWebSearch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "webSearchRequestResponse",
+        value: create(WebSearchRequestResponseSchema, {
+          result: {
+            case: "rejected",
+            value: create(WebSearchRequestResponse_RejectedSchema, { reason: PI_REJECT_REASON }),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function approveExaSearch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "exaSearchRequestResponse",
+        value: create(ExaSearchRequestResponseSchema, {
+          result: {
+            case: "approved",
+            value: create(ExaSearchRequestResponse_ApprovedSchema, {}),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function rejectExaSearch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "exaSearchRequestResponse",
+        value: create(ExaSearchRequestResponseSchema, {
+          result: {
+            case: "rejected",
+            value: create(ExaSearchRequestResponse_RejectedSchema, { reason: PI_REJECT_REASON }),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function approveExaFetch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "exaFetchRequestResponse",
+        value: create(ExaFetchRequestResponseSchema, {
+          result: {
+            case: "approved",
+            value: create(ExaFetchRequestResponse_ApprovedSchema, {}),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function rejectExaFetch(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "exaFetchRequestResponse",
+        value: create(ExaFetchRequestResponseSchema, {
+          result: {
+            case: "rejected",
+            value: create(ExaFetchRequestResponse_RejectedSchema, { reason: PI_REJECT_REASON }),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function rejectSwitchMode(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "switchModeRequestResponse",
+        value: create(SwitchModeRequestResponseSchema, {
+          result: {
+            case: "rejected",
+            value: create(SwitchModeRequestResponse_RejectedSchema, { reason: PI_REJECT_REASON }),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function skipAskQuestion(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "askQuestionInteractionResponse",
+        value: create(AskQuestionInteractionResponseSchema, {
+          result: create(AskQuestionResultSchema, {
+            result: {
+              case: "error",
+              value: create(AskQuestionErrorSchema, {
+                errorMessage:
+                  "Interactive questions are not available in Pi. Continue with a reasonable default or ask the user in chat.",
+              }),
+            },
+          }),
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function skipCreatePlan(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "createPlanRequestResponse",
+        value: create(CreatePlanRequestResponseSchema, {
+          result: create(CreatePlanResultSchema, {
+            planUri: "",
+            result: {
+              case: "error",
+              value: create(CreatePlanErrorSchema, {
+                error: "Create-plan UI is not available in Pi. Write the plan with Pi file tools.",
+              }),
+            },
+          }),
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+function skipSetupVm(id: number, sendFrame: (data: Uint8Array) => void): void {
+  sendInteractionResponse(
+    create(InteractionResponseSchema, {
+      id,
+      result: {
+        case: "setupVmEnvironmentResult",
+        value: create(SetupVmEnvironmentResultSchema, {
+          result: {
+            case: "success",
+            value: create(SetupVmEnvironmentSuccessSchema, {}),
+          },
+        }),
+      },
+    }),
+    sendFrame,
+  );
+}
+
+export type InteractionQueryHandleResult = {
+  handled: boolean;
+  action: string;
+  queryCase: string | undefined;
+};
+
+/**
+ * Always attempt to answer InteractionQuery so the upstream run does not park.
+ * Web/search is rejected by default so Cursor-side fetches do not run under the
+ * user's subscription; pass `{ approveWeb: true }` only in tests or explicit opt-in.
+ */
+export function handleInteractionQuery(
+  query: InteractionQuery,
+  sendFrame: (data: Uint8Array) => void,
+  options?: { approveWeb?: boolean },
+): InteractionQueryHandleResult {
+  const approveWeb = options?.approveWeb === true;
+  const queryCase = query.query.case;
+
+  // Field #9 is unnamed in the generated proto. Approving it is indistinguishable
+  // from granting a future destructive capability if Cursor reuses the number.
+  // Reject with a real InteractionResponse so Cursor unblocks instead of parking,
+  // and so processServerMessage does not throw and kill the turn (#10).
+  if (hasUnknownInteractionField(query, CURSOR_WEB_FETCH_INTERACTION_FIELD)) {
+    sendFrame(frameConnectMessage(buildCursorWebFetchInteractionRejectionBytes(query.id)));
+    return {
+      handled: true,
+      action: "unknown_field_9_rejected",
+      queryCase: queryCase ?? "unknown_field_9",
+    };
+  }
+
+  switch (queryCase) {
+    case "webSearchRequestQuery":
+      if (approveWeb) approveWebSearch(query.id, sendFrame);
+      else rejectWebSearch(query.id, sendFrame);
+      return {
+        handled: true,
+        action: approveWeb ? "web_search_approved" : "web_search_rejected",
+        queryCase,
+      };
+    case "exaSearchRequestQuery":
+      if (approveWeb) approveExaSearch(query.id, sendFrame);
+      else rejectExaSearch(query.id, sendFrame);
+      return {
+        handled: true,
+        action: approveWeb ? "exa_search_approved" : "exa_search_rejected",
+        queryCase,
+      };
+    case "exaFetchRequestQuery":
+      if (approveWeb) approveExaFetch(query.id, sendFrame);
+      else rejectExaFetch(query.id, sendFrame);
+      return {
+        handled: true,
+        action: approveWeb ? "exa_fetch_approved" : "exa_fetch_rejected",
+        queryCase,
+      };
+    case "switchModeRequestQuery":
+      rejectSwitchMode(query.id, sendFrame);
+      return { handled: true, action: "switch_mode_rejected", queryCase };
+    case "askQuestionInteractionQuery":
+      skipAskQuestion(query.id, sendFrame);
+      return { handled: true, action: "ask_question_skipped", queryCase };
+    case "createPlanRequestQuery":
+      skipCreatePlan(query.id, sendFrame);
+      return { handled: true, action: "create_plan_skipped", queryCase };
+    case "setupVmEnvironmentArgs":
+      skipSetupVm(query.id, sendFrame);
+      return { handled: true, action: "setup_vm_acked", queryCase };
+    default: {
+      // Protocol drift must fail closed. An empty result for an unknown field is
+      // indistinguishable from approval and could grant a future destructive capability.
+      const unknown = (query as unknown as { $unknown?: Array<{ no: number }> }).$unknown ?? [];
+      if (unknown.length > 0) {
+        const fieldNo = unknown[0]!.no;
+        return {
+          handled: false,
+          action: `unknown_field_${fieldNo}_rejected`,
+          queryCase: queryCase ?? "unknown",
+        };
+      }
+      // No case and no unknown fields — still send a switch-mode-style reject is impossible.
+      // Best effort: skip ask-question style is wrong. Log as unhandled.
+      return { handled: false, action: "unhandled", queryCase: queryCase ?? "undefined" };
+    }
+  }
+}