pi-codex-search 0.1.1 → 0.1.3

package/src/modes/responses.ts

@@ -0,0 +1,310 @@
+import {
+  CodexError,
+  classifyEventErrorMessage,
+  classifyHttpStatus,
+  formatHttpErrorBody,
+} from "../errors.ts";
+import type { CodexTransport } from "../transport.ts";
+import type {
+  CodexWebSearchResult,
+  CodexCitation,
+  CodexSearchCall,
+  SearchContextSize,
+} from "./types.ts";
+
+export interface ResponsesSearchOptions {
+  query: string;
+  model: string;
+  transport: CodexTransport;
+  externalWebAccess: boolean;
+  indexGatedWebAccess?: true;
+  searchContextSize?: SearchContextSize;
+  sessionId?: string;
+  threadId?: string;
+  signal?: AbortSignal;
+  onTextDelta?: (delta: string) => void;
+}
+
+interface SseEvent {
+  type: string;
+  data?: unknown;
+  raw?: string;
+}
+
+interface ResponseOutputText {
+  type?: string;
+  text?: string;
+  annotations?: Array<{
+    type?: string;
+    title?: string;
+    url?: string;
+    start_index?: number;
+    end_index?: number;
+  }>;
+}
+
+interface ResponseOutputItem {
+  id?: string;
+  type?: string;
+  status?: string;
+  role?: string;
+  action?: {
+    type?: string;
+    query?: string;
+    queries?: string[];
+    url?: string;
+  };
+  content?: ResponseOutputText[];
+}
+
+interface ResponseUsage {
+  input_tokens?: number;
+  output_tokens?: number;
+  total_tokens?: number;
+}
+
+interface ResponseEnvelope {
+  id?: string;
+  usage?: ResponseUsage;
+}
+
+interface ResponseEventData {
+  response?: ResponseEnvelope;
+  item?: ResponseOutputItem;
+  delta?: string;
+  error?: {
+    message?: string;
+    code?: string;
+  };
+}
+
+export async function runResponsesSearch(
+  options: ResponsesSearchOptions,
+): Promise<CodexWebSearchResult> {
+  const {
+    transport,
+    query,
+    model,
+    externalWebAccess,
+    searchContextSize,
+    sessionId,
+    threadId,
+    signal,
+    onTextDelta,
+  } = options;
+  const headers = transport.buildHeaders("text/event-stream");
+  if (sessionId) headers.set("session-id", sessionId);
+  if (threadId) {
+    headers.set("thread-id", threadId);
+    headers.set("x-client-request-id", threadId);
+  }
+
+  const webSearchTool: Record<string, unknown> = {
+    type: "web_search",
+    external_web_access: externalWebAccess,
+    search_context_size: searchContextSize ?? "medium",
+  };
+
+  const response = await transport.fetch(transport.resolveEndpoint("responses"), {
+    method: "POST",
+    headers,
+    body: JSON.stringify({
+      model,
+      instructions:
+        "You are a concise web search assistant. Use web search, answer the query, and preserve source citations from annotations.",
+      input: [
+        {
+          type: "message",
+          role: "user",
+          content: [{ type: "input_text", text: query }],
+        },
+      ],
+      tools: [webSearchTool],
+      tool_choice: "required",
+      parallel_tool_calls: true,
+      store: false,
+      stream: true,
+      include: [],
+    }),
+    signal,
+  });
+
+  if (!response.ok) {
+    const status = response.status;
+    const text = formatHttpErrorBody(await response.text(), "responses");
+    throw new CodexError(
+      classifyHttpStatus(status),
+      `Codex responses request failed: HTTP ${status}: ${text}`,
+      status,
+    );
+  }
+  if (!response.body) {
+    throw new Error("Codex responses response did not include a body");
+  }
+
+  let responseId: string | undefined;
+  let usage: ResponseUsage | undefined;
+  let streamedText = "";
+  const messageTextParts: string[] = [];
+  const searchCalls = new Map<string, CodexSearchCall>();
+  const citations = new Map<string, CodexCitation>();
+
+  for await (const event of parseSse(response.body)) {
+    const data = event.data as ResponseEventData | undefined;
+    if (!data) continue;
+
+    if (event.type === "response.created") {
+      responseId = data.response?.id;
+      continue;
+    }
+
+    if (event.type === "response.output_text.delta") {
+      const delta = data.delta ?? "";
+      streamedText += delta;
+      onTextDelta?.(delta);
+      continue;
+    }
+
+    if (event.type === "response.output_item.added" && data.item?.type === "web_search_call") {
+      const item = data.item;
+      if (item.id) {
+        searchCalls.set(item.id, {
+          id: item.id,
+          status: item.status,
+        });
+      }
+      continue;
+    }
+
+    if (event.type === "response.output_item.done") {
+      collectOutputItem(data.item, searchCalls, messageTextParts, citations);
+      continue;
+    }
+
+    if (event.type === "response.completed") {
+      usage = data.response?.usage;
+      continue;
+    }
+
+    if (event.type === "response.failed") {
+      const message = data.error?.message ?? data.error?.code ?? "Codex web search failed";
+      throw new CodexError(classifyEventErrorMessage(message), message);
+    }
+  }
+
+  return {
+    responseId,
+    model,
+    text: messageTextParts.join("") || streamedText,
+    searchCalls: [...searchCalls.values()],
+    citations: [...citations.values()],
+    usage: usage
+      ? {
+          inputTokens: usage.input_tokens,
+          outputTokens: usage.output_tokens,
+          totalTokens: usage.total_tokens,
+        }
+      : undefined,
+  };
+}
+
+async function* parseSse(body: ReadableStream<Uint8Array>): AsyncGenerator<SseEvent> {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  let doneReading = false;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        doneReading = true;
+        break;
+      }
+      buffer += decoder.decode(value, { stream: true });
+
+      let separator = findSseSeparator(buffer);
+      while (separator) {
+        const frame = buffer.slice(0, separator.index);
+        buffer = buffer.slice(separator.index + separator.length);
+        const event = parseSseFrame(frame);
+        if (event) yield event;
+        separator = findSseSeparator(buffer);
+      }
+    }
+  } finally {
+    if (!doneReading) await reader.cancel().catch(() => undefined);
+    reader.releaseLock();
+  }
+
+  buffer += decoder.decode();
+  const event = parseSseFrame(buffer);
+  if (event) yield event;
+}
+
+function findSseSeparator(buffer: string): { index: number; length: number } | undefined {
+  const match = /\r?\n\r?\n/.exec(buffer);
+  return match?.index === undefined ? undefined : { index: match.index, length: match[0].length };
+}
+
+function parseSseFrame(frame: string): SseEvent | undefined {
+  const lines = frame.split(/\r?\n/);
+  let type = "";
+  const dataLines: string[] = [];
+
+  for (const line of lines) {
+    if (line.startsWith("event:")) {
+      type = line.slice("event:".length).trim();
+    } else if (line.startsWith("data:")) {
+      dataLines.push(line.slice("data:".length).trimStart());
+    }
+  }
+
+  if (dataLines.length === 0) return undefined;
+  const raw = dataLines.join("\n");
+  if (raw === "[DONE]") return undefined;
+
+  try {
+    return { type, data: JSON.parse(raw) };
+  } catch {
+    return { type, raw };
+  }
+}
+
+function collectOutputItem(
+  item: ResponseOutputItem | undefined,
+  searchCalls: Map<string, CodexSearchCall>,
+  messageTextParts: string[],
+  citations: Map<string, CodexCitation>,
+): void {
+  if (!item) return;
+
+  if (item.type === "web_search_call") {
+    const key = item.id ?? `search-${searchCalls.size + 1}`;
+    const query = item.action?.query ?? item.action?.queries?.join(", ");
+    searchCalls.set(key, {
+      id: item.id,
+      status: item.status,
+      query,
+      url: item.action?.url,
+      actionType: item.action?.type,
+    });
+    return;
+  }
+
+  if (item.type !== "message" || item.role !== "assistant") return;
+
+  for (const part of item.content ?? []) {
+    if (part.type !== "output_text") continue;
+    messageTextParts.push(part.text ?? "");
+    for (const annotation of part.annotations ?? []) {
+      if (annotation.type !== "url_citation" || !annotation.url) continue;
+      citations.set(annotation.url, {
+        title: annotation.title,
+        url: annotation.url,
+        startIndex: annotation.start_index,
+        endIndex: annotation.end_index,
+      });
+    }
+  }
+}

package/src/modes/standalone.ts

@@ -0,0 +1,378 @@
+import {
+  CodexError,
+  classifyHttpStatus,
+  formatHttpErrorBody,
+  isCloudflareChallenge,
+} from "../errors.ts";
+import type { CodexTransport } from "../transport.ts";
+import type {
+  CodexWebSearchResult,
+  CodexCitation,
+  CodexSearchCall,
+  SearchContextSize,
+  Freshness,
+  StandaloneExternalWebAccess,
+  ResponseLength,
+} from "./types.ts";
+
+export interface SearchQuery {
+  q: string;
+  recency?: number;
+  domains?: string[];
+}
+
+export interface OpenCommand {
+  refId: string;
+  lineno?: number;
+}
+
+export interface FindCommand {
+  refId: string;
+  pattern: string;
+}
+
+export interface ClickCommand {
+  refId: string;
+  id: number;
+}
+
+export interface ScreenshotCommand {
+  refId: string;
+  pageno: number;
+}
+
+export interface FinanceCommand {
+  ticker: string;
+  type: "equity" | "fund" | "crypto" | "index";
+  market?: string;
+}
+
+export interface WeatherCommand {
+  location: string;
+  start?: string;
+  duration?: number;
+}
+
+export type SportsLeague =
+  | "nba"
+  | "wnba"
+  | "nfl"
+  | "nhl"
+  | "mlb"
+  | "epl"
+  | "ncaamb"
+  | "ncaawb"
+  | "ipl";
+
+export interface SportsCommand {
+  fn: "schedule" | "standings";
+  league: SportsLeague;
+  team?: string;
+  opponent?: string;
+  date_from?: string;
+  date_to?: string;
+  num_games?: number;
+  locale?: string;
+}
+
+export interface TimeCommand {
+  utc_offset: string;
+}
+
+export interface StandaloneCommandsOptions {
+  model: string;
+  transport: CodexTransport;
+  sessionId: string;
+  searchQuery?: SearchQuery[];
+  imageQuery?: SearchQuery[];
+  open?: OpenCommand[];
+  find?: FindCommand[];
+  click?: ClickCommand[];
+  screenshot?: ScreenshotCommand[];
+  finance?: FinanceCommand[];
+  weather?: WeatherCommand[];
+  sports?: SportsCommand[];
+  time?: TimeCommand[];
+  freshness: Freshness;
+  searchContextSize?: SearchContextSize;
+  responseLength?: ResponseLength;
+  maxOutputTokens?: number;
+  signal?: AbortSignal;
+}
+
+interface StandaloneSearchResponse {
+  encrypted_output?: string;
+  output?: string;
+}
+
+export function externalWebAccessForFreshness(freshness: Freshness): StandaloneExternalWebAccess {
+  if (freshness === "cached") return false;
+  if (freshness === "indexed") return "indexed";
+  return true;
+}
+
+export function hasAnyCommand(options: StandaloneCommandsOptions): boolean {
+  return countCommands(options) > 0;
+}
+
+export function isUnsupportedStandaloneCombination(
+  searchContextSize: SearchContextSize | undefined,
+  _freshness: Freshness,
+): boolean {
+  return (searchContextSize ?? "medium") === "low";
+}
+
+export function assertSupportedStandaloneCombination(
+  searchContextSize: SearchContextSize | undefined,
+  freshness: Freshness,
+): void {
+  if (isUnsupportedStandaloneCombination(searchContextSize, freshness)) {
+    throw new CodexError(
+      "schema",
+      'standalone/low is disabled because Codex returns Cloudflare challenges for low-context standalone requests. Use search_context_size "medium" or "high".',
+    );
+  }
+}
+
+function countCommands(options: StandaloneCommandsOptions): number {
+  return (
+    (options.searchQuery?.length ?? 0) +
+    (options.imageQuery?.length ?? 0) +
+    (options.open?.length ?? 0) +
+    (options.find?.length ?? 0) +
+    (options.click?.length ?? 0) +
+    (options.screenshot?.length ?? 0) +
+    (options.finance?.length ?? 0) +
+    (options.weather?.length ?? 0) +
+    (options.sports?.length ?? 0) +
+    (options.time?.length ?? 0)
+  );
+}
+
+export async function runStandaloneCommands(
+  options: StandaloneCommandsOptions,
+): Promise<CodexWebSearchResult> {
+  if (!hasAnyCommand(options)) {
+    throw new CodexError("schema", "Codex standalone commands require at least one command");
+  }
+  if (countCommands(options) > 1) {
+    throw new CodexError("schema", "Codex standalone actions must be sent one per request");
+  }
+  assertSupportedStandaloneCombination(options.searchContextSize, options.freshness);
+
+  const {
+    transport,
+    model,
+    sessionId,
+    freshness,
+    searchContextSize,
+    responseLength,
+    maxOutputTokens,
+    signal,
+  } = options;
+  const headers = transport.buildHeaders("application/json");
+  headers.set("OpenAI-Beta", "responses=experimental");
+  headers.set("content-type", "application/json");
+
+  const commands: Record<string, unknown> = {};
+  if (options.searchQuery?.length) commands.search_query = options.searchQuery;
+  if (options.imageQuery?.length) commands.image_query = options.imageQuery;
+  if (options.open?.length)
+    commands.open = options.open.map((c) => ({ ref_id: c.refId, lineno: c.lineno }));
+  if (options.find?.length)
+    commands.find = options.find.map((c) => ({ ref_id: c.refId, pattern: c.pattern }));
+  if (options.click?.length)
+    commands.click = options.click.map((c) => ({ ref_id: c.refId, id: c.id }));
+  if (options.screenshot?.length) {
+    commands.screenshot = options.screenshot.map((c) => ({ ref_id: c.refId, pageno: c.pageno }));
+  }
+  if (options.finance?.length) {
+    commands.finance = options.finance.map((c) => ({
+      ticker: c.ticker,
+      type: c.type,
+      market: c.market,
+    }));
+  }
+  if (options.weather?.length) {
+    commands.weather = options.weather.map((c) => ({
+      location: c.location,
+      start: c.start,
+      duration: c.duration,
+    }));
+  }
+  if (options.sports?.length) {
+    commands.sports = options.sports.map((c) => ({
+      fn: c.fn,
+      league: c.league,
+      team: c.team,
+      opponent: c.opponent,
+      date_from: c.date_from,
+      date_to: c.date_to,
+      num_games: c.num_games,
+      locale: c.locale,
+    }));
+  }
+  if (options.time?.length) commands.time = options.time.map((c) => ({ utc_offset: c.utc_offset }));
+  if (responseLength) commands.response_length = responseLength;
+
+  const body: Record<string, unknown> = {
+    id: sessionId,
+    model,
+    input: buildInput(options),
+    commands,
+    settings: {
+      search_context_size: searchContextSize ?? "medium",
+      allowed_callers: ["direct"],
+      external_web_access: externalWebAccessForFreshness(freshness),
+    },
+  };
+  body.max_output_tokens = maxOutputTokens ?? 8000;
+
+  const bodyText = JSON.stringify(body);
+  let response = await transport.fetch(transport.resolveSearchEndpoint(), {
+    method: "POST",
+    headers,
+    body: bodyText,
+    signal,
+  });
+
+  if (!response.ok) {
+    let status = response.status;
+    let rawText = await response.text();
+    if (status === 403 && isCloudflareChallenge(rawText) && !signal?.aborted) {
+      await delay(750, signal);
+      if (signal?.aborted) {
+        throw new CodexError("timeout", "Codex standalone request was aborted before retry.");
+      }
+      response = await transport.fetch(transport.resolveSearchEndpoint(), {
+        method: "POST",
+        headers,
+        body: bodyText,
+        signal,
+      });
+      if (!response.ok) {
+        status = response.status;
+        rawText = await response.text();
+      }
+    }
+    if (!response.ok) {
+      const text = formatHttpErrorBody(rawText, "standalone");
+      throw new CodexError(
+        classifyHttpStatus(status),
+        `Codex standalone search request failed: HTTP ${status}: ${text}`,
+        status,
+      );
+    }
+  }
+
+  const data = (await response.json()) as StandaloneSearchResponse;
+  const text = typeof data.output === "string" ? data.output : "";
+  const refIds = extractRefIds(text);
+  const searchCalls = inferSearchCalls(options);
+
+  const result: CodexWebSearchResult = {
+    model,
+    text,
+    searchCalls,
+    citations: extractMarkdownCitations(text),
+    refIds,
+  };
+  if (data.encrypted_output !== undefined) result.encryptedOutput = data.encrypted_output;
+  return result;
+}
+
+async function delay(ms: number, signal: AbortSignal | undefined): Promise<void> {
+  if (signal?.aborted) return;
+  await new Promise<void>((resolve) => {
+    const timeout = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timeout);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+function buildInput(options: StandaloneCommandsOptions): unknown[] {
+  const texts: string[] = [];
+  options.searchQuery?.forEach((q) => texts.push(q.q));
+  options.imageQuery?.forEach((q) => texts.push(q.q));
+  options.open?.forEach((c) => texts.push(c.refId));
+  options.find?.forEach((c) => texts.push(`find "${c.pattern}" in ${c.refId}`));
+  options.click?.forEach((c) => texts.push(`click ${c.id} in ${c.refId}`));
+  options.screenshot?.forEach((c) => texts.push(`screenshot ${c.pageno} of ${c.refId}`));
+  options.finance?.forEach((c) => texts.push(`finance ${c.ticker} ${c.type} ${c.market ?? ""}`));
+  options.weather?.forEach((c) => texts.push(`weather ${c.location}`));
+  options.sports?.forEach((c) => texts.push(`sports ${c.fn} ${c.league}`));
+  options.time?.forEach((c) => texts.push(`time ${c.utc_offset}`));
+
+  const prompt = texts.filter(Boolean).join("\n");
+  return [
+    {
+      type: "message",
+      role: "user",
+      content: [{ type: "input_text", text: prompt }],
+    },
+  ];
+}
+
+function inferSearchCalls(options: StandaloneCommandsOptions): CodexSearchCall[] {
+  const calls: CodexSearchCall[] = [];
+  options.searchQuery?.forEach((q) =>
+    calls.push({ status: "completed", query: q.q, actionType: "search_query" }),
+  );
+  options.imageQuery?.forEach((q) =>
+    calls.push({ status: "completed", query: q.q, actionType: "image_query" }),
+  );
+  options.open?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "open_page" }),
+  );
+  options.find?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "find_in_page" }),
+  );
+  options.click?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "click" }),
+  );
+  options.screenshot?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "screenshot" }),
+  );
+  options.finance?.forEach((c) =>
+    calls.push({ status: "completed", query: `${c.ticker}`, actionType: "finance" }),
+  );
+  options.weather?.forEach((c) =>
+    calls.push({ status: "completed", query: c.location, actionType: "weather" }),
+  );
+  options.sports?.forEach((c) =>
+    calls.push({ status: "completed", query: `${c.fn} ${c.league}`, actionType: "sports" }),
+  );
+  options.time?.forEach((c) =>
+    calls.push({ status: "completed", query: c.utc_offset, actionType: "time" }),
+  );
+  return calls;
+}
+
+const REF_ID_PATTERN = /\b(turn\d+(?:search|fetch|view)\d+)\b/g;
+
+function extractRefIds(text: string): Record<string, string> {
+  const refs: Record<string, string> = {};
+  for (const match of text.matchAll(REF_ID_PATTERN)) {
+    const refId = match[1];
+    if (refId) refs[refId] = refId;
+  }
+  return refs;
+}
+
+function extractMarkdownCitations(text: string): CodexCitation[] {
+  const citations = new Map<string, CodexCitation>();
+  const markdownLinkPattern = /\[([^\]\n]{1,200})\]\((https?:\/\/[^)\s]+)\)/g;
+  for (const match of text.matchAll(markdownLinkPattern)) {
+    const title = match[1]?.trim();
+    const url = match[2]?.trim();
+    if (!url || citations.has(url)) continue;
+    citations.set(url, { title: title || url, url, startIndex: match.index });
+  }
+  return [...citations.values()];
+}