pi-codex-search 0.1.2 → 0.1.4

package/src/modes/standalone.ts

@@ -0,0 +1,378 @@
+import {
+  CodexError,
+  classifyHttpStatus,
+  formatHttpErrorBody,
+  isCloudflareChallenge,
+} from "../errors.ts";
+import type { CodexTransport } from "../transport.ts";
+import type {
+  CodexWebSearchResult,
+  CodexCitation,
+  CodexSearchCall,
+  SearchContextSize,
+  Freshness,
+  StandaloneExternalWebAccess,
+  ResponseLength,
+} from "./types.ts";
+
+export interface SearchQuery {
+  q: string;
+  recency?: number;
+  domains?: string[];
+}
+
+export interface OpenCommand {
+  refId: string;
+  lineno?: number;
+}
+
+export interface FindCommand {
+  refId: string;
+  pattern: string;
+}
+
+export interface ClickCommand {
+  refId: string;
+  id: number;
+}
+
+export interface ScreenshotCommand {
+  refId: string;
+  pageno: number;
+}
+
+export interface FinanceCommand {
+  ticker: string;
+  type: "equity" | "fund" | "crypto" | "index";
+  market?: string;
+}
+
+export interface WeatherCommand {
+  location: string;
+  start?: string;
+  duration?: number;
+}
+
+export type SportsLeague =
+  | "nba"
+  | "wnba"
+  | "nfl"
+  | "nhl"
+  | "mlb"
+  | "epl"
+  | "ncaamb"
+  | "ncaawb"
+  | "ipl";
+
+export interface SportsCommand {
+  fn: "schedule" | "standings";
+  league: SportsLeague;
+  team?: string;
+  opponent?: string;
+  date_from?: string;
+  date_to?: string;
+  num_games?: number;
+  locale?: string;
+}
+
+export interface TimeCommand {
+  utc_offset: string;
+}
+
+export interface StandaloneCommandsOptions {
+  model: string;
+  transport: CodexTransport;
+  sessionId: string;
+  searchQuery?: SearchQuery[];
+  imageQuery?: SearchQuery[];
+  open?: OpenCommand[];
+  find?: FindCommand[];
+  click?: ClickCommand[];
+  screenshot?: ScreenshotCommand[];
+  finance?: FinanceCommand[];
+  weather?: WeatherCommand[];
+  sports?: SportsCommand[];
+  time?: TimeCommand[];
+  freshness: Freshness;
+  searchContextSize?: SearchContextSize;
+  responseLength?: ResponseLength;
+  maxOutputTokens?: number;
+  signal?: AbortSignal;
+}
+
+interface StandaloneSearchResponse {
+  encrypted_output?: string;
+  output?: string;
+}
+
+export function externalWebAccessForFreshness(freshness: Freshness): StandaloneExternalWebAccess {
+  if (freshness === "cached") return false;
+  if (freshness === "indexed") return "indexed";
+  return true;
+}
+
+export function hasAnyCommand(options: StandaloneCommandsOptions): boolean {
+  return countCommands(options) > 0;
+}
+
+export function isUnsupportedStandaloneCombination(
+  searchContextSize: SearchContextSize | undefined,
+  _freshness: Freshness,
+): boolean {
+  return (searchContextSize ?? "medium") === "low";
+}
+
+export function assertSupportedStandaloneCombination(
+  searchContextSize: SearchContextSize | undefined,
+  freshness: Freshness,
+): void {
+  if (isUnsupportedStandaloneCombination(searchContextSize, freshness)) {
+    throw new CodexError(
+      "schema",
+      'standalone/low is disabled because Codex returns Cloudflare challenges for low-context standalone requests. Use search_context_size "medium" or "high".',
+    );
+  }
+}
+
+function countCommands(options: StandaloneCommandsOptions): number {
+  return (
+    (options.searchQuery?.length ?? 0) +
+    (options.imageQuery?.length ?? 0) +
+    (options.open?.length ?? 0) +
+    (options.find?.length ?? 0) +
+    (options.click?.length ?? 0) +
+    (options.screenshot?.length ?? 0) +
+    (options.finance?.length ?? 0) +
+    (options.weather?.length ?? 0) +
+    (options.sports?.length ?? 0) +
+    (options.time?.length ?? 0)
+  );
+}
+
+export async function runStandaloneCommands(
+  options: StandaloneCommandsOptions,
+): Promise<CodexWebSearchResult> {
+  if (!hasAnyCommand(options)) {
+    throw new CodexError("schema", "Codex standalone commands require at least one command");
+  }
+  if (countCommands(options) > 1) {
+    throw new CodexError("schema", "Codex standalone actions must be sent one per request");
+  }
+  assertSupportedStandaloneCombination(options.searchContextSize, options.freshness);
+
+  const {
+    transport,
+    model,
+    sessionId,
+    freshness,
+    searchContextSize,
+    responseLength,
+    maxOutputTokens,
+    signal,
+  } = options;
+  const headers = transport.buildHeaders("application/json");
+  headers.set("OpenAI-Beta", "responses=experimental");
+  headers.set("content-type", "application/json");
+
+  const commands: Record<string, unknown> = {};
+  if (options.searchQuery?.length) commands.search_query = options.searchQuery;
+  if (options.imageQuery?.length) commands.image_query = options.imageQuery;
+  if (options.open?.length)
+    commands.open = options.open.map((c) => ({ ref_id: c.refId, lineno: c.lineno }));
+  if (options.find?.length)
+    commands.find = options.find.map((c) => ({ ref_id: c.refId, pattern: c.pattern }));
+  if (options.click?.length)
+    commands.click = options.click.map((c) => ({ ref_id: c.refId, id: c.id }));
+  if (options.screenshot?.length) {
+    commands.screenshot = options.screenshot.map((c) => ({ ref_id: c.refId, pageno: c.pageno }));
+  }
+  if (options.finance?.length) {
+    commands.finance = options.finance.map((c) => ({
+      ticker: c.ticker,
+      type: c.type,
+      market: c.market,
+    }));
+  }
+  if (options.weather?.length) {
+    commands.weather = options.weather.map((c) => ({
+      location: c.location,
+      start: c.start,
+      duration: c.duration,
+    }));
+  }
+  if (options.sports?.length) {
+    commands.sports = options.sports.map((c) => ({
+      fn: c.fn,
+      league: c.league,
+      team: c.team,
+      opponent: c.opponent,
+      date_from: c.date_from,
+      date_to: c.date_to,
+      num_games: c.num_games,
+      locale: c.locale,
+    }));
+  }
+  if (options.time?.length) commands.time = options.time.map((c) => ({ utc_offset: c.utc_offset }));
+  if (responseLength) commands.response_length = responseLength;
+
+  const body: Record<string, unknown> = {
+    id: sessionId,
+    model,
+    input: buildInput(options),
+    commands,
+    settings: {
+      search_context_size: searchContextSize ?? "medium",
+      allowed_callers: ["direct"],
+      external_web_access: externalWebAccessForFreshness(freshness),
+    },
+  };
+  body.max_output_tokens = maxOutputTokens ?? 8000;
+
+  const bodyText = JSON.stringify(body);
+  let response = await transport.fetch(transport.resolveSearchEndpoint(), {
+    method: "POST",
+    headers,
+    body: bodyText,
+    signal,
+  });
+
+  if (!response.ok) {
+    let status = response.status;
+    let rawText = await response.text();
+    if (status === 403 && isCloudflareChallenge(rawText) && !signal?.aborted) {
+      await delay(750, signal);
+      if (signal?.aborted) {
+        throw new CodexError("timeout", "Codex standalone request was aborted before retry.");
+      }
+      response = await transport.fetch(transport.resolveSearchEndpoint(), {
+        method: "POST",
+        headers,
+        body: bodyText,
+        signal,
+      });
+      if (!response.ok) {
+        status = response.status;
+        rawText = await response.text();
+      }
+    }
+    if (!response.ok) {
+      const text = formatHttpErrorBody(rawText, "standalone");
+      throw new CodexError(
+        classifyHttpStatus(status),
+        `Codex standalone search request failed: HTTP ${status}: ${text}`,
+        status,
+      );
+    }
+  }
+
+  const data = (await response.json()) as StandaloneSearchResponse;
+  const text = typeof data.output === "string" ? data.output : "";
+  const refIds = extractRefIds(text);
+  const searchCalls = inferSearchCalls(options);
+
+  const result: CodexWebSearchResult = {
+    model,
+    text,
+    searchCalls,
+    citations: extractMarkdownCitations(text),
+    refIds,
+  };
+  if (data.encrypted_output !== undefined) result.encryptedOutput = data.encrypted_output;
+  return result;
+}
+
+async function delay(ms: number, signal: AbortSignal | undefined): Promise<void> {
+  if (signal?.aborted) return;
+  await new Promise<void>((resolve) => {
+    const timeout = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timeout);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+function buildInput(options: StandaloneCommandsOptions): unknown[] {
+  const texts: string[] = [];
+  options.searchQuery?.forEach((q) => texts.push(q.q));
+  options.imageQuery?.forEach((q) => texts.push(q.q));
+  options.open?.forEach((c) => texts.push(c.refId));
+  options.find?.forEach((c) => texts.push(`find "${c.pattern}" in ${c.refId}`));
+  options.click?.forEach((c) => texts.push(`click ${c.id} in ${c.refId}`));
+  options.screenshot?.forEach((c) => texts.push(`screenshot ${c.pageno} of ${c.refId}`));
+  options.finance?.forEach((c) => texts.push(`finance ${c.ticker} ${c.type} ${c.market ?? ""}`));
+  options.weather?.forEach((c) => texts.push(`weather ${c.location}`));
+  options.sports?.forEach((c) => texts.push(`sports ${c.fn} ${c.league}`));
+  options.time?.forEach((c) => texts.push(`time ${c.utc_offset}`));
+
+  const prompt = texts.filter(Boolean).join("\n");
+  return [
+    {
+      type: "message",
+      role: "user",
+      content: [{ type: "input_text", text: prompt }],
+    },
+  ];
+}
+
+function inferSearchCalls(options: StandaloneCommandsOptions): CodexSearchCall[] {
+  const calls: CodexSearchCall[] = [];
+  options.searchQuery?.forEach((q) =>
+    calls.push({ status: "completed", query: q.q, actionType: "search_query" }),
+  );
+  options.imageQuery?.forEach((q) =>
+    calls.push({ status: "completed", query: q.q, actionType: "image_query" }),
+  );
+  options.open?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "open_page" }),
+  );
+  options.find?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "find_in_page" }),
+  );
+  options.click?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "click" }),
+  );
+  options.screenshot?.forEach((c) =>
+    calls.push({ status: "completed", refId: c.refId, actionType: "screenshot" }),
+  );
+  options.finance?.forEach((c) =>
+    calls.push({ status: "completed", query: `${c.ticker}`, actionType: "finance" }),
+  );
+  options.weather?.forEach((c) =>
+    calls.push({ status: "completed", query: c.location, actionType: "weather" }),
+  );
+  options.sports?.forEach((c) =>
+    calls.push({ status: "completed", query: `${c.fn} ${c.league}`, actionType: "sports" }),
+  );
+  options.time?.forEach((c) =>
+    calls.push({ status: "completed", query: c.utc_offset, actionType: "time" }),
+  );
+  return calls;
+}
+
+const REF_ID_PATTERN = /\b(turn\d+(?:search|fetch|view)\d+)\b/g;
+
+function extractRefIds(text: string): Record<string, string> {
+  const refs: Record<string, string> = {};
+  for (const match of text.matchAll(REF_ID_PATTERN)) {
+    const refId = match[1];
+    if (refId) refs[refId] = refId;
+  }
+  return refs;
+}
+
+function extractMarkdownCitations(text: string): CodexCitation[] {
+  const citations = new Map<string, CodexCitation>();
+  const markdownLinkPattern = /\[([^\]\n]{1,200})\]\((https?:\/\/[^)\s]+)\)/g;
+  for (const match of text.matchAll(markdownLinkPattern)) {
+    const title = match[1]?.trim();
+    const url = match[2]?.trim();
+    if (!url || citations.has(url)) continue;
+    citations.set(url, { title: title || url, url, startIndex: match.index });
+  }
+  return [...citations.values()];
+}

package/src/modes/types.ts

@@ -0,0 +1,41 @@
+export type SearchContextSize = "low" | "medium" | "high";
+export type Freshness = "live" | "cached" | "indexed";
+export type StandaloneExternalWebAccess = boolean | "indexed";
+export type ResponseLength = "short" | "medium" | "long";
+
+export interface CodexCitation {
+  title?: string;
+  url: string;
+  startIndex?: number;
+  endIndex?: number;
+}
+
+export interface CodexSearchCall {
+  id?: string;
+  status?: string;
+  query?: string;
+  url?: string;
+  actionType?: string;
+  refId?: string;
+}
+
+export interface CodexWebSearchResult {
+  responseId?: string;
+  model: string;
+  text: string;
+  searchCalls: CodexSearchCall[];
+  citations: CodexCitation[];
+  refIds?: Record<string, string>;
+  usage?: {
+    inputTokens?: number;
+    outputTokens?: number;
+    totalTokens?: number;
+  };
+  encryptedOutput?: string;
+}
+
+export interface CodexModel {
+  id: string;
+  name?: string;
+  isDefault?: boolean;
+}

package/src/ref-store.ts

@@ -0,0 +1,74 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export interface RefStore {
+  resolveRefId(url: string): string | undefined;
+  remember(url: string, refId: string): Promise<void>;
+  load(sessionDir: string): Promise<void>;
+}
+
+interface StoredRefs {
+  urlToRefId: Record<string, string>;
+}
+
+const STORE_FILE = "pi-codex-search-refs.json";
+const PERSIST_QUEUES = new Map<string, Promise<void>>();
+
+export function createRefStore(): RefStore {
+  const urlToRefId = new Map<string, string>();
+  let sessionDir: string | undefined;
+
+  return {
+    resolveRefId(url: string): string | undefined {
+      return urlToRefId.get(url) ?? undefined;
+    },
+
+    async remember(url: string, refId: string): Promise<void> {
+      urlToRefId.set(url, refId);
+      if (sessionDir) {
+        await enqueuePersist(sessionDir, urlToRefId);
+      }
+    },
+
+    async load(dir: string): Promise<void> {
+      sessionDir = dir;
+      const stored = await loadStored(dir);
+      for (const [url, refId] of Object.entries(stored.urlToRefId)) {
+        urlToRefId.set(url, refId);
+      }
+    },
+  };
+}
+
+async function loadStored(dir: string): Promise<StoredRefs> {
+  try {
+    const raw = await readFile(join(dir, STORE_FILE), "utf-8");
+    const parsed = JSON.parse(raw) as StoredRefs;
+    return { urlToRefId: parsed.urlToRefId ?? {} };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return { urlToRefId: {} };
+    throw error;
+  }
+}
+
+async function enqueuePersist(dir: string, map: Map<string, string>): Promise<void> {
+  const previous = PERSIST_QUEUES.get(dir) ?? Promise.resolve();
+  const next = previous.catch(() => undefined).then(() => persistMerged(dir, map));
+  PERSIST_QUEUES.set(dir, next);
+  try {
+    await next;
+  } finally {
+    if (PERSIST_QUEUES.get(dir) === next) PERSIST_QUEUES.delete(dir);
+  }
+}
+
+async function persistMerged(dir: string, map: Map<string, string>): Promise<void> {
+  const current = await loadStored(dir);
+  const stored: StoredRefs = {
+    urlToRefId: {
+      ...current.urlToRefId,
+      ...Object.fromEntries(map),
+    },
+  };
+  await writeFile(join(dir, STORE_FILE), JSON.stringify(stored, null, 2), "utf-8");
+}

package/src/transport.ts

@@ -0,0 +1,110 @@
+/**
+ * Codex-faithful HTTP transport.
+ *
+ * Builds the same headers and cookie behavior used by the codex Rust client:
+ * - originator: codex_cli_rs
+ * - User-Agent: codex_cli_rs/{ver} (os ver; arch) terminal
+ * - Authorization: Bearer {token}
+ * - ChatGPT-Account-ID: {account_id}
+ * - ChatGPT Cloudflare cookie store on all outbound/inbound requests
+ */
+
+import { getCodexOriginator, buildCodexUserAgent } from "./ua.ts";
+import { wrapFetchWithCookies, type FetchLike } from "./cookies.ts";
+
+export interface TransportOptions {
+  token: string;
+  accountId: string;
+  baseUrl?: string;
+  fetchImpl?: FetchLike;
+}
+
+export const DEFAULT_BASE_URL = "https://chatgpt.com/backend-api";
+export const DEFAULT_CLIENT_VERSION = "1.0.0";
+
+export interface CodexTransport {
+  fetch: FetchLike;
+  baseUrl: string;
+  token: string;
+  accountId: string;
+  buildHeaders(accept: string): Headers;
+  resolveEndpoint(path: "models" | "responses"): string;
+  resolveSearchEndpoint(): string;
+}
+
+export function normalizeCodexBaseUrl(baseUrl: string | undefined): string {
+  const raw = baseUrl?.trim() ? baseUrl : DEFAULT_BASE_URL;
+  let normalized = raw.replace(/\/+$/, "");
+  if (normalized.endsWith("/codex/responses")) {
+    normalized = normalized.slice(0, -"/codex/responses".length);
+  }
+  if (normalized.endsWith("/codex")) {
+    normalized = normalized.slice(0, -"/codex".length);
+  }
+  return normalized;
+}
+
+function isOpenAiRootBaseUrl(baseUrl: string): boolean {
+  try {
+    const url = new URL(baseUrl);
+    return url.hostname === "api.openai.com" && (url.pathname === "" || url.pathname === "/");
+  } catch {
+    return false;
+  }
+}
+
+export function resolveCodexEndpoint(
+  baseUrl: string | undefined,
+  path: "models" | "responses",
+): string {
+  return `${normalizeCodexBaseUrl(baseUrl)}/codex/${path}`;
+}
+
+export function resolveCodexSearchEndpoint(baseUrl: string | undefined): string {
+  const raw = baseUrl?.trim() ? baseUrl : DEFAULT_BASE_URL;
+  let normalized = raw.replace(/\/+$/, "");
+  if (normalized.endsWith("/codex/responses")) {
+    normalized = normalized.slice(0, -"/responses".length);
+  }
+  if (normalized.endsWith("/codex/models")) {
+    normalized = normalized.slice(0, -"/models".length);
+  }
+  if (normalized.endsWith("/codex/alpha/search") || normalized.endsWith("/alpha/search")) {
+    return normalized;
+  }
+  if (normalized.endsWith("/codex")) return `${normalized}/alpha/search`;
+  if (normalized.endsWith("/v1")) return `${normalized}/alpha/search`;
+  if (isOpenAiRootBaseUrl(normalized)) return `${normalized}/v1/alpha/search`;
+  return `${normalized}/codex/alpha/search`;
+}
+
+export function createTransport(options: TransportOptions): CodexTransport {
+  const baseUrl = normalizeCodexBaseUrl(options.baseUrl);
+  const rawFetch: FetchLike = options.fetchImpl ?? globalThis.fetch.bind(globalThis);
+  const fetch = wrapFetchWithCookies(rawFetch);
+
+  return {
+    fetch,
+    baseUrl,
+    token: options.token,
+    accountId: options.accountId,
+    buildHeaders(accept: string): Headers {
+      const headers = new Headers();
+      headers.set("Authorization", `Bearer ${options.token}`);
+      headers.set("chatgpt-account-id", options.accountId);
+      headers.set("originator", getCodexOriginator());
+      headers.set("accept", accept);
+      if (accept === "text/event-stream") {
+        headers.set("content-type", "application/json");
+      }
+      headers.set("User-Agent", buildCodexUserAgent());
+      return headers;
+    },
+    resolveEndpoint(path) {
+      return resolveCodexEndpoint(baseUrl, path);
+    },
+    resolveSearchEndpoint() {
+      return resolveCodexSearchEndpoint(baseUrl);
+    },
+  };
+}

package/src/ua.ts

@@ -0,0 +1,67 @@
+import { release } from "node:os";
+
+export const DEFAULT_CODEX_VERSION = "0.143.0";
+export const DEFAULT_CODEX_ORIGINATOR = "codex_cli_rs";
+
+function mapPlatform(platform: string): string {
+  switch (platform) {
+    case "darwin":
+      return "Mac OS";
+    case "win32":
+      return "Windows";
+    case "linux":
+      return "Linux";
+    case "freebsd":
+      return "FreeBSD";
+    default:
+      return platform;
+  }
+}
+
+function mapArch(arch: string): string {
+  switch (arch) {
+    case "x64":
+      return "x86_64";
+    case "arm64":
+      return "arm64";
+    case "arm":
+      return "arm";
+    default:
+      return arch;
+  }
+}
+
+function terminalUserAgent(): string {
+  const program = process.env.TERM_PROGRAM;
+  if (program) {
+    const version = process.env.TERM_PROGRAM_VERSION;
+    const suffix = version ? ` ${version}` : "";
+    return `${program}${suffix}`.trim();
+  }
+  if (process.env.WT_SESSION) {
+    return "WindowsTerminal";
+  }
+  if (process.env.KITTY_WINDOW_ID) {
+    return "kitty";
+  }
+  if (process.env.TMUX) {
+    return "tmux";
+  }
+  const term = process.env.TERM;
+  if (term && term !== "dumb") {
+    return term;
+  }
+  return "unknown";
+}
+
+export function buildCodexUserAgent(version = DEFAULT_CODEX_VERSION): string {
+  const osType = mapPlatform(process.platform);
+  const osVersion = release();
+  const arch = mapArch(process.arch);
+  const terminal = terminalUserAgent();
+  return `${DEFAULT_CODEX_ORIGINATOR}/${version} (${osType} ${osVersion}; ${arch}) ${terminal}`;
+}
+
+export function getCodexOriginator(): string {
+  return DEFAULT_CODEX_ORIGINATOR;
+}